mirror of
https://github.com/Such-Software/adaptive-multi-protocol-gateway.git
synced 2026-10-17 07:31:32 +02:00
Add Gemini renderer and semantic audit
This commit is contained in:
@@ -97,6 +97,7 @@ The current implementation is dependency-free Python:
|
|||||||
python3 -m ampg --config examples/wownero.gateway.toml plan
|
python3 -m ampg --config examples/wownero.gateway.toml plan
|
||||||
python3 -m ampg --config examples/wownero.gateway.toml plan --write-artifacts
|
python3 -m ampg --config examples/wownero.gateway.toml plan --write-artifacts
|
||||||
python3 -m ampg --config examples/wownero.gateway.toml build
|
python3 -m ampg --config examples/wownero.gateway.toml build
|
||||||
|
python3 -m ampg --config examples/wownero.gateway.toml audit
|
||||||
python3 -m ampg --config examples/i2p-only.gateway.toml build
|
python3 -m ampg --config examples/i2p-only.gateway.toml build
|
||||||
python3 -m ampg docs generate
|
python3 -m ampg docs generate
|
||||||
python3 -m unittest discover -s tests
|
python3 -m unittest discover -s tests
|
||||||
|
|||||||
+125
@@ -0,0 +1,125 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .build import _iter_source_files
|
||||||
|
from .config import GatewayConfig, SiteConfig
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AuditIssue:
|
||||||
|
site_id: str
|
||||||
|
path: Path
|
||||||
|
severity: str
|
||||||
|
code: str
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
def audit_gateway(config: GatewayConfig) -> list[AuditIssue]:
|
||||||
|
issues: list[AuditIssue] = []
|
||||||
|
for site in config.sites:
|
||||||
|
issues.extend(audit_site(site))
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
def audit_site(site: SiteConfig) -> list[AuditIssue]:
|
||||||
|
issues: list[AuditIssue] = []
|
||||||
|
for path in _iter_source_files(site.source.path):
|
||||||
|
if path.suffix.lower() not in {".html", ".htm"}:
|
||||||
|
continue
|
||||||
|
html = path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
rel_path = path.relative_to(site.source.path)
|
||||||
|
page_issues = SemanticHtmlAuditor(site.id, rel_path).audit(html)
|
||||||
|
issues.extend(page_issues)
|
||||||
|
return issues
|
||||||
|
|
||||||
|
|
||||||
|
class SemanticHtmlAuditor(HTMLParser):
|
||||||
|
def __init__(self, site_id: str, path: Path) -> None:
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.site_id = site_id
|
||||||
|
self.path = path
|
||||||
|
self.issues: list[AuditIssue] = []
|
||||||
|
self.last_heading_level = 0
|
||||||
|
self.h1_count = 0
|
||||||
|
self.anchor_stack: list[tuple[str, list[str], bool]] = []
|
||||||
|
self.skip_stack: list[str] = []
|
||||||
|
|
||||||
|
def audit(self, html: str) -> list[AuditIssue]:
|
||||||
|
self.feed(html)
|
||||||
|
self.close()
|
||||||
|
if self.h1_count == 0:
|
||||||
|
self._warn("missing_h1", "page has no h1 heading")
|
||||||
|
elif self.h1_count > 1:
|
||||||
|
self._warn("multiple_h1", f"page has {self.h1_count} h1 headings")
|
||||||
|
return self.issues
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
|
tag = tag.lower()
|
||||||
|
attrs_dict = {name.lower(): value for name, value in attrs}
|
||||||
|
if tag in {"script", "style"}:
|
||||||
|
self.skip_stack.append(tag)
|
||||||
|
return
|
||||||
|
if self.skip_stack:
|
||||||
|
return
|
||||||
|
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||||
|
self._handle_heading(int(tag[1]))
|
||||||
|
elif tag == "img":
|
||||||
|
alt = attrs_dict.get("alt")
|
||||||
|
if alt is None or not _normalize_text(alt):
|
||||||
|
self._warn("missing_alt", "image is missing meaningful alt text")
|
||||||
|
if self.anchor_stack:
|
||||||
|
href, text, _ = self.anchor_stack.pop()
|
||||||
|
self.anchor_stack.append((href, text, True))
|
||||||
|
elif tag == "a":
|
||||||
|
href = attrs_dict.get("href")
|
||||||
|
if href:
|
||||||
|
self.anchor_stack.append((href, [], False))
|
||||||
|
|
||||||
|
def handle_endtag(self, tag: str) -> None:
|
||||||
|
tag = tag.lower()
|
||||||
|
if self.skip_stack:
|
||||||
|
if self.skip_stack[-1] == tag:
|
||||||
|
self.skip_stack.pop()
|
||||||
|
return
|
||||||
|
if tag == "a" and self.anchor_stack:
|
||||||
|
href, text_parts, has_image = self.anchor_stack.pop()
|
||||||
|
text = _normalize_text("".join(text_parts))
|
||||||
|
if not text and not has_image and not href.startswith("#"):
|
||||||
|
self._warn("empty_link_text", f"link to {href!r} has no text")
|
||||||
|
|
||||||
|
def handle_data(self, data: str) -> None:
|
||||||
|
if self.skip_stack:
|
||||||
|
return
|
||||||
|
if self.anchor_stack:
|
||||||
|
href, text_parts, has_image = self.anchor_stack.pop()
|
||||||
|
text_parts.append(data)
|
||||||
|
self.anchor_stack.append((href, text_parts, has_image))
|
||||||
|
|
||||||
|
def _handle_heading(self, level: int) -> None:
|
||||||
|
if level == 1:
|
||||||
|
self.h1_count += 1
|
||||||
|
if self.last_heading_level and level > self.last_heading_level + 1:
|
||||||
|
self._warn(
|
||||||
|
"heading_level_skip",
|
||||||
|
f"heading jumps from h{self.last_heading_level} to h{level}",
|
||||||
|
)
|
||||||
|
self.last_heading_level = level
|
||||||
|
|
||||||
|
def _warn(self, code: str, message: str) -> None:
|
||||||
|
self.issues.append(
|
||||||
|
AuditIssue(
|
||||||
|
site_id=self.site_id,
|
||||||
|
path=self.path,
|
||||||
|
severity="warn",
|
||||||
|
code=code,
|
||||||
|
message=message,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_text(value: str) -> str:
|
||||||
|
return re.sub(r"\s+", " ", value).strip()
|
||||||
+52
-1
@@ -5,7 +5,7 @@ from pathlib import Path
|
|||||||
import shutil
|
import shutil
|
||||||
|
|
||||||
from .config import GatewayConfig, ProtocolConfig, SiteConfig
|
from .config import GatewayConfig, ProtocolConfig, SiteConfig
|
||||||
from .renderers import PrivacyRenderStats, render_privacy_html
|
from .renderers import PrivacyRenderStats, render_gemtext, render_privacy_html
|
||||||
|
|
||||||
|
|
||||||
IGNORED_DIRS = {".git", ".hg", ".svn", ".vscode", "__pycache__"}
|
IGNORED_DIRS = {".git", ".hg", ".svn", ".vscode", "__pycache__"}
|
||||||
@@ -41,6 +41,8 @@ def build_protocol(site: SiteConfig, protocol: ProtocolConfig) -> BuildResult:
|
|||||||
return _copy_clearnet(site, protocol, output_root)
|
return _copy_clearnet(site, protocol, output_root)
|
||||||
if protocol.renderer == "privacy-html":
|
if protocol.renderer == "privacy-html":
|
||||||
return _render_privacy_html_tree(site, protocol, output_root)
|
return _render_privacy_html_tree(site, protocol, output_root)
|
||||||
|
if protocol.renderer == "gemtext":
|
||||||
|
return _render_gemtext_tree(site, protocol, output_root)
|
||||||
raise ValueError(f"{site.id}/{protocol.name}: unsupported renderer {protocol.renderer!r}")
|
raise ValueError(f"{site.id}/{protocol.name}: unsupported renderer {protocol.renderer!r}")
|
||||||
|
|
||||||
|
|
||||||
@@ -100,6 +102,39 @@ def _render_privacy_html_tree(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _render_gemtext_tree(
|
||||||
|
site: SiteConfig, protocol: ProtocolConfig, output_root: Path
|
||||||
|
) -> BuildResult:
|
||||||
|
files_written = 0
|
||||||
|
files_skipped = 0
|
||||||
|
max_asset_bytes = int(protocol.options.get("max_asset_bytes", 1024 * 1024))
|
||||||
|
for source_path in _iter_source_files(site.source.path):
|
||||||
|
if source_path.suffix.lower() in {".html", ".htm"}:
|
||||||
|
target_path = _target_with_suffix(site.source.path, output_root, source_path, ".gmi")
|
||||||
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
html = source_path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
target_path.write_text(render_gemtext(html, rewrite_link=_rewrite_gemtext_link), encoding="utf-8")
|
||||||
|
elif source_path.suffix.lower() in {".js", ".css", ".map"}:
|
||||||
|
files_skipped += 1
|
||||||
|
continue
|
||||||
|
elif source_path.stat().st_size > max_asset_bytes:
|
||||||
|
files_skipped += 1
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
target_path = output_root / source_path.relative_to(site.source.path)
|
||||||
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
shutil.copy2(source_path, target_path)
|
||||||
|
files_written += 1
|
||||||
|
return BuildResult(
|
||||||
|
site_id=site.id,
|
||||||
|
protocol=protocol.name,
|
||||||
|
renderer=protocol.renderer,
|
||||||
|
output_root=output_root,
|
||||||
|
files_written=files_written,
|
||||||
|
files_skipped=files_skipped,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _iter_source_files(root: Path):
|
def _iter_source_files(root: Path):
|
||||||
for path in sorted(root.rglob("*")):
|
for path in sorted(root.rglob("*")):
|
||||||
if not path.is_file():
|
if not path.is_file():
|
||||||
@@ -131,3 +166,19 @@ def _prepare_output_root(output_root: Path) -> None:
|
|||||||
shutil.rmtree(output_root)
|
shutil.rmtree(output_root)
|
||||||
output_root.mkdir(parents=True, exist_ok=True)
|
output_root.mkdir(parents=True, exist_ok=True)
|
||||||
marker.write_text("generated by AMPG\n", encoding="utf-8")
|
marker.write_text("generated by AMPG\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _target_with_suffix(source_root: Path, output_root: Path, source_path: Path, suffix: str) -> Path:
|
||||||
|
rel_path = source_path.relative_to(source_root)
|
||||||
|
return (output_root / rel_path).with_suffix(suffix)
|
||||||
|
|
||||||
|
|
||||||
|
def _rewrite_gemtext_link(href: str) -> str:
|
||||||
|
if href.startswith("#"):
|
||||||
|
return href
|
||||||
|
if href.startswith(("http://", "https://", "gemini://", "mailto:")):
|
||||||
|
return href
|
||||||
|
path, sep, fragment = href.partition("#")
|
||||||
|
if path.endswith(".html") or path.endswith(".htm"):
|
||||||
|
path = path.rsplit(".", 1)[0] + ".gmi"
|
||||||
|
return path + (sep + fragment if sep else "")
|
||||||
|
|||||||
+26
@@ -4,6 +4,7 @@ import argparse
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
from .audit import audit_gateway
|
||||||
from .build import build_gateway
|
from .build import build_gateway
|
||||||
from .config import load_config
|
from .config import load_config
|
||||||
from .docsgen import generate_docs
|
from .docsgen import generate_docs
|
||||||
@@ -26,6 +27,12 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
help="Write generated config snippets to the configured plan root.",
|
help="Write generated config snippets to the configured plan root.",
|
||||||
)
|
)
|
||||||
subcommands.add_parser("build", help="Build enabled protocol outputs.")
|
subcommands.add_parser("build", help="Build enabled protocol outputs.")
|
||||||
|
audit_parser = subcommands.add_parser("audit", help="Audit source HTML quality.")
|
||||||
|
audit_parser.add_argument(
|
||||||
|
"--fail-on-warn",
|
||||||
|
action="store_true",
|
||||||
|
help="Exit non-zero when warnings are found.",
|
||||||
|
)
|
||||||
docs_parser = subcommands.add_parser("docs", help="Generate or check generated docs.")
|
docs_parser = subcommands.add_parser("docs", help="Generate or check generated docs.")
|
||||||
docs_subcommands = docs_parser.add_subparsers(dest="docs_command", required=True)
|
docs_subcommands = docs_parser.add_subparsers(dest="docs_command", required=True)
|
||||||
docs_generate = docs_subcommands.add_parser("generate", help="Generate docs from code.")
|
docs_generate = docs_subcommands.add_parser("generate", help="Generate docs from code.")
|
||||||
@@ -45,6 +52,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
return _cmd_plan(config, write_artifacts=args.write_artifacts)
|
return _cmd_plan(config, write_artifacts=args.write_artifacts)
|
||||||
if args.command == "build":
|
if args.command == "build":
|
||||||
return _cmd_build(config)
|
return _cmd_build(config)
|
||||||
|
if args.command == "audit":
|
||||||
|
return _cmd_audit(config, fail_on_warn=args.fail_on_warn)
|
||||||
except Exception as exc: # noqa: BLE001 - CLI should print concise failures.
|
except Exception as exc: # noqa: BLE001 - CLI should print concise failures.
|
||||||
print(f"AMPG status=error message={exc}", file=sys.stderr)
|
print(f"AMPG status=error message={exc}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
@@ -96,6 +105,23 @@ def _cmd_build(config) -> int:
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_audit(config, *, fail_on_warn: bool) -> int:
|
||||||
|
issues = audit_gateway(config)
|
||||||
|
for issue in issues:
|
||||||
|
print(
|
||||||
|
"AMPG_AUDIT "
|
||||||
|
f"site={issue.site_id} "
|
||||||
|
f"severity={issue.severity} "
|
||||||
|
f"code={issue.code} "
|
||||||
|
f"path={issue.path} "
|
||||||
|
f"message=\"{issue.message}\""
|
||||||
|
)
|
||||||
|
print(f"AMPG_AUDIT_SUMMARY issues={len(issues)}")
|
||||||
|
if issues and fail_on_warn:
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def _cmd_docs(args) -> int:
|
def _cmd_docs(args) -> int:
|
||||||
if args.docs_command == "generate":
|
if args.docs_command == "generate":
|
||||||
changed = generate_docs(Path.cwd(), check=args.check)
|
changed = generate_docs(Path.cwd(), check=args.check)
|
||||||
|
|||||||
@@ -132,6 +132,18 @@ RENDER_PROFILES = (
|
|||||||
"Removes remote src/poster/link asset references.",
|
"Removes remote src/poster/link asset references.",
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
RenderProfile(
|
||||||
|
name="gemtext",
|
||||||
|
summary="Text-first Gemini output from semantic HTML.",
|
||||||
|
output="Gemtext plus linked/downloadable assets",
|
||||||
|
defaults=(
|
||||||
|
"Converts HTML headings to Gemtext heading lines.",
|
||||||
|
"Converts paragraphs and list items to flowing text.",
|
||||||
|
"Converts anchors and images to Gemtext link lines.",
|
||||||
|
"Rewrites local .html links to .gmi links.",
|
||||||
|
"Skips JavaScript, CSS, source maps, and oversized assets.",
|
||||||
|
),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -166,4 +178,14 @@ DAEMON_ADAPTERS = (
|
|||||||
"Existing RPC/P2P tunnel keys are not reused for web publishing.",
|
"Existing RPC/P2P tunnel keys are not reused for web publishing.",
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
DaemonAdapter(
|
||||||
|
daemon=DEFAULT_DAEMONS["gemini"],
|
||||||
|
protocols=("gemini",),
|
||||||
|
default_policy=DEFAULT_POLICIES["gemini"],
|
||||||
|
generated_artifacts=("Agate plan values",),
|
||||||
|
notes=(
|
||||||
|
"Gemini serves generated Gemtext output directly.",
|
||||||
|
"Certificate and key paths are plan values until install/apply support exists.",
|
||||||
|
),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -74,6 +74,10 @@ def plan_artifacts(site: SiteConfig, protocol: ProtocolConfig) -> list[PlanArtif
|
|||||||
artifacts.append(_i2pd_artifact(site, protocol))
|
artifacts.append(_i2pd_artifact(site, protocol))
|
||||||
artifacts.append(_nginx_loopback_artifact(site, protocol))
|
artifacts.append(_nginx_loopback_artifact(site, protocol))
|
||||||
return artifacts
|
return artifacts
|
||||||
|
if protocol.name == "gemini":
|
||||||
|
if protocol.daemon == "agate":
|
||||||
|
return [_agate_artifact(site, protocol)]
|
||||||
|
return []
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
@@ -155,3 +159,19 @@ port = {port}
|
|||||||
keys = {key_file}
|
keys = {key_file}
|
||||||
"""
|
"""
|
||||||
return PlanArtifact(path=path, content=content)
|
return PlanArtifact(path=path, content=content)
|
||||||
|
|
||||||
|
|
||||||
|
def _agate_artifact(site: SiteConfig, protocol: ProtocolConfig) -> PlanArtifact:
|
||||||
|
port = int(protocol.options.get("port", 1965))
|
||||||
|
cert_path = protocol.options.get("certificate_path", f"/var/lib/ampg/gemini/{site.id}.crt")
|
||||||
|
key_path = protocol.options.get("key_path", f"/var/lib/ampg/gemini/{site.id}.key")
|
||||||
|
path = _protocol_plan_dir(site, protocol) / "agate-plan.txt"
|
||||||
|
content = f"""# Generated by AMPG. Review before installing.
|
||||||
|
daemon = agate
|
||||||
|
hostname = {site.domain}
|
||||||
|
listen = [::]:{port}
|
||||||
|
content_root = {site.outputs.root / protocol.name}
|
||||||
|
certificate = {cert_path}
|
||||||
|
private_key = {key_path}
|
||||||
|
"""
|
||||||
|
return PlanArtifact(path=path, content=content)
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from html import escape
|
from html import escape
|
||||||
from html.parser import HTMLParser
|
from html.parser import HTMLParser
|
||||||
|
import re
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
|
||||||
VOID_ELEMENTS = {
|
VOID_ELEMENTS = {
|
||||||
@@ -158,6 +160,118 @@ def render_privacy_html(html: str) -> tuple[str, PrivacyRenderStats]:
|
|||||||
return PrivacyHtmlRenderer().render(html)
|
return PrivacyHtmlRenderer().render(html)
|
||||||
|
|
||||||
|
|
||||||
|
class GemtextRenderer(HTMLParser):
|
||||||
|
def __init__(self, rewrite_link: Callable[[str], str] | None = None) -> None:
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.rewrite_link = rewrite_link or (lambda href: href)
|
||||||
|
self.lines: list[str] = []
|
||||||
|
self.text_parts: list[str] = []
|
||||||
|
self.links: list[tuple[str, str]] = []
|
||||||
|
self.skip_stack: list[str] = []
|
||||||
|
self.heading_level: int | None = None
|
||||||
|
self.list_depth = 0
|
||||||
|
self.anchor_stack: list[tuple[str, list[str]]] = []
|
||||||
|
self.in_pre = False
|
||||||
|
self.pre_parts: list[str] = []
|
||||||
|
|
||||||
|
def render(self, html: str) -> str:
|
||||||
|
self.feed(html)
|
||||||
|
self.close()
|
||||||
|
self._flush_block()
|
||||||
|
return _squash_blank_lines(self.lines)
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||||
|
tag = tag.lower()
|
||||||
|
attrs_dict = {name.lower(): value for name, value in attrs}
|
||||||
|
if tag in ACTIVE_TAGS or tag in {"nav", "noscript", "svg"}:
|
||||||
|
self.skip_stack.append(tag)
|
||||||
|
return
|
||||||
|
if self.skip_stack:
|
||||||
|
return
|
||||||
|
if tag in {"p", "div", "section", "article", "header", "footer"}:
|
||||||
|
self._flush_block()
|
||||||
|
elif tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||||
|
self._flush_block()
|
||||||
|
self.heading_level = min(int(tag[1]), 3)
|
||||||
|
elif tag == "li":
|
||||||
|
self._flush_block()
|
||||||
|
self.list_depth += 1
|
||||||
|
elif tag == "br":
|
||||||
|
self._append_text(" ")
|
||||||
|
elif tag == "a":
|
||||||
|
href = attrs_dict.get("href")
|
||||||
|
if href:
|
||||||
|
self.anchor_stack.append((href, []))
|
||||||
|
elif tag == "img":
|
||||||
|
src = attrs_dict.get("src")
|
||||||
|
alt = attrs_dict.get("alt") or "image"
|
||||||
|
if src:
|
||||||
|
self.links.append((self.rewrite_link(src), f"Image: {_normalize_text(alt)}"))
|
||||||
|
elif tag == "pre":
|
||||||
|
self._flush_block()
|
||||||
|
self.in_pre = True
|
||||||
|
self.pre_parts = []
|
||||||
|
|
||||||
|
def handle_endtag(self, tag: str) -> None:
|
||||||
|
tag = tag.lower()
|
||||||
|
if self.skip_stack:
|
||||||
|
if self.skip_stack[-1] == tag:
|
||||||
|
self.skip_stack.pop()
|
||||||
|
return
|
||||||
|
if tag in {"h1", "h2", "h3", "h4", "h5", "h6"}:
|
||||||
|
self._flush_block()
|
||||||
|
self.heading_level = None
|
||||||
|
elif tag in {"p", "div", "section", "article", "header", "footer"}:
|
||||||
|
self._flush_block()
|
||||||
|
elif tag == "li":
|
||||||
|
self._flush_block(prefix="- ")
|
||||||
|
self.list_depth = max(0, self.list_depth - 1)
|
||||||
|
elif tag == "a" and self.anchor_stack:
|
||||||
|
href, parts = self.anchor_stack.pop()
|
||||||
|
label = _normalize_text("".join(parts)) or href
|
||||||
|
self.links.append((self.rewrite_link(href), label))
|
||||||
|
elif tag == "pre" and self.in_pre:
|
||||||
|
self.lines.append("```")
|
||||||
|
self.lines.extend("".join(self.pre_parts).strip("\n").splitlines())
|
||||||
|
self.lines.append("```")
|
||||||
|
self.lines.append("")
|
||||||
|
self.in_pre = False
|
||||||
|
self.pre_parts = []
|
||||||
|
|
||||||
|
def handle_data(self, data: str) -> None:
|
||||||
|
if self.skip_stack:
|
||||||
|
return
|
||||||
|
if self.in_pre:
|
||||||
|
self.pre_parts.append(data)
|
||||||
|
return
|
||||||
|
self._append_text(data)
|
||||||
|
|
||||||
|
def _append_text(self, data: str) -> None:
|
||||||
|
if self.anchor_stack:
|
||||||
|
self.anchor_stack[-1][1].append(data)
|
||||||
|
self.text_parts.append(data)
|
||||||
|
|
||||||
|
def _flush_block(self, prefix: str = "") -> None:
|
||||||
|
text = _normalize_text("".join(self.text_parts))
|
||||||
|
if text:
|
||||||
|
if self.heading_level is not None:
|
||||||
|
self.lines.append(f"{'#' * self.heading_level} {text}")
|
||||||
|
else:
|
||||||
|
self.lines.append(f"{prefix}{text}")
|
||||||
|
for href, label in self.links:
|
||||||
|
if href.startswith("#"):
|
||||||
|
continue
|
||||||
|
self.lines.append(f"=> {href} {label}")
|
||||||
|
if text or self.links:
|
||||||
|
self.lines.append("")
|
||||||
|
self.text_parts = []
|
||||||
|
self.links = []
|
||||||
|
|
||||||
|
|
||||||
|
def render_gemtext(html: str, rewrite_link: Callable[[str], str] | None = None) -> str:
|
||||||
|
return GemtextRenderer(rewrite_link=rewrite_link).render(html)
|
||||||
|
|
||||||
|
|
||||||
def _format_attrs(attrs: list[tuple[str, str | None]]) -> str:
|
def _format_attrs(attrs: list[tuple[str, str | None]]) -> str:
|
||||||
if not attrs:
|
if not attrs:
|
||||||
return ""
|
return ""
|
||||||
@@ -173,3 +287,21 @@ def _format_attrs(attrs: list[tuple[str, str | None]]) -> str:
|
|||||||
def _is_remote_url(value: str) -> bool:
|
def _is_remote_url(value: str) -> bool:
|
||||||
lower = value.lower()
|
lower = value.lower()
|
||||||
return lower.startswith("http://") or lower.startswith("https://") or lower.startswith("//")
|
return lower.startswith("http://") or lower.startswith("https://") or lower.startswith("//")
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_text(value: str) -> str:
|
||||||
|
return re.sub(r"\s+", " ", value).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _squash_blank_lines(lines: list[str]) -> str:
|
||||||
|
output: list[str] = []
|
||||||
|
previous_blank = False
|
||||||
|
for line in lines:
|
||||||
|
blank = not line.strip()
|
||||||
|
if blank and previous_blank:
|
||||||
|
continue
|
||||||
|
output.append(line.rstrip())
|
||||||
|
previous_blank = blank
|
||||||
|
while output and not output[-1]:
|
||||||
|
output.pop()
|
||||||
|
return "\n".join(output) + "\n"
|
||||||
|
|||||||
@@ -32,6 +32,17 @@ source site
|
|||||||
: Reads the built output of another static site generator. AMPG does not own that
|
: Reads the built output of another static site generator. AMPG does not own that
|
||||||
generator; it adapts the generated files.
|
generator; it adapts the generated files.
|
||||||
|
|
||||||
|
## Source quality
|
||||||
|
|
||||||
|
AMPG gets better results from semantic, accessible source documents. Authors should use
|
||||||
|
one `h1`, ordered heading levels, meaningful link text, alt text for images, and ordinary
|
||||||
|
flowing document structure before decorative layout. This keeps clearnet accessible and
|
||||||
|
gives constrained renderers a clean outline for Gemtext, Micron, and other text-first
|
||||||
|
targets.
|
||||||
|
|
||||||
|
`ampg audit` reports source-quality warnings. The audit is advisory by default and can be
|
||||||
|
made strict in CI with `--fail-on-warn`.
|
||||||
|
|
||||||
## Content graph
|
## Content graph
|
||||||
|
|
||||||
The internal graph is intentionally small:
|
The internal graph is intentionally small:
|
||||||
|
|||||||
@@ -59,11 +59,12 @@ max_asset_bytes = 1048576
|
|||||||
script_policy = "strip"
|
script_policy = "strip"
|
||||||
|
|
||||||
[site.protocols.gemini]
|
[site.protocols.gemini]
|
||||||
enabled = false
|
enabled = true
|
||||||
renderer = "gemtext"
|
renderer = "gemtext"
|
||||||
daemon = "agate"
|
daemon = "agate"
|
||||||
daemon_policy = "auto"
|
daemon_policy = "auto"
|
||||||
port = 1965
|
port = 1965
|
||||||
|
max_asset_bytes = 1048576
|
||||||
|
|
||||||
[site.protocols.reticulum]
|
[site.protocols.reticulum]
|
||||||
enabled = false
|
enabled = false
|
||||||
@@ -103,6 +104,8 @@ workflow.
|
|||||||
- one `.gmi` page per HTML or Markdown page.
|
- one `.gmi` page per HTML or Markdown page.
|
||||||
- inline links emitted after the paragraph that referenced them.
|
- inline links emitted after the paragraph that referenced them.
|
||||||
- images and PDFs emitted as links with file size metadata when known.
|
- images and PDFs emitted as links with file size metadata when known.
|
||||||
|
- local `.html` links rewritten to `.gmi`.
|
||||||
|
- JavaScript, CSS, source maps, and oversized assets skipped.
|
||||||
|
|
||||||
`micron` defaults:
|
`micron` defaults:
|
||||||
|
|
||||||
@@ -216,6 +219,7 @@ Generated snippets are not installed by `plan`:
|
|||||||
- nginx server blocks for static HTTP roots.
|
- nginx server blocks for static HTTP roots.
|
||||||
- Tor hidden-service snippets for loopback HTTP targets.
|
- Tor hidden-service snippets for loopback HTTP targets.
|
||||||
- i2pd server tunnel snippets for I2P web publishing.
|
- i2pd server tunnel snippets for I2P web publishing.
|
||||||
|
- Agate plan values for Gemini publishing.
|
||||||
|
|
||||||
I2P web tunnels should use a web-specific key file such as `wownero-web.dat`; AMPG does
|
I2P web tunnels should use a web-specific key file such as `wownero-web.dat`; AMPG does
|
||||||
not reuse daemon RPC/P2P tunnel keys for web publishing.
|
not reuse daemon RPC/P2P tunnel keys for web publishing.
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ and include backup warnings in the plan.
|
|||||||
Default daemon: Agate.
|
Default daemon: Agate.
|
||||||
|
|
||||||
Gemini is static-friendly and can be managed safely. AMPG generates `.gmi` output,
|
Gemini is static-friendly and can be managed safely. AMPG generates `.gmi` output,
|
||||||
certificates when requested, and an Agate config pointing at the Gemini output root.
|
certificates when requested, and Agate plan values pointing at the Gemini output root.
|
||||||
|
|
||||||
### Reticulum
|
### Reticulum
|
||||||
|
|
||||||
|
|||||||
@@ -34,3 +34,13 @@ Notes:
|
|||||||
Notes:
|
Notes:
|
||||||
- Web tunnels should use web-specific key files.
|
- Web tunnels should use web-specific key files.
|
||||||
- Existing RPC/P2P tunnel keys are not reused for web publishing.
|
- Existing RPC/P2P tunnel keys are not reused for web publishing.
|
||||||
|
|
||||||
|
## agate
|
||||||
|
|
||||||
|
- Protocols: gemini
|
||||||
|
- Default policy: `auto`
|
||||||
|
- Generated artifacts: Agate plan values
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- Gemini serves generated Gemtext output directly.
|
||||||
|
- Certificate and key paths are plan values until install/apply support exists.
|
||||||
|
|||||||
@@ -29,3 +29,16 @@ Defaults:
|
|||||||
- Skips JavaScript and source-map files.
|
- Skips JavaScript and source-map files.
|
||||||
- Skips oversized assets above max_asset_bytes.
|
- Skips oversized assets above max_asset_bytes.
|
||||||
- Removes remote src/poster/link asset references.
|
- Removes remote src/poster/link asset references.
|
||||||
|
|
||||||
|
## gemtext
|
||||||
|
|
||||||
|
Text-first Gemini output from semantic HTML.
|
||||||
|
|
||||||
|
- Output: Gemtext plus linked/downloadable assets
|
||||||
|
|
||||||
|
Defaults:
|
||||||
|
- Converts HTML headings to Gemtext heading lines.
|
||||||
|
- Converts paragraphs and list items to flowing text.
|
||||||
|
- Converts anchors and images to Gemtext link lines.
|
||||||
|
- Rewrites local .html links to .gmi links.
|
||||||
|
- Skips JavaScript, CSS, source maps, and oversized assets.
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ daemon_policy = "auto"
|
|||||||
## Acceptance Criteria
|
## Acceptance Criteria
|
||||||
|
|
||||||
- `ampg plan` identifies the Wownero source tree and selected protocols.
|
- `ampg plan` identifies the Wownero source tree and selected protocols.
|
||||||
- `ampg build` creates clearnet and Tor output without requiring Markdown.
|
- `ampg build` creates clearnet, Tor, I2P, and Gemini output without requiring Markdown.
|
||||||
- Tor output contains no `<script>` tags or inline event handlers.
|
- Tor output contains no `<script>` tags or inline event handlers.
|
||||||
- Generated Gemini output has readable headings, paragraphs, and links.
|
- Generated Gemini output has readable headings, paragraphs, and links.
|
||||||
- Generated Micron output fits terminal-first browsing.
|
- Generated Micron output fits terminal-first browsing.
|
||||||
|
|||||||
@@ -50,11 +50,12 @@ max_asset_bytes = 1048576
|
|||||||
script_policy = "strip"
|
script_policy = "strip"
|
||||||
|
|
||||||
[site.protocols.gemini]
|
[site.protocols.gemini]
|
||||||
enabled = false
|
enabled = true
|
||||||
renderer = "gemtext"
|
renderer = "gemtext"
|
||||||
daemon = "agate"
|
daemon = "agate"
|
||||||
daemon_policy = "auto"
|
daemon_policy = "auto"
|
||||||
port = 1965
|
port = 1965
|
||||||
|
max_asset_bytes = 1048576
|
||||||
|
|
||||||
[site.protocols.reticulum]
|
[site.protocols.reticulum]
|
||||||
enabled = false
|
enabled = false
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ampg.audit import audit_gateway
|
||||||
|
from ampg.config import load_config
|
||||||
|
|
||||||
|
|
||||||
|
class AuditTest(unittest.TestCase):
|
||||||
|
def test_reports_semantic_html_warnings(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
source = root / "source"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "index.html").write_text(
|
||||||
|
'<h1>Title</h1><h3>Skipped</h3><img src="x.png"><a href="page.html"></a>',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
config_path = root / "gateway.toml"
|
||||||
|
config_path.write_text(
|
||||||
|
"""
|
||||||
|
[[site]]
|
||||||
|
id = "example"
|
||||||
|
domain = "example.test"
|
||||||
|
|
||||||
|
[site.source]
|
||||||
|
kind = "static-html"
|
||||||
|
path = "./source"
|
||||||
|
|
||||||
|
[site.outputs]
|
||||||
|
root = "./out"
|
||||||
|
""",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = audit_gateway(load_config(config_path))
|
||||||
|
codes = {issue.code for issue in issues}
|
||||||
|
|
||||||
|
self.assertIn("heading_level_skip", codes)
|
||||||
|
self.assertIn("missing_alt", codes)
|
||||||
|
self.assertIn("empty_link_text", codes)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ampg.build import build_gateway
|
||||||
|
from ampg.config import load_config
|
||||||
|
from ampg.plan import plan_gateway, write_plan_artifacts
|
||||||
|
|
||||||
|
|
||||||
|
class GeminiBuildTest(unittest.TestCase):
|
||||||
|
def test_existing_html_source_can_build_only_gemini(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
source = root / "source"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "index.html").write_text(
|
||||||
|
'<h1>Home</h1><p><a href="about.html">About</a></p><script>bad()</script>',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(source / "about.html").write_text("<h1>About</h1>", encoding="utf-8")
|
||||||
|
(source / "app.js").write_text("bad()", encoding="utf-8")
|
||||||
|
config_path = root / "gateway.toml"
|
||||||
|
config_path.write_text(
|
||||||
|
"""
|
||||||
|
[[site]]
|
||||||
|
id = "gemini_only"
|
||||||
|
domain = "example.test"
|
||||||
|
|
||||||
|
[site.source]
|
||||||
|
kind = "static-html"
|
||||||
|
path = "./source"
|
||||||
|
|
||||||
|
[site.outputs]
|
||||||
|
root = "./out"
|
||||||
|
plan_root = "./plan"
|
||||||
|
|
||||||
|
[site.protocols.gemini]
|
||||||
|
enabled = true
|
||||||
|
renderer = "gemtext"
|
||||||
|
daemon = "agate"
|
||||||
|
daemon_policy = "auto"
|
||||||
|
port = 1965
|
||||||
|
max_asset_bytes = 1024
|
||||||
|
""",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
config = load_config(config_path)
|
||||||
|
plan = plan_gateway(config)
|
||||||
|
artifacts = write_plan_artifacts(config)
|
||||||
|
results = build_gateway(config)
|
||||||
|
|
||||||
|
self.assertEqual([line.protocol for line in plan], ["gemini"])
|
||||||
|
self.assertEqual([result.protocol for result in results], ["gemini"])
|
||||||
|
self.assertEqual(len(artifacts), 1)
|
||||||
|
self.assertTrue((root / "out/gemini/index.gmi").exists())
|
||||||
|
self.assertTrue((root / "out/gemini/about.gmi").exists())
|
||||||
|
self.assertFalse((root / "out/gemini/app.js").exists())
|
||||||
|
self.assertFalse((root / "out/clearnet").exists())
|
||||||
|
gemtext = (root / "out/gemini/index.gmi").read_text(encoding="utf-8")
|
||||||
|
self.assertIn("# Home", gemtext)
|
||||||
|
self.assertIn("=> about.gmi About", gemtext)
|
||||||
|
self.assertNotIn("bad", gemtext)
|
||||||
|
agate_plan = (root / "plan/gemini_only/gemini/agate-plan.txt").read_text()
|
||||||
|
self.assertIn("daemon = agate", agate_plan)
|
||||||
|
self.assertIn("content_root =", agate_plan)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from ampg.renderers import render_gemtext
|
||||||
|
|
||||||
|
|
||||||
|
class GemtextRendererTest(unittest.TestCase):
|
||||||
|
def test_converts_basic_html_to_gemtext(self):
|
||||||
|
html = """
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head><script>ignored()</script></head>
|
||||||
|
<body>
|
||||||
|
<h1>Hello</h1>
|
||||||
|
<p>Read the <a href="about.html">about page</a>.</p>
|
||||||
|
<ul><li>One</li><li>Two</li></ul>
|
||||||
|
<img src="logo.png" alt="Logo">
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
gemtext = render_gemtext(
|
||||||
|
html,
|
||||||
|
rewrite_link=lambda href: href.replace(".html", ".gmi"),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIn("# Hello", gemtext)
|
||||||
|
self.assertIn("Read the about page.", gemtext)
|
||||||
|
self.assertIn("=> about.gmi about page", gemtext)
|
||||||
|
self.assertIn("- One", gemtext)
|
||||||
|
self.assertIn("- Two", gemtext)
|
||||||
|
self.assertIn("=> logo.png Image: Logo", gemtext)
|
||||||
|
self.assertNotIn("ignored", gemtext)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -44,6 +44,13 @@ daemon_policy = "auto"
|
|||||||
local_port = 19081
|
local_port = 19081
|
||||||
keys_file = "example-web.dat"
|
keys_file = "example-web.dat"
|
||||||
tunnel_name = "example-web"
|
tunnel_name = "example-web"
|
||||||
|
|
||||||
|
[site.protocols.gemini]
|
||||||
|
enabled = true
|
||||||
|
renderer = "gemtext"
|
||||||
|
daemon = "agate"
|
||||||
|
daemon_policy = "auto"
|
||||||
|
port = 1965
|
||||||
""",
|
""",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
@@ -52,19 +59,21 @@ tunnel_name = "example-web"
|
|||||||
lines = plan_gateway(config)
|
lines = plan_gateway(config)
|
||||||
artifacts = write_plan_artifacts(config)
|
artifacts = write_plan_artifacts(config)
|
||||||
|
|
||||||
self.assertEqual(len(lines), 2)
|
self.assertEqual(len(lines), 3)
|
||||||
self.assertEqual(len(artifacts), 4)
|
self.assertEqual(len(artifacts), 5)
|
||||||
|
|
||||||
torrc = root / "plan/example/tor/torrc-snippet.conf"
|
torrc = root / "plan/example/tor/torrc-snippet.conf"
|
||||||
tor_nginx = root / "plan/example/tor/nginx-loopback.conf"
|
tor_nginx = root / "plan/example/tor/nginx-loopback.conf"
|
||||||
i2p_tunnel = root / "plan/example/i2p/i2pd-tunnel.conf"
|
i2p_tunnel = root / "plan/example/i2p/i2pd-tunnel.conf"
|
||||||
i2p_nginx = root / "plan/example/i2p/nginx-loopback.conf"
|
i2p_nginx = root / "plan/example/i2p/nginx-loopback.conf"
|
||||||
|
agate_plan = root / "plan/example/gemini/agate-plan.txt"
|
||||||
|
|
||||||
self.assertIn("HiddenServicePort 80 127.0.0.1:19080", torrc.read_text())
|
self.assertIn("HiddenServicePort 80 127.0.0.1:19080", torrc.read_text())
|
||||||
self.assertIn("listen 127.0.0.1:19080;", tor_nginx.read_text())
|
self.assertIn("listen 127.0.0.1:19080;", tor_nginx.read_text())
|
||||||
self.assertIn("[example-web]", i2p_tunnel.read_text())
|
self.assertIn("[example-web]", i2p_tunnel.read_text())
|
||||||
self.assertIn("keys = example-web.dat", i2p_tunnel.read_text())
|
self.assertIn("keys = example-web.dat", i2p_tunnel.read_text())
|
||||||
self.assertIn("listen 127.0.0.1:19081;", i2p_nginx.read_text())
|
self.assertIn("listen 127.0.0.1:19081;", i2p_nginx.read_text())
|
||||||
|
self.assertIn("daemon = agate", agate_plan.read_text())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user