Capture transport addresses in state

This commit is contained in:
jwinterm
2026-07-06 07:00:04 -04:00
parent c52e700b6f
commit bf2a171ec6
16 changed files with 907 additions and 20 deletions
+385
View File
@@ -0,0 +1,385 @@
from __future__ import annotations
from dataclasses import dataclass
import json
from pathlib import Path
from typing import Any
from .config import GatewayConfig, ProtocolConfig, SiteConfig
SCHEMA = "ampg.address-registry.v1"
CAPTURE_PROTOCOLS = {"tor", "i2p", "reticulum", "ipfs"}
@dataclass(frozen=True)
class AddressRecord:
site_id: str
protocol: str
url: str
address_status: str
source: str
@dataclass(frozen=True)
class AddressCaptureResult:
site_id: str
protocol: str
status: str
url: str
source: str
path: Path | None
message: str
def address_registry_path(config: GatewayConfig) -> Path:
if config.paths:
return config.paths.state_dir / "addresses.json"
return config.config_path.parent / ".ampg/state/addresses.json"
def load_address_registry(config: GatewayConfig) -> dict[tuple[str, str], AddressRecord]:
path = address_registry_path(config)
if not path.exists():
return {}
try:
data = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise ValueError(f"invalid address registry {path}: {exc}") from exc
if data.get("schema") != SCHEMA:
raise ValueError(f"invalid address registry schema in {path}")
records: dict[tuple[str, str], AddressRecord] = {}
for raw_record in data.get("addresses", []):
if not isinstance(raw_record, dict):
continue
record = AddressRecord(
site_id=str(raw_record.get("site", "")),
protocol=str(raw_record.get("protocol", "")),
url=str(raw_record.get("url", "")),
address_status=str(raw_record.get("address_status", "captured")),
source=str(raw_record.get("source", "registry")),
)
if record.site_id and record.protocol and record.url:
records[(record.site_id, record.protocol)] = record
return records
def write_address_registry(
config: GatewayConfig,
records: dict[tuple[str, str], AddressRecord],
) -> Path:
path = address_registry_path(config)
path.parent.mkdir(parents=True, exist_ok=True)
data = {
"schema": SCHEMA,
"addresses": [
{
"site": record.site_id,
"protocol": record.protocol,
"url": record.url,
"address_status": record.address_status,
"source": record.source,
}
for record in sorted(records.values(), key=lambda item: (item.site_id, item.protocol))
],
}
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
return path
def set_address(
config: GatewayConfig,
*,
site_id: str,
protocol: str,
url: str,
source: str = "manual",
) -> AddressRecord:
_require_declared_protocol(config, site_id, protocol)
records = load_address_registry(config)
record = AddressRecord(
site_id=site_id,
protocol=protocol,
url=normalize_address_url(protocol, url),
address_status="captured",
source=source,
)
records[(site_id, protocol)] = record
write_address_registry(config, records)
return record
def effective_address_records(config: GatewayConfig) -> list[AddressRecord]:
registry = load_address_registry(config)
records: list[AddressRecord] = []
for site in config.sites:
for protocol in site.protocols.values():
if not protocol.enabled:
continue
configured = configured_address(site, protocol)
if configured:
records.append(configured)
continue
captured = registry.get((site.id, protocol.name))
if captured:
records.append(captured)
continue
records.append(_placeholder_or_derived_address(site, protocol))
return records
def effective_address(
config: GatewayConfig,
site: SiteConfig,
protocol: ProtocolConfig,
) -> AddressRecord | None:
configured = configured_address(site, protocol)
if configured:
return configured
return load_address_registry(config).get((site.id, protocol.name))
def configured_address(site: SiteConfig, protocol: ProtocolConfig) -> AddressRecord | None:
raw_url = protocol.options.get("fixture_url") or protocol.options.get("browser_url")
if raw_url is None:
raw_url = _protocol_specific_configured_url(protocol)
if raw_url:
return AddressRecord(
site_id=site.id,
protocol=protocol.name,
url=normalize_address_url(protocol.name, str(raw_url)),
address_status="configured",
source="config",
)
if protocol.name == "ipfs" and protocol.options.get("cid"):
return AddressRecord(
site_id=site.id,
protocol=protocol.name,
url=f"ipfs://{protocol.options['cid']}",
address_status="configured",
source="config",
)
return None
def _protocol_specific_configured_url(protocol: ProtocolConfig) -> Any:
if protocol.name == "tor":
onion_url = protocol.options.get("onion_url")
if onion_url:
return onion_url
onion_location = protocol.options.get("onion_location")
if onion_location and onion_location != "auto":
return onion_location
return None
if protocol.name == "i2p":
return protocol.options.get("i2p_url") or protocol.options.get("i2p_hostname")
if protocol.name == "gemini":
return protocol.options.get("gemini_url")
if protocol.name == "reticulum":
return protocol.options.get("rns_url")
if protocol.name == "ipfs":
return protocol.options.get("ipfs_url")
if protocol.name == "clearnet":
return protocol.options.get("url")
return None
def capture_addresses(config: GatewayConfig) -> list[AddressCaptureResult]:
records = load_address_registry(config)
results: list[AddressCaptureResult] = []
changed = False
for site in config.sites:
for protocol in site.protocols.values():
if not protocol.enabled:
continue
result = _capture_protocol_address(config, site, protocol)
results.append(result)
if result.status == "captured":
records[(site.id, protocol.name)] = AddressRecord(
site_id=site.id,
protocol=protocol.name,
url=result.url,
address_status="captured",
source=result.source,
)
changed = True
if changed:
write_address_registry(config, records)
return results
def normalize_address_url(protocol: str, value: str) -> str:
address = value.strip()
if protocol == "tor":
return _with_scheme(address, "http")
if protocol == "i2p":
return _with_scheme(address, "http")
if protocol == "gemini":
return _with_scheme(address, "gemini")
if protocol == "reticulum":
return _with_scheme(address, "rns")
if protocol == "ipfs":
if address.startswith(("/ipfs/", "/ipns/")):
return address
return _with_scheme(address, "ipfs")
if protocol == "clearnet":
return _with_scheme(address, "https")
return _with_trailing_slash(address)
def _capture_protocol_address(
config: GatewayConfig,
site: SiteConfig,
protocol: ProtocolConfig,
) -> AddressCaptureResult:
configured = configured_address(site, protocol)
if configured:
return AddressCaptureResult(
site_id=site.id,
protocol=protocol.name,
status="configured",
url=configured.url,
source=configured.source,
path=None,
message="address is set in config; registry unchanged",
)
candidates = _address_file_candidates(config, site, protocol)
if protocol.name not in CAPTURE_PROTOCOLS and not protocol.options.get("address_file"):
return AddressCaptureResult(
site_id=site.id,
protocol=protocol.name,
status="skipped",
url="",
source="-",
path=None,
message="protocol address is derived from site config",
)
for path in candidates:
if not path.exists():
continue
raw_value = _first_nonempty_line(path)
if not raw_value:
continue
url = normalize_address_url(protocol.name, raw_value)
return AddressCaptureResult(
site_id=site.id,
protocol=protocol.name,
status="captured",
url=url,
source=f"file:{path}",
path=path,
message="captured public transport address",
)
candidate_text = ", ".join(str(path) for path in candidates)
return AddressCaptureResult(
site_id=site.id,
protocol=protocol.name,
status="missing",
url="",
source="-",
path=None,
message=f"no address file found; checked {candidate_text}",
)
def _address_file_candidates(
config: GatewayConfig,
site: SiteConfig,
protocol: ProtocolConfig,
) -> tuple[Path, ...]:
configured_file = protocol.options.get("address_file")
candidates: list[Path] = []
if configured_file:
path = Path(str(configured_file)).expanduser()
if not path.is_absolute():
path = config.config_path.parent / path
candidates.append(path.resolve())
state_dir = _state_dir(config, site, protocol)
if protocol.name == "tor":
candidates.extend(
[
state_dir / "hidden-service/hostname",
state_dir / "hostname",
state_dir / "address.txt",
]
)
elif protocol.name == "i2p":
candidates.extend(
[
state_dir / "hostname.txt",
state_dir / "b32.txt",
state_dir / "address.txt",
]
)
else:
candidates.extend([state_dir / "address.txt", state_dir / "hostname"])
return tuple(candidates)
def _placeholder_or_derived_address(site: SiteConfig, protocol: ProtocolConfig) -> AddressRecord:
if protocol.name == "clearnet":
url = site.source.canonical_url or f"https://{site.domain}/"
return AddressRecord(site.id, protocol.name, _with_trailing_slash(url), "derived", "site")
if protocol.name == "tor":
return AddressRecord(
site.id,
protocol.name,
f"http://{site.id}.onion/",
"placeholder",
"placeholder",
)
if protocol.name == "i2p":
return AddressRecord(
site.id,
protocol.name,
f"http://{site.id}.i2p/",
"placeholder",
"placeholder",
)
if protocol.name == "gemini":
return AddressRecord(site.id, protocol.name, f"gemini://{site.domain}/", "derived", "site")
if protocol.name == "reticulum":
return AddressRecord(site.id, protocol.name, f"rns://{site.id}", "placeholder", "placeholder")
if protocol.name == "ipfs":
return AddressRecord(site.id, protocol.name, f"/ipfs/{site.id}", "placeholder", "placeholder")
return AddressRecord(site.id, protocol.name, f"https://{site.domain}/", "derived", "site")
def _require_declared_protocol(config: GatewayConfig, site_id: str, protocol: str) -> None:
for site in config.sites:
if site.id != site_id:
continue
if protocol not in site.protocols:
raise ValueError(f"{site_id}: protocol {protocol!r} is not declared")
return
raise ValueError(f"unknown site {site_id!r}")
def _state_dir(config: GatewayConfig, site: SiteConfig, protocol: ProtocolConfig) -> Path:
root = config.paths.state_dir if config.paths else config.config_path.parent / ".ampg/state"
return root / site.id / protocol.name
def _first_nonempty_line(path: Path) -> str:
for line in path.read_text(encoding="utf-8").splitlines():
value = line.strip()
if value:
return value
return ""
def _with_scheme(value: str, scheme: str) -> str:
if "://" in value:
return _with_trailing_slash(value)
return _with_trailing_slash(f"{scheme}://{value}")
def _with_trailing_slash(value: str) -> str:
if value.startswith(("rns://", "lxmf://", "nomad://", "ipfs://", "ipns://")):
return value
if value.endswith("/"):
return value
return value + "/"
+118
View File
@@ -10,6 +10,14 @@ from .activation import (
blocked_steps,
write_activation_artifacts,
)
from .addresses import (
AddressCaptureResult,
AddressRecord,
address_registry_path,
capture_addresses,
effective_address_records,
set_address,
)
from .audit import audit_gateway
from .build import build_gateway
from .config import load_config
@@ -132,6 +140,41 @@ def main(argv: list[str] | None = None) -> int:
_add_target_selection(build_parser)
fixture_parser = subcommands.add_parser("manifest", help="Write AMPB fixture manifests.")
_add_target_selection(fixture_parser)
addresses_parser = subcommands.add_parser(
"addresses",
help="Inspect or update captured transport addresses.",
)
addresses_subcommands = addresses_parser.add_subparsers(
dest="addresses_command",
required=True,
)
addresses_list = addresses_subcommands.add_parser(
"list",
help="Print effective configured, captured, derived, and placeholder addresses.",
)
_add_target_selection(addresses_list)
addresses_capture = addresses_subcommands.add_parser(
"capture",
help="Capture generated transport addresses from daemon state files.",
)
_add_target_selection(addresses_capture)
addresses_set = addresses_subcommands.add_parser(
"set",
help="Manually store a generated transport address in AMPG state.",
)
addresses_set.add_argument("--site", required=True, help="Site id to update.")
addresses_set.add_argument(
"--protocol",
dest="address_protocol",
required=True,
help="Protocol name to update.",
)
addresses_set.add_argument("--url", required=True, help="Public transport URL or hostname.")
addresses_set.add_argument(
"--source",
default="manual",
help="Source label stored with the captured address.",
)
preview_parser = subcommands.add_parser("preview", help="Preview generated outputs locally.")
preview_subcommands = preview_parser.add_subparsers(dest="preview_command", required=True)
for preview_command in ("endpoints", "manifest", "serve"):
@@ -224,6 +267,8 @@ def main(argv: list[str] | None = None) -> int:
return _cmd_build(config)
if args.command == "manifest":
return _cmd_manifest(config)
if args.command == "addresses":
return _cmd_addresses(config, args)
if args.command == "preview":
return _cmd_preview(config, args)
if args.command == "routes":
@@ -529,6 +574,79 @@ def _cmd_manifest(config) -> int:
return 0
def _cmd_addresses(config, args) -> int:
if args.addresses_command == "list":
records = effective_address_records(config)
for record in records:
_print_address(record)
print(
"AMPG_ADDRESS_SUMMARY "
f"sites={len(config.sites)} "
f"addresses={len(records)} "
f"configured={sum(1 for record in records if record.address_status == 'configured')} "
f"captured={sum(1 for record in records if record.address_status == 'captured')} "
f"derived={sum(1 for record in records if record.address_status == 'derived')} "
f"placeholder={sum(1 for record in records if record.address_status == 'placeholder')} "
f"registry={address_registry_path(config)}"
)
return 0
if args.addresses_command == "capture":
results = capture_addresses(config)
for result in results:
_print_address_capture(result)
print(
"AMPG_ADDRESS_CAPTURE_SUMMARY "
f"sites={len(config.sites)} "
f"results={len(results)} "
f"captured={sum(1 for result in results if result.status == 'captured')} "
f"configured={sum(1 for result in results if result.status == 'configured')} "
f"skipped={sum(1 for result in results if result.status == 'skipped')} "
f"missing={sum(1 for result in results if result.status == 'missing')} "
f"registry={address_registry_path(config)}"
)
return 0
if args.addresses_command == "set":
record = set_address(
config,
site_id=args.site,
protocol=args.address_protocol,
url=args.url,
source=args.source,
)
_print_address(record)
print(f"AMPG_ADDRESS_SET status=written registry={address_registry_path(config)}")
return 0
return 1
def _print_address(record: AddressRecord) -> None:
print(
"AMPG_ADDRESS "
f"site={record.site_id} "
f"protocol={record.protocol} "
f"url=\"{_quote(record.url)}\" "
f"address_status={record.address_status} "
f"source=\"{_quote(record.source)}\""
)
def _print_address_capture(result: AddressCaptureResult) -> None:
path = str(result.path) if result.path else "-"
print(
"AMPG_ADDRESS_CAPTURE "
f"site={result.site_id} "
f"protocol={result.protocol} "
f"status={result.status} "
f"url=\"{_quote(result.url)}\" "
f"source=\"{_quote(result.source)}\" "
f"path=\"{_quote(path)}\" "
f"message=\"{_quote(result.message)}\""
)
def _cmd_preview(config, args) -> int:
endpoints = preview_endpoints(config, base_port=args.base_port, host=args.host)
if args.preview_command == "endpoints":
+27 -1
View File
@@ -87,11 +87,20 @@ class ProfileConfig:
description: str = ""
@dataclass(frozen=True)
class GatewayPaths:
state_dir: Path
cache_dir: Path
run_dir: Path
user: str | None = None
@dataclass(frozen=True)
class GatewayConfig:
config_path: Path
sites: list[SiteConfig]
profiles: dict[str, ProfileConfig] = field(default_factory=dict)
paths: GatewayPaths | None = None
def load_config(path: Path) -> GatewayConfig:
@@ -101,7 +110,24 @@ def load_config(path: Path) -> GatewayConfig:
if not sites:
raise ValueError("config must declare at least one [[site]]")
profiles = _parse_profiles(data.get("profiles", {}))
return GatewayConfig(config_path=config_path, sites=sites, profiles=profiles)
paths = _parse_gateway_paths(data.get("gateway", {}), config_path.parent)
return GatewayConfig(config_path=config_path, sites=sites, profiles=profiles, paths=paths)
def _parse_gateway_paths(raw_gateway: Any, base_dir: Path) -> GatewayPaths:
if raw_gateway is None:
raw_gateway = {}
if not isinstance(raw_gateway, dict):
raise ValueError("gateway must be a table")
user = raw_gateway.get("user")
if user is not None and not isinstance(user, str):
raise ValueError("gateway.user must be a string")
return GatewayPaths(
state_dir=_resolve_path(base_dir, str(raw_gateway.get("state_dir", "./.ampg/state"))),
cache_dir=_resolve_path(base_dir, str(raw_gateway.get("cache_dir", "./.ampg/cache"))),
run_dir=_resolve_path(base_dir, str(raw_gateway.get("run_dir", "./.ampg/run"))),
user=user,
)
def _parse_profiles(raw_profiles: Any) -> dict[str, ProfileConfig]:
+4 -3
View File
@@ -40,7 +40,7 @@ def health_plan(
endpoints = preview_endpoints(config, base_port=base_port, host=host) if mode == "preview" else []
checks: list[HealthCheck] = []
for site in config.sites:
manifest = _manifest_for_mode(site, mode=mode, endpoints=endpoints)
manifest = _manifest_for_mode(config, site, mode=mode, endpoints=endpoints)
for fixture in manifest["fixtures"]:
checks.append(_health_check(site, fixture, mode=mode))
return checks
@@ -51,14 +51,15 @@ def blocked_health_checks(checks: list[HealthCheck]) -> list[HealthCheck]:
def _manifest_for_mode(
config: GatewayConfig,
site: SiteConfig,
*,
mode: str,
endpoints,
) -> dict[str, Any]:
if mode == "preview":
return preview_fixture_manifest(site, endpoints)
return fixture_manifest(site)
return preview_fixture_manifest(config, site, endpoints)
return fixture_manifest(config, site)
def _health_check(site: SiteConfig, fixture: dict[str, Any], *, mode: str) -> HealthCheck:
+15 -6
View File
@@ -5,6 +5,7 @@ import json
from pathlib import Path
from typing import Any
from .addresses import AddressRecord, effective_address
from .config import GatewayConfig, ProtocolConfig, RoutePolicyConfig, SiteConfig
from .route_policy import fixture_path, route_exposed
@@ -19,14 +20,15 @@ class ManifestWriteResult:
fixture_count: int
def fixture_manifest(site: SiteConfig) -> dict[str, Any]:
def fixture_manifest(config: GatewayConfig, site: SiteConfig) -> dict[str, Any]:
fixtures = []
for protocol in site.protocols.values():
if not protocol.enabled:
continue
fixtures.append(_fixture_entry(site, protocol, None))
address = effective_address(config, site, protocol)
fixtures.append(_fixture_entry(site, protocol, None, address))
fixtures.extend(
_fixture_entry(site, protocol, route_policy)
_fixture_entry(site, protocol, route_policy, address)
for route_policy in site.interactions.routes
if route_exposed(site, protocol, route_policy)
)
@@ -44,7 +46,7 @@ def fixture_manifest(site: SiteConfig) -> dict[str, Any]:
def write_fixture_manifests(config: GatewayConfig) -> list[ManifestWriteResult]:
results: list[ManifestWriteResult] = []
for site in config.sites:
manifest = fixture_manifest(site)
manifest = fixture_manifest(config, site)
path = manifest_path(site)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
@@ -69,8 +71,9 @@ def _fixture_entry(
site: SiteConfig,
protocol: ProtocolConfig,
route_policy: RoutePolicyConfig | None,
address: AddressRecord | None,
) -> dict[str, Any]:
url, address_status = _fixture_url(site, protocol)
url, address_status = _fixture_url(site, protocol, address)
transport = _transport_for(protocol.name)
entry = {
"protocol": protocol.name,
@@ -134,10 +137,16 @@ def _output_path_for_route(protocol_name: str, route_policy: RoutePolicyConfig |
return protocol_name + fixture_path(route_policy.match)
def _fixture_url(site: SiteConfig, protocol: ProtocolConfig) -> tuple[str, str]:
def _fixture_url(
site: SiteConfig,
protocol: ProtocolConfig,
address: AddressRecord | None,
) -> tuple[str, str]:
configured_url = protocol.options.get("fixture_url") or protocol.options.get("browser_url")
if configured_url:
return _with_trailing_slash(str(configured_url)), "configured"
if address:
return address.url, address.address_status
if protocol.name == "clearnet":
if site.source.canonical_url:
+56
View File
@@ -68,6 +68,38 @@ class TransportInteractionCapability:
CONFIG_FIELDS = (
ConfigField(
"gateway",
"state_dir",
"path",
False,
"./.ampg/state",
"AMPG-owned daemon state and captured address registry root.",
),
ConfigField(
"gateway",
"cache_dir",
"path",
False,
"./.ampg/cache",
"AMPG cache root for generated intermediate data.",
),
ConfigField(
"gateway",
"run_dir",
"path",
False,
"./.ampg/run",
"Runtime socket and process metadata root for managed daemons.",
),
ConfigField(
"gateway",
"user",
"string",
False,
"",
"Preferred service user for generated managed-daemon artifacts.",
),
ConfigField("site", "id", "string", True, "", "Stable site identifier used in output paths."),
ConfigField("site", "domain", "string", True, "", "Canonical public domain for the site."),
ConfigField("site.source", "kind", "enum", True, "static-html", "Source adapter kind."),
@@ -179,6 +211,30 @@ CONFIG_FIELDS = (
"protocol default",
"Whether AMPG adopts, manages, or only renders config.",
),
ConfigField(
"site.protocols.<name>",
"fixture_url",
"url",
False,
"",
"Published URL used in fixture manifests and health plans.",
),
ConfigField(
"site.protocols.<name>",
"browser_url",
"url",
False,
"",
"Browser-facing URL alias used when fixture_url is not set.",
),
ConfigField(
"site.protocols.<name>",
"address_file",
"path",
False,
"",
"Daemon-written public address file consumed by addresses capture.",
),
ConfigField(
"site.protocols.<name>",
"max_asset_bytes",
+3 -2
View File
@@ -67,6 +67,7 @@ def preview_endpoints(
def preview_fixture_manifest(
config: GatewayConfig,
site: SiteConfig,
endpoints: list[PreviewEndpoint],
) -> dict[str, Any]:
@@ -75,7 +76,7 @@ def preview_fixture_manifest(
for endpoint in endpoints
if endpoint.site_id == site.id
}
manifest = fixture_manifest(site)
manifest = fixture_manifest(config, site)
manifest["mode"] = "preview"
for fixture in manifest["fixtures"]:
endpoint = by_protocol[fixture["protocol"]]
@@ -110,7 +111,7 @@ def write_preview_fixture_manifests(
endpoints = preview_endpoints(config, base_port=base_port, host=host)
results: list[PreviewManifestWriteResult] = []
for site in config.sites:
manifest = preview_fixture_manifest(site, endpoints)
manifest = preview_fixture_manifest(config, site, endpoints)
path = preview_manifest_path(site)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+1
View File
@@ -60,6 +60,7 @@ def select_protocols(config: GatewayConfig, protocols: tuple[str, ...]) -> Gatew
config_path=config.config_path,
sites=sites,
profiles=config.profiles,
paths=config.paths,
)