CIS490/receiver/app.py
max 7c9f9582ca Lab-host shipper + receiver /v1/ping + install scripts
Implements the deployment loop end-to-end on the CIS490 side:

shipper/
  config.py      ShipperConfig (host_id, paths, receiver endpoint, mTLS)
  transport.py   httpx-based PUT + ping with mTLS + bearer support
  queue.py       scan data/episodes/, tar+zstd via system zstd, ship,
                 retire to data/shipped/. Idempotent across crashes per
                 the state machine in docs/transport.md.
  __main__.py    CLI: --ping (smoke test), --once (one pass), or daemon

receiver/app.py: new POST /v1/ping that requires the same auth as PUT
  /v1/episodes but writes nothing. Used by `cis490-shipper --ping`
  during lab-host bring-up to verify the WG/Caddy/mTLS path before
  shipping any real bytes.

etc/
  cis490-shipper.service       systemd unit for the lab-host shipper
  cis490-orchestrator.service  systemd unit for the lab-host queue
                               (kept disabled by default until queue
                               mode lands)
  lab-host.toml.example        config template

scripts/
  install-lab-host.sh   idempotent installer; verifies prereqs,
                        creates cis490 service user, syncs repo to
                        /opt/cis490, builds venv, drops systemd units
                        and config template
  install-receiver.sh   same, for the receiver role on the central WG
                        node (Pi5 in our setup)

tests/test_shipper.py  11 end-to-end tests against a real Uvicorn
                       server hosting the receiver app. Exercises
                       ping, tar+ship, idempotent re-ship, 409
                       conflict, transient (receiver down), tarball
                       round-trip via system zstd.

AGENTS.md  guidance for AI agents working on this and sibling repos.
           Headline: when you hit an issue you can't fully fix in
           scope, file a Forgejo issue rather than leaving a TODO.

51/51 tests pass.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-04-29 23:41:32 -05:00

153 lines
5.2 KiB
Python

from __future__ import annotations
import logging
import secrets
import time
from pathlib import Path
from typing import Awaitable, Callable
from starlette.applications import Starlette
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from .store import EpisodeStore, is_valid_id
log = logging.getLogger("cis490.receiver")
SUFFIX = ".tar.zst"
SCHEMA_VERSION = 1
def _bearer_check(request: Request, expected: str | None) -> Response | None:
if expected is None:
return None
auth = request.headers.get("authorization", "")
prefix = "Bearer "
if not auth.startswith(prefix):
return JSONResponse({"error": "missing bearer token"}, status_code=401)
presented = auth[len(prefix):]
if not secrets.compare_digest(presented, expected):
return JSONResponse({"error": "bad bearer token"}, status_code=401)
return None
def make_app(
store: EpisodeStore,
max_episode_bytes: int,
bearer_token: str | None = None,
) -> Starlette:
async def health(request: Request) -> JSONResponse:
return JSONResponse({"status": "ok"})
async def ping(request: Request) -> JSONResponse:
"""Smoke-test endpoint. Verifies that the auth layer and the
WG/Caddy/receiver pipe are alive end-to-end without persisting
anything — index.jsonl is untouched. Used by ``cis490-shipper
--ping`` during initial bring-up of a new lab host."""
guard = _bearer_check(request, bearer_token)
if guard is not None:
return guard
return JSONResponse(
{
"ok": True,
"host_id": request.headers.get("x-lab-host"),
"t_wall_ns": time.time_ns(),
"schema_version": SCHEMA_VERSION,
}
)
async def put_episode(request: Request) -> JSONResponse:
guard = _bearer_check(request, bearer_token)
if guard is not None:
return guard
host_id: str = request.path_params["host_id"]
filename: str = request.path_params["filename"]
if not filename.endswith(SUFFIX):
return JSONResponse(
{"error": f"expected {SUFFIX} suffix"}, status_code=400
)
episode_id = filename[: -len(SUFFIX)]
if not is_valid_id(host_id) or not is_valid_id(episode_id):
return JSONResponse({"error": "bad host_id or episode_id"}, status_code=400)
expected_sha = request.headers.get("x-content-sha256")
if not expected_sha or len(expected_sha) != 64:
return JSONResponse(
{"error": "X-Content-SHA256 header (64 hex chars) required"},
status_code=400,
)
expected_sha = expected_sha.lower()
try:
schema_version = int(request.headers.get("x-schema-version", "1"))
except ValueError:
return JSONResponse({"error": "bad X-Schema-Version"}, status_code=400)
cl = request.headers.get("content-length")
if cl is not None:
try:
if int(cl) > max_episode_bytes:
return JSONResponse(
{"error": "episode exceeds max size"}, status_code=413
)
except ValueError:
return JSONResponse({"error": "bad Content-Length"}, status_code=400)
result = await store.ingest_stream(
host_id=host_id,
episode_id=episode_id,
expected_sha256=expected_sha,
schema_version=schema_version,
body=request.stream(),
max_bytes=max_episode_bytes,
)
if result.status == "stored":
log.info(
"stored episode host=%s id=%s sha=%s size=%d",
host_id, episode_id, result.sha256, result.size_bytes,
)
return JSONResponse(
{"status": "stored", "sha256": result.sha256, "size_bytes": result.size_bytes},
status_code=201,
)
if result.status == "already-present":
return JSONResponse(
{"status": "already-present", "sha256": result.sha256},
status_code=200,
)
if result.status == "conflict":
log.warning(
"conflict host=%s id=%s existing=%s",
host_id, episode_id, result.existing_sha256,
)
return JSONResponse(
{"status": "conflict", "existing_sha256": result.existing_sha256},
status_code=409,
)
if result.status == "sha-mismatch":
return JSONResponse(
{"status": "sha-mismatch", "actual_sha256": result.sha256},
status_code=400,
)
if result.status == "too-large":
return JSONResponse({"error": "episode exceeds max size"}, status_code=413)
return JSONResponse({"error": "unknown ingest result"}, status_code=500)
routes = [
Route("/v1/health", health, methods=["GET"]),
Route("/v1/ping", ping, methods=["POST"]),
Route(
"/v1/episodes/{host_id}/{filename}",
put_episode,
methods=["PUT"],
),
]
return Starlette(routes=routes)