diff --git a/training/README.md b/training/README.md index 28b4b52..96f25d5 100644 --- a/training/README.md +++ b/training/README.md @@ -32,7 +32,9 @@ training/ _metrics.py macro F1 + per-class F1 with bootstrap 95 % CIs; paired-bootstrap significance for model-vs-model breakdown.py per-profile, per-host metric tables - dashboard/producers/ live event emitters — see ../dashboard/PRODUCERS.md + producers/ live event sources for the dashboard + (replay, metrics, perf, profiles) + → see training/dashboard/PRODUCERS.md ``` ## The honesty rules this implements @@ -162,7 +164,7 @@ uv run --group training python -m training.eval_.run \ ## Live dashboard -Producers under `training/dashboard/producers/` push events to the +Producers under `training/producers/` push events to the `dashboard.wg` WebSocket via the canonical `training.dashboard.client.Publisher` (loopback HTTP, stdlib-only). See [`../dashboard/PRODUCERS.md`](../dashboard/PRODUCERS.md) for the @@ -170,7 +172,7 @@ event contract. ```sh # After training, push live model_metric + model_perf bars: -uv run --group training python -m training.dashboard.producers.metrics \ +uv run --group training python -m training.producers.metrics \ --validation data/processed/validation_v1.parquet \ --artifacts artifacts \ --summary data/processed/features_window_v1.parquet \ @@ -178,7 +180,7 @@ uv run --group training python -m training.dashboard.producers.metrics \ # Replay one episode at wall-clock speed (drives phase + prediction + # embedding events): -uv run --group training python -m training.dashboard.producers.replay \ +uv run --group training python -m training.producers.replay \ --episode /var/lib/cis490/episodes/elliott-thinkpad/.tar.zst \ --host-id elliott-thinkpad \ --artifacts artifacts diff --git a/training/eval_/run.py b/training/eval_/run.py index b0620e1..90cd645 100644 --- a/training/eval_/run.py +++ b/training/eval_/run.py @@ -24,7 +24,7 @@ from training._features import PHASES from training._split import ( held_out_host, held_out_sample, held_out_time, ) -from training.dashboard.producers._models import load_models +from training.producers._models import load_models from training.eval_._metrics import ( bootstrap_macro_f1, bootstrap_per_class_f1, confusion_matrix, paired_bootstrap_macro_f1_diff, diff --git a/training/dashboard/producers/__init__.py b/training/producers/__init__.py similarity index 100% rename from training/dashboard/producers/__init__.py rename to training/producers/__init__.py diff --git a/training/dashboard/producers/__main__.py b/training/producers/__main__.py similarity index 54% rename from training/dashboard/producers/__main__.py rename to training/producers/__main__.py index 912f13a..5b7b93a 100644 --- a/training/dashboard/producers/__main__.py +++ b/training/producers/__main__.py @@ -1,9 +1,9 @@ """CLI dispatcher for dashboard producers. - python -m training.dashboard.producers replay --episode … --host-id … - python -m training.dashboard.producers metrics --window … --schema … - python -m training.dashboard.producers perf --window … --schema … - python -m training.dashboard.producers profiles --validation … --store … + python -m training.producers replay --episode … --host-id … + python -m training.producers metrics --window … --schema … + python -m training.producers perf --window … --schema … + python -m training.producers profiles --validation … --store … Each subcommand forwards remaining argv to the matching module's main(). """ @@ -13,16 +13,16 @@ import sys SUBCOMMANDS = { - "replay": "training.dashboard.producers.replay", - "metrics": "training.dashboard.producers.metrics", - "perf": "training.dashboard.producers.perf", - "profiles": "training.dashboard.producers.profiles", + "replay": "training.producers.replay", + "metrics": "training.producers.metrics", + "perf": "training.producers.perf", + "profiles": "training.producers.profiles", } def main() -> int: if len(sys.argv) < 2 or sys.argv[1] in {"-h", "--help"}: - print("usage: python -m training.dashboard.producers " + print("usage: python -m training.producers " " [args]", file=sys.stderr) return 2 sub = sys.argv[1] diff --git a/training/dashboard/producers/_models.py b/training/producers/_models.py similarity index 100% rename from training/dashboard/producers/_models.py rename to training/producers/_models.py diff --git a/training/dashboard/producers/_publish.py b/training/producers/_publish.py similarity index 100% rename from training/dashboard/producers/_publish.py rename to training/producers/_publish.py diff --git a/training/dashboard/producers/metrics.py b/training/producers/metrics.py similarity index 98% rename from training/dashboard/producers/metrics.py rename to training/producers/metrics.py index a77e81b..ad5f9d3 100644 --- a/training/dashboard/producers/metrics.py +++ b/training/producers/metrics.py @@ -28,8 +28,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[3])) from training._split import ( held_out_host, held_out_sample, held_out_time, ) -from training.dashboard.producers._models import load_models -from training.dashboard.producers._publish import ( +from training.producers._models import load_models +from training.producers._publish import ( PublishFn, http_publisher, null_publisher, ) from training.eval_._metrics import _macro_f1 diff --git a/training/dashboard/producers/perf.py b/training/producers/perf.py similarity index 95% rename from training/dashboard/producers/perf.py rename to training/producers/perf.py index 28474b5..c363617 100644 --- a/training/dashboard/producers/perf.py +++ b/training/producers/perf.py @@ -20,10 +20,10 @@ import pyarrow.parquet as pq sys.path.insert(0, str(Path(__file__).resolve().parents[3])) from training._split import held_out_host -from training.dashboard.producers._models import ( +from training.producers._models import ( latency_us_batched, load_models, ) -from training.dashboard.producers._publish import ( +from training.producers._publish import ( PublishFn, http_publisher, null_publisher, ) from training.eval_._metrics import _macro_f1 @@ -37,7 +37,7 @@ async def emit_perf(*, publish: PublishFn, artifacts_dir: Path, summary_path: Path | None, tensors_root: Path | None, batch_for_scatter: int = 64) -> int: - from training.dashboard.producers.metrics import _build_test_set + from training.producers.metrics import _build_test_set models = load_models(artifacts_dir) if not models: return 0 diff --git a/training/dashboard/producers/profiles.py b/training/producers/profiles.py similarity index 99% rename from training/dashboard/producers/profiles.py rename to training/producers/profiles.py index a13f27b..5d96205 100644 --- a/training/dashboard/producers/profiles.py +++ b/training/producers/profiles.py @@ -29,7 +29,7 @@ import pyarrow.parquet as pq sys.path.insert(0, str(Path(__file__).resolve().parents[3])) from training._episode_io import open_episode from training._features import ALL_CHANNELS, channel_arrays -from training.dashboard.producers._publish import ( +from training.producers._publish import ( PublishFn, http_publisher, null_publisher, ) diff --git a/training/dashboard/producers/replay.py b/training/producers/replay.py similarity index 98% rename from training/dashboard/producers/replay.py rename to training/producers/replay.py index bebf882..48127bd 100644 --- a/training/dashboard/producers/replay.py +++ b/training/producers/replay.py @@ -32,10 +32,10 @@ from training._episode_io import open_episode from training._features import ( PHASE_TO_INT, summary_windows, tensor_windows, ) -from training.dashboard.producers._models import ( +from training.producers._models import ( load_models, model_display_name, ) -from training.dashboard.producers._publish import ( +from training.producers._publish import ( PublishFn, http_publisher, null_publisher, ) from training.models import BaseModel