training/producers: move out of dashboard/ per ownership boundary
Producers are event *sources* — the renderer is everything inside training/dashboard/. Sibling layout makes the dependency direction one-way (producers import from training.dashboard.events; dashboard never reaches into producers). training/dashboard/producers/ → training/producers/ Internal imports rewritten via sed; eval_/run.py and training/README.md cross-references updated. CLI entry stays via `python -m training.producers.<sub>` (replay / metrics / perf / profiles). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
f303337a1e
commit
697e36a315
10 changed files with 24 additions and 22 deletions
|
|
@ -32,7 +32,9 @@ training/
|
||||||
_metrics.py macro F1 + per-class F1 with bootstrap 95 % CIs;
|
_metrics.py macro F1 + per-class F1 with bootstrap 95 % CIs;
|
||||||
paired-bootstrap significance for model-vs-model
|
paired-bootstrap significance for model-vs-model
|
||||||
breakdown.py per-profile, per-host metric tables
|
breakdown.py per-profile, per-host metric tables
|
||||||
dashboard/producers/ live event emitters — see ../dashboard/PRODUCERS.md
|
producers/ live event sources for the dashboard
|
||||||
|
(replay, metrics, perf, profiles)
|
||||||
|
→ see training/dashboard/PRODUCERS.md
|
||||||
```
|
```
|
||||||
|
|
||||||
## The honesty rules this implements
|
## The honesty rules this implements
|
||||||
|
|
@ -162,7 +164,7 @@ uv run --group training python -m training.eval_.run \
|
||||||
|
|
||||||
## Live dashboard
|
## Live dashboard
|
||||||
|
|
||||||
Producers under `training/dashboard/producers/` push events to the
|
Producers under `training/producers/` push events to the
|
||||||
`dashboard.wg` WebSocket via the canonical
|
`dashboard.wg` WebSocket via the canonical
|
||||||
`training.dashboard.client.Publisher` (loopback HTTP, stdlib-only).
|
`training.dashboard.client.Publisher` (loopback HTTP, stdlib-only).
|
||||||
See [`../dashboard/PRODUCERS.md`](../dashboard/PRODUCERS.md) for the
|
See [`../dashboard/PRODUCERS.md`](../dashboard/PRODUCERS.md) for the
|
||||||
|
|
@ -170,7 +172,7 @@ event contract.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
# After training, push live model_metric + model_perf bars:
|
# After training, push live model_metric + model_perf bars:
|
||||||
uv run --group training python -m training.dashboard.producers.metrics \
|
uv run --group training python -m training.producers.metrics \
|
||||||
--validation data/processed/validation_v1.parquet \
|
--validation data/processed/validation_v1.parquet \
|
||||||
--artifacts artifacts \
|
--artifacts artifacts \
|
||||||
--summary data/processed/features_window_v1.parquet \
|
--summary data/processed/features_window_v1.parquet \
|
||||||
|
|
@ -178,7 +180,7 @@ uv run --group training python -m training.dashboard.producers.metrics \
|
||||||
|
|
||||||
# Replay one episode at wall-clock speed (drives phase + prediction +
|
# Replay one episode at wall-clock speed (drives phase + prediction +
|
||||||
# embedding events):
|
# embedding events):
|
||||||
uv run --group training python -m training.dashboard.producers.replay \
|
uv run --group training python -m training.producers.replay \
|
||||||
--episode /var/lib/cis490/episodes/elliott-thinkpad/<id>.tar.zst \
|
--episode /var/lib/cis490/episodes/elliott-thinkpad/<id>.tar.zst \
|
||||||
--host-id elliott-thinkpad \
|
--host-id elliott-thinkpad \
|
||||||
--artifacts artifacts
|
--artifacts artifacts
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ from training._features import PHASES
|
||||||
from training._split import (
|
from training._split import (
|
||||||
held_out_host, held_out_sample, held_out_time,
|
held_out_host, held_out_sample, held_out_time,
|
||||||
)
|
)
|
||||||
from training.dashboard.producers._models import load_models
|
from training.producers._models import load_models
|
||||||
from training.eval_._metrics import (
|
from training.eval_._metrics import (
|
||||||
bootstrap_macro_f1, bootstrap_per_class_f1,
|
bootstrap_macro_f1, bootstrap_per_class_f1,
|
||||||
confusion_matrix, paired_bootstrap_macro_f1_diff,
|
confusion_matrix, paired_bootstrap_macro_f1_diff,
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,9 @@
|
||||||
"""CLI dispatcher for dashboard producers.
|
"""CLI dispatcher for dashboard producers.
|
||||||
|
|
||||||
python -m training.dashboard.producers replay --episode … --host-id …
|
python -m training.producers replay --episode … --host-id …
|
||||||
python -m training.dashboard.producers metrics --window … --schema …
|
python -m training.producers metrics --window … --schema …
|
||||||
python -m training.dashboard.producers perf --window … --schema …
|
python -m training.producers perf --window … --schema …
|
||||||
python -m training.dashboard.producers profiles --validation … --store …
|
python -m training.producers profiles --validation … --store …
|
||||||
|
|
||||||
Each subcommand forwards remaining argv to the matching module's main().
|
Each subcommand forwards remaining argv to the matching module's main().
|
||||||
"""
|
"""
|
||||||
|
|
@ -13,16 +13,16 @@ import sys
|
||||||
|
|
||||||
|
|
||||||
SUBCOMMANDS = {
|
SUBCOMMANDS = {
|
||||||
"replay": "training.dashboard.producers.replay",
|
"replay": "training.producers.replay",
|
||||||
"metrics": "training.dashboard.producers.metrics",
|
"metrics": "training.producers.metrics",
|
||||||
"perf": "training.dashboard.producers.perf",
|
"perf": "training.producers.perf",
|
||||||
"profiles": "training.dashboard.producers.profiles",
|
"profiles": "training.producers.profiles",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
if len(sys.argv) < 2 or sys.argv[1] in {"-h", "--help"}:
|
if len(sys.argv) < 2 or sys.argv[1] in {"-h", "--help"}:
|
||||||
print("usage: python -m training.dashboard.producers "
|
print("usage: python -m training.producers "
|
||||||
"<replay|metrics|perf|profiles> [args]", file=sys.stderr)
|
"<replay|metrics|perf|profiles> [args]", file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
sub = sys.argv[1]
|
sub = sys.argv[1]
|
||||||
|
|
@ -28,8 +28,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||||
from training._split import (
|
from training._split import (
|
||||||
held_out_host, held_out_sample, held_out_time,
|
held_out_host, held_out_sample, held_out_time,
|
||||||
)
|
)
|
||||||
from training.dashboard.producers._models import load_models
|
from training.producers._models import load_models
|
||||||
from training.dashboard.producers._publish import (
|
from training.producers._publish import (
|
||||||
PublishFn, http_publisher, null_publisher,
|
PublishFn, http_publisher, null_publisher,
|
||||||
)
|
)
|
||||||
from training.eval_._metrics import _macro_f1
|
from training.eval_._metrics import _macro_f1
|
||||||
|
|
@ -20,10 +20,10 @@ import pyarrow.parquet as pq
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||||
from training._split import held_out_host
|
from training._split import held_out_host
|
||||||
from training.dashboard.producers._models import (
|
from training.producers._models import (
|
||||||
latency_us_batched, load_models,
|
latency_us_batched, load_models,
|
||||||
)
|
)
|
||||||
from training.dashboard.producers._publish import (
|
from training.producers._publish import (
|
||||||
PublishFn, http_publisher, null_publisher,
|
PublishFn, http_publisher, null_publisher,
|
||||||
)
|
)
|
||||||
from training.eval_._metrics import _macro_f1
|
from training.eval_._metrics import _macro_f1
|
||||||
|
|
@ -37,7 +37,7 @@ async def emit_perf(*, publish: PublishFn, artifacts_dir: Path,
|
||||||
summary_path: Path | None,
|
summary_path: Path | None,
|
||||||
tensors_root: Path | None,
|
tensors_root: Path | None,
|
||||||
batch_for_scatter: int = 64) -> int:
|
batch_for_scatter: int = 64) -> int:
|
||||||
from training.dashboard.producers.metrics import _build_test_set
|
from training.producers.metrics import _build_test_set
|
||||||
models = load_models(artifacts_dir)
|
models = load_models(artifacts_dir)
|
||||||
if not models:
|
if not models:
|
||||||
return 0
|
return 0
|
||||||
|
|
@ -29,7 +29,7 @@ import pyarrow.parquet as pq
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
|
||||||
from training._episode_io import open_episode
|
from training._episode_io import open_episode
|
||||||
from training._features import ALL_CHANNELS, channel_arrays
|
from training._features import ALL_CHANNELS, channel_arrays
|
||||||
from training.dashboard.producers._publish import (
|
from training.producers._publish import (
|
||||||
PublishFn, http_publisher, null_publisher,
|
PublishFn, http_publisher, null_publisher,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -32,10 +32,10 @@ from training._episode_io import open_episode
|
||||||
from training._features import (
|
from training._features import (
|
||||||
PHASE_TO_INT, summary_windows, tensor_windows,
|
PHASE_TO_INT, summary_windows, tensor_windows,
|
||||||
)
|
)
|
||||||
from training.dashboard.producers._models import (
|
from training.producers._models import (
|
||||||
load_models, model_display_name,
|
load_models, model_display_name,
|
||||||
)
|
)
|
||||||
from training.dashboard.producers._publish import (
|
from training.producers._publish import (
|
||||||
PublishFn, http_publisher, null_publisher,
|
PublishFn, http_publisher, null_publisher,
|
||||||
)
|
)
|
||||||
from training.models import BaseModel
|
from training.models import BaseModel
|
||||||
Loading…
Add table
Reference in a new issue