feat(predict): enrich YAML sidecar with provenance and timing
CI / Sync project version with tag (hand-pushed tags only) (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 48s
CI / Lint (ruff check) (pull_request) Successful in 49s
CI / Format (ruff format) (pull_request) Successful in 49s
CI / Tests (pull_request) Successful in 3m14s
CI / Release (bump, changelog, badges, tag) on merge to master (pull_request) Has been skipped
CI / Sync project version with tag (hand-pushed tags only) (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 48s
CI / Lint (ruff check) (pull_request) Successful in 49s
CI / Format (ruff format) (pull_request) Successful in 49s
CI / Tests (pull_request) Successful in 3m14s
CI / Release (bump, changelog, badges, tag) on merge to master (pull_request) Has been skipped
`giant predict`'s sidecar previously stopped at kind/prediction_id/ output/dataset/checkpoint/timestamp, unlike `giant rollout`'s, which carries full run provenance (model_config, training_epoch, training_config, timing, ...) that flows into analysis gallery metadata. `analyze --prediction` consumed the same thin sidecar, so a prediction series in an analysis run was nearly unlabeled compared to its rollout counterparts. - `_write_prediction_ref` takes an `extra: dict | None` merged into the sidecar; `giant rollout` now uses it instead of a load/update/rewrite round trip (identical output). - New `_build_predict_timing`, key-compatible with `_build_rollout_timing`, from timers now wrapping predict's setup/ sample/write phases. - `giant predict` writes coord, has_truth, schema_version, steps, weights, device, batch_size(+auto), row/skip/unknown-pdg counts, timing, and the checkpoint's model_config/config_overrides/ training_epoch/best_val_loss/training_config/training_meta. - `giant/analysis/condor.py`'s `_PLOT_META_KEYS` forwards the new predict-only keys (plus rollout's previously-unforwarded config_overrides) into each plot's gallery metadata.yaml. - Fixes a `ty` regression from the prior commit in tests/test_cli_predict.py (Command has no static `.commands`). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PpxE9nij3ujg9XcuzvQ26q
This commit is contained in:
@@ -6,6 +6,7 @@ from typer.testing import CliRunner
|
||||
|
||||
from giant.cli import (
|
||||
_CEPH_PREDICTIONS,
|
||||
_build_predict_timing,
|
||||
_resolve_prediction_output,
|
||||
_write_prediction_ref,
|
||||
app,
|
||||
@@ -145,6 +146,46 @@ def test_ref_timestamp_is_iso_format(tmp_path):
|
||||
assert ts.tzinfo is not None
|
||||
|
||||
|
||||
def test_ref_yaml_merges_extra_after_base_fields(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
checkpoint = ckpt_dir / "best.pt"
|
||||
checkpoint.touch()
|
||||
|
||||
out = tmp_path / "pred.parquet"
|
||||
dataset = tmp_path / "full.manifest"
|
||||
pred_uuid = str(uuid.uuid4())
|
||||
|
||||
ref_path = _write_prediction_ref(
|
||||
checkpoint,
|
||||
pred_uuid,
|
||||
out,
|
||||
dataset,
|
||||
extra={"coord": "global", "n_rows": 42, "timing": {"setup_s": 1.0}},
|
||||
)
|
||||
data = yaml.safe_load(ref_path.read_text())
|
||||
|
||||
# Base fields untouched, extras layered on top.
|
||||
assert data["kind"] == "prediction"
|
||||
assert data["prediction_id"] == pred_uuid
|
||||
assert data["coord"] == "global"
|
||||
assert data["n_rows"] == 42
|
||||
assert data["timing"] == {"setup_s": 1.0}
|
||||
|
||||
|
||||
def test_ref_yaml_without_extra_matches_today(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
checkpoint = ckpt_dir / "best.pt"
|
||||
checkpoint.touch()
|
||||
|
||||
pred_uuid = str(uuid.uuid4())
|
||||
ref_path = _write_prediction_ref(checkpoint, pred_uuid, tmp_path / "p.parquet", tmp_path / "d")
|
||||
data = yaml.safe_load(ref_path.read_text())
|
||||
|
||||
assert set(data) == {"kind", "prediction_id", "output", "dataset", "checkpoint", "timestamp"}
|
||||
|
||||
|
||||
def test_ref_checkpoint_path_is_absolute(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
@@ -232,13 +273,52 @@ def test_predict_truth_metadata_key_exists():
|
||||
assert PREDICT_TRUTH_METADATA_KEY == "giant.predict.has_truth"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_predict_timing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_predict_timing_computes_per_step_cost():
|
||||
timing = _build_predict_timing(
|
||||
setup_s=1.0,
|
||||
predict_s=10.0,
|
||||
write_s=2.0,
|
||||
n_rows=100,
|
||||
device="cpu",
|
||||
torch_threads=4,
|
||||
)
|
||||
assert timing["n_rows"] == 100
|
||||
assert timing["sample_s"] == 8.0 # predict_s - write_s
|
||||
assert timing["us_per_step"] == 8.0 / 100 * 1e6
|
||||
assert timing["write_us_per_step"] == 2.0 / 100 * 1e6
|
||||
assert timing["rows_per_s"] == 10.0
|
||||
assert timing["device"] == "cpu" and timing["torch_threads"] == 4
|
||||
|
||||
|
||||
def test_build_predict_timing_handles_zero_rows():
|
||||
timing = _build_predict_timing(
|
||||
setup_s=1.0,
|
||||
predict_s=0.0,
|
||||
write_s=0.0,
|
||||
n_rows=0,
|
||||
device="cpu",
|
||||
torch_threads=1,
|
||||
)
|
||||
assert timing["us_per_step"] is None
|
||||
assert timing["write_us_per_step"] is None
|
||||
assert timing["rows_per_s"] is None
|
||||
|
||||
|
||||
def test_predict_has_truth_flag_default_on():
|
||||
# Inspecting rendered --help text is brittle across terminal
|
||||
# widths/color settings (wraps or re-colors mid-flag); go straight to
|
||||
# the underlying click command's registered option instead.
|
||||
import typer
|
||||
from typing import cast
|
||||
|
||||
predict_cmd = typer.main.get_command(app).commands["predict"]
|
||||
import typer
|
||||
from click import Group
|
||||
|
||||
predict_cmd = cast(Group, typer.main.get_command(app)).commands["predict"]
|
||||
truth_param = next(p for p in predict_cmd.params if p.name == "truth")
|
||||
assert truth_param.opts == ["--truth"]
|
||||
assert truth_param.secondary_opts == ["--no-truth"]
|
||||
|
||||
Reference in New Issue
Block a user