feat(predict): enrich YAML sidecar with provenance and timing
CI / Sync project version with tag (hand-pushed tags only) (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 48s
CI / Lint (ruff check) (pull_request) Successful in 49s
CI / Format (ruff format) (pull_request) Successful in 49s
CI / Tests (pull_request) Successful in 3m14s
CI / Release (bump, changelog, badges, tag) on merge to master (pull_request) Has been skipped
CI / Sync project version with tag (hand-pushed tags only) (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 48s
CI / Lint (ruff check) (pull_request) Successful in 49s
CI / Format (ruff format) (pull_request) Successful in 49s
CI / Tests (pull_request) Successful in 3m14s
CI / Release (bump, changelog, badges, tag) on merge to master (pull_request) Has been skipped
`giant predict`'s sidecar previously stopped at kind/prediction_id/ output/dataset/checkpoint/timestamp, unlike `giant rollout`'s, which carries full run provenance (model_config, training_epoch, training_config, timing, ...) that flows into analysis gallery metadata. `analyze --prediction` consumed the same thin sidecar, so a prediction series in an analysis run was nearly unlabeled compared to its rollout counterparts. - `_write_prediction_ref` takes an `extra: dict | None` merged into the sidecar; `giant rollout` now uses it instead of a load/update/rewrite round trip (identical output). - New `_build_predict_timing`, key-compatible with `_build_rollout_timing`, from timers now wrapping predict's setup/ sample/write phases. - `giant predict` writes coord, has_truth, schema_version, steps, weights, device, batch_size(+auto), row/skip/unknown-pdg counts, timing, and the checkpoint's model_config/config_overrides/ training_epoch/best_val_loss/training_config/training_meta. - `giant/analysis/condor.py`'s `_PLOT_META_KEYS` forwards the new predict-only keys (plus rollout's previously-unforwarded config_overrides) into each plot's gallery metadata.yaml. - Fixes a `ty` regression from the prior commit in tests/test_cli_predict.py (Command has no static `.commands`). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PpxE9nij3ujg9XcuzvQ26q
This commit is contained in:
@@ -6,6 +6,7 @@ from typer.testing import CliRunner
|
||||
|
||||
from giant.cli import (
|
||||
_CEPH_PREDICTIONS,
|
||||
_build_predict_timing,
|
||||
_resolve_prediction_output,
|
||||
_write_prediction_ref,
|
||||
app,
|
||||
@@ -145,6 +146,46 @@ def test_ref_timestamp_is_iso_format(tmp_path):
|
||||
assert ts.tzinfo is not None
|
||||
|
||||
|
||||
def test_ref_yaml_merges_extra_after_base_fields(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
checkpoint = ckpt_dir / "best.pt"
|
||||
checkpoint.touch()
|
||||
|
||||
out = tmp_path / "pred.parquet"
|
||||
dataset = tmp_path / "full.manifest"
|
||||
pred_uuid = str(uuid.uuid4())
|
||||
|
||||
ref_path = _write_prediction_ref(
|
||||
checkpoint,
|
||||
pred_uuid,
|
||||
out,
|
||||
dataset,
|
||||
extra={"coord": "global", "n_rows": 42, "timing": {"setup_s": 1.0}},
|
||||
)
|
||||
data = yaml.safe_load(ref_path.read_text())
|
||||
|
||||
# Base fields untouched, extras layered on top.
|
||||
assert data["kind"] == "prediction"
|
||||
assert data["prediction_id"] == pred_uuid
|
||||
assert data["coord"] == "global"
|
||||
assert data["n_rows"] == 42
|
||||
assert data["timing"] == {"setup_s": 1.0}
|
||||
|
||||
|
||||
def test_ref_yaml_without_extra_matches_today(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
checkpoint = ckpt_dir / "best.pt"
|
||||
checkpoint.touch()
|
||||
|
||||
pred_uuid = str(uuid.uuid4())
|
||||
ref_path = _write_prediction_ref(checkpoint, pred_uuid, tmp_path / "p.parquet", tmp_path / "d")
|
||||
data = yaml.safe_load(ref_path.read_text())
|
||||
|
||||
assert set(data) == {"kind", "prediction_id", "output", "dataset", "checkpoint", "timestamp"}
|
||||
|
||||
|
||||
def test_ref_checkpoint_path_is_absolute(tmp_path):
|
||||
ckpt_dir = tmp_path / "checkpoints"
|
||||
ckpt_dir.mkdir()
|
||||
@@ -232,13 +273,52 @@ def test_predict_truth_metadata_key_exists():
|
||||
assert PREDICT_TRUTH_METADATA_KEY == "giant.predict.has_truth"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _build_predict_timing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_predict_timing_computes_per_step_cost():
|
||||
timing = _build_predict_timing(
|
||||
setup_s=1.0,
|
||||
predict_s=10.0,
|
||||
write_s=2.0,
|
||||
n_rows=100,
|
||||
device="cpu",
|
||||
torch_threads=4,
|
||||
)
|
||||
assert timing["n_rows"] == 100
|
||||
assert timing["sample_s"] == 8.0 # predict_s - write_s
|
||||
assert timing["us_per_step"] == 8.0 / 100 * 1e6
|
||||
assert timing["write_us_per_step"] == 2.0 / 100 * 1e6
|
||||
assert timing["rows_per_s"] == 10.0
|
||||
assert timing["device"] == "cpu" and timing["torch_threads"] == 4
|
||||
|
||||
|
||||
def test_build_predict_timing_handles_zero_rows():
|
||||
timing = _build_predict_timing(
|
||||
setup_s=1.0,
|
||||
predict_s=0.0,
|
||||
write_s=0.0,
|
||||
n_rows=0,
|
||||
device="cpu",
|
||||
torch_threads=1,
|
||||
)
|
||||
assert timing["us_per_step"] is None
|
||||
assert timing["write_us_per_step"] is None
|
||||
assert timing["rows_per_s"] is None
|
||||
|
||||
|
||||
def test_predict_has_truth_flag_default_on():
|
||||
# Inspecting rendered --help text is brittle across terminal
|
||||
# widths/color settings (wraps or re-colors mid-flag); go straight to
|
||||
# the underlying click command's registered option instead.
|
||||
import typer
|
||||
from typing import cast
|
||||
|
||||
predict_cmd = typer.main.get_command(app).commands["predict"]
|
||||
import typer
|
||||
from click import Group
|
||||
|
||||
predict_cmd = cast(Group, typer.main.get_command(app)).commands["predict"]
|
||||
truth_param = next(p for p in predict_cmd.params if p.name == "truth")
|
||||
assert truth_param.opts == ["--truth"]
|
||||
assert truth_param.secondary_opts == ["--no-truth"]
|
||||
|
||||
@@ -256,6 +256,46 @@ def test_prep_with_prediction_writes_run_meta(tmp_path: Path):
|
||||
assert partial.data["available"]
|
||||
|
||||
|
||||
def test_prep_forwards_predict_only_metadata_keys(tmp_path: Path):
|
||||
"""A rich `giant predict` sidecar's provenance/timing keys reach
|
||||
run_meta.json's plot_meta, same as a rollout's do — a thin legacy
|
||||
sidecar (no such keys) still loads fine (see _write_prediction_yaml)."""
|
||||
rollout_yaml = _write_inputs(tmp_path)
|
||||
reference = load_rollout_yaml(rollout_yaml)["dataset"]
|
||||
pred = tmp_path / "pred_rich.parquet"
|
||||
_write_prediction(pred, coord="global")
|
||||
yaml_path = tmp_path / "pred_rich.yaml"
|
||||
yaml_path.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"kind": "prediction",
|
||||
"prediction_id": "richpred12",
|
||||
"output": str(pred),
|
||||
"dataset": str(reference),
|
||||
"checkpoint": "/ckpt/rich.pt",
|
||||
"coord": "global",
|
||||
"has_truth": True,
|
||||
"schema_version": "3",
|
||||
"n_input_rows": 1000,
|
||||
"n_files": 1,
|
||||
"n_skipped_rows": 3,
|
||||
"unknown_pdg_counts": {"999999": 3},
|
||||
"batch_size_auto": False,
|
||||
"timing": {"us_per_step": 12.5},
|
||||
}
|
||||
)
|
||||
)
|
||||
run_dir = _prep([rollout_yaml], prediction_yamls=[yaml_path])
|
||||
meta = RunMeta.load(run_dir / "run_meta.json")
|
||||
plot_meta = meta.predictions[0]["plot_meta"]
|
||||
assert plot_meta["coord"] == "global"
|
||||
assert plot_meta["has_truth"] is True
|
||||
assert plot_meta["n_input_rows"] == 1000
|
||||
assert plot_meta["n_skipped_rows"] == 3
|
||||
assert plot_meta["unknown_pdg_counts"] == {"999999": 3}
|
||||
assert plot_meta["timing"] == {"us_per_step": 12.5}
|
||||
|
||||
|
||||
def test_derive_run_dir_next_to_rollout():
|
||||
y = {"output": "/data/roll.parquet", "prediction_id": "abcd1234ef", "dataset": "d"}
|
||||
assert derive_run_dir([y]) == Path("/data/analysis_abcd1234")
|
||||
|
||||
Reference in New Issue
Block a user