lars
2358a75ee1
CI / Sync project version with tag (pull_request) Has been skipped
CI / Format (ruff format) (pull_request) Successful in 43s
CI / Lint (ruff check) (pull_request) Successful in 54s
CI / Type check (ty) (pull_request) Successful in 53s
CI / Tests (pull_request) Successful in 2m33s
CI / Bump version, tag, and update changelog on merge to master (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
Closes the roadmap's long-standing "no eval-latency number exists for any configuration" gap. Instruments `giant rollout` to record per-physical-step wall-clock cost in its YAML sidecar, adds a measured Geant4/miniCaloSim per-step reference (giant/analysis/geant4_reference.py, from a 3-energy, 4-event-count-per-energy local benchmark), and wires both into a new eval_cost_per_step PlotSpec in the giant analyze gallery. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
100 lines
3.1 KiB
Python
100 lines
3.1 KiB
Python
"""Thin CLI smoke coverage for `giant rollout` (issues.md Issue 5) — confirms
|
|
the CheckpointCompatibilityError raised by giant.checkpoint_io.load_for_inference
|
|
surfaces as a clean typer.Exit(1) with the expected message, end-to-end
|
|
through the CLI, not just at the giant.checkpoint_io unit level."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import torch
|
|
from typer.testing import CliRunner
|
|
|
|
from giant.cli import _build_rollout_timing, app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def test_build_rollout_timing_excludes_synthetic_rows_from_per_step_cost():
|
|
# 100 rows total, 30 of them synthetic termination markers (escape) ->
|
|
# us_per_step should be normalized over the 70 physical rows only, the
|
|
# same unit giant.analysis.geant4_reference measures Geant4 in.
|
|
timing = _build_rollout_timing(
|
|
setup_s=1.0,
|
|
rollout_s=10.0,
|
|
write_s=2.0,
|
|
n_rows=100,
|
|
termination_reason_counts={"escaped": 30, "natural_end": 70},
|
|
n_seed_events=5,
|
|
device="cpu",
|
|
torch_threads=4,
|
|
)
|
|
assert timing["n_rows"] == 100
|
|
assert timing["n_physical_rows"] == 70
|
|
assert timing["n_physical_rows"] < timing["n_rows"]
|
|
assert timing["sample_s"] == 8.0 # rollout_s - write_s
|
|
assert timing["us_per_step"] == 8.0 / 70 * 1e6
|
|
assert timing["write_us_per_step"] == 2.0 / 70 * 1e6
|
|
assert timing["ms_per_event"] == 10.0 / 5 * 1e3
|
|
assert timing["device"] == "cpu" and timing["torch_threads"] == 4
|
|
|
|
|
|
def test_build_rollout_timing_handles_zero_physical_rows_and_events():
|
|
timing = _build_rollout_timing(
|
|
setup_s=1.0,
|
|
rollout_s=1.0,
|
|
write_s=0.0,
|
|
n_rows=5,
|
|
termination_reason_counts={"escaped": 5},
|
|
n_seed_events=0,
|
|
device="cpu",
|
|
torch_threads=1,
|
|
)
|
|
assert timing["n_physical_rows"] == 0
|
|
assert timing["us_per_step"] is None
|
|
assert timing["write_us_per_step"] is None
|
|
assert timing["ms_per_event"] is None
|
|
|
|
|
|
def test_rollout_exits_1_on_checkpoint_missing_model_config(tmp_path):
|
|
checkpoint = tmp_path / "bad.pt"
|
|
torch.save({"sec_decoder": {}, "normalizer": {"sec_phys": {}}}, checkpoint)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"rollout",
|
|
"dummy.parquet",
|
|
"--checkpoint",
|
|
str(checkpoint),
|
|
"--geometry",
|
|
"dummy_geometry.pkl",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
assert "checkpoint has no model_config" in result.output
|
|
|
|
|
|
def test_rollout_set_flag_disallowed_path_surfaces_compat_error(tmp_path):
|
|
checkpoint = tmp_path / "ckpt.pt"
|
|
torch.save(
|
|
{"model_config": {"stage1_model": {}, "stage2_model": {}}, "sec_decoder": {}, "normalizer": {"sec_phys": {}}},
|
|
checkpoint,
|
|
)
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"rollout",
|
|
"dummy.parquet",
|
|
"--checkpoint",
|
|
str(checkpoint),
|
|
"--geometry",
|
|
"dummy_geometry.pkl",
|
|
"--set",
|
|
"stage2_model.n_sec.typo=sample",
|
|
],
|
|
)
|
|
|
|
assert result.exit_code == 1
|
|
assert "not an inference-safe override" in result.output
|