feat: add eval-cost benchmark — Geant4 reference vs surrogate rollout timing
CI / Sync project version with tag (pull_request) Has been skipped
CI / Format (ruff format) (pull_request) Successful in 43s
CI / Lint (ruff check) (pull_request) Successful in 54s
CI / Type check (ty) (pull_request) Successful in 53s
CI / Tests (pull_request) Successful in 2m33s
CI / Bump version, tag, and update changelog on merge to master (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
CI / Sync project version with tag (pull_request) Has been skipped
CI / Format (ruff format) (pull_request) Successful in 43s
CI / Lint (ruff check) (pull_request) Successful in 54s
CI / Type check (ty) (pull_request) Successful in 53s
CI / Tests (pull_request) Successful in 2m33s
CI / Bump version, tag, and update changelog on merge to master (pull_request) Has been skipped
CI / Publish package to Gitea package registry (pull_request) Has been skipped
Closes the roadmap's long-standing "no eval-latency number exists for any configuration" gap. Instruments `giant rollout` to record per-physical-step wall-clock cost in its YAML sidecar, adds a measured Geant4/miniCaloSim per-step reference (giant/analysis/geant4_reference.py, from a 3-energy, 4-event-count-per-energy local benchmark), and wires both into a new eval_cost_per_step PlotSpec in the giant analyze gallery. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,11 +8,52 @@ from __future__ import annotations
|
||||
import torch
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from giant.cli import app
|
||||
from giant.cli import _build_rollout_timing, app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def test_build_rollout_timing_excludes_synthetic_rows_from_per_step_cost():
|
||||
# 100 rows total, 30 of them synthetic termination markers (escape) ->
|
||||
# us_per_step should be normalized over the 70 physical rows only, the
|
||||
# same unit giant.analysis.geant4_reference measures Geant4 in.
|
||||
timing = _build_rollout_timing(
|
||||
setup_s=1.0,
|
||||
rollout_s=10.0,
|
||||
write_s=2.0,
|
||||
n_rows=100,
|
||||
termination_reason_counts={"escaped": 30, "natural_end": 70},
|
||||
n_seed_events=5,
|
||||
device="cpu",
|
||||
torch_threads=4,
|
||||
)
|
||||
assert timing["n_rows"] == 100
|
||||
assert timing["n_physical_rows"] == 70
|
||||
assert timing["n_physical_rows"] < timing["n_rows"]
|
||||
assert timing["sample_s"] == 8.0 # rollout_s - write_s
|
||||
assert timing["us_per_step"] == 8.0 / 70 * 1e6
|
||||
assert timing["write_us_per_step"] == 2.0 / 70 * 1e6
|
||||
assert timing["ms_per_event"] == 10.0 / 5 * 1e3
|
||||
assert timing["device"] == "cpu" and timing["torch_threads"] == 4
|
||||
|
||||
|
||||
def test_build_rollout_timing_handles_zero_physical_rows_and_events():
|
||||
timing = _build_rollout_timing(
|
||||
setup_s=1.0,
|
||||
rollout_s=1.0,
|
||||
write_s=0.0,
|
||||
n_rows=5,
|
||||
termination_reason_counts={"escaped": 5},
|
||||
n_seed_events=0,
|
||||
device="cpu",
|
||||
torch_threads=1,
|
||||
)
|
||||
assert timing["n_physical_rows"] == 0
|
||||
assert timing["us_per_step"] is None
|
||||
assert timing["write_us_per_step"] is None
|
||||
assert timing["ms_per_event"] is None
|
||||
|
||||
|
||||
def test_rollout_exits_1_on_checkpoint_missing_model_config(tmp_path):
|
||||
checkpoint = tmp_path / "bad.pt"
|
||||
torch.save({"sec_decoder": {}, "normalizer": {"sec_phys": {}}}, checkpoint)
|
||||
|
||||
Reference in New Issue
Block a user