Rewrite analysis as streaming rollout-vs-reference plotting pipeline

Replace the monolithic giant/analysis.py (predict-local + RolloutVsTruth
diagnostics) with a lean giant/analysis/ package that compares one
autoregressive `giant rollout` for a checkpoint against a held-out
miniCaloSim reference file, and generates publication-styled plots in
parallel on HTCondor.

Rollout output and a raw reference file share a world-frame physical
column subset under identical names, so the old ALR/local-frame decode
machinery is gone — everything is world-frame mm/MeV.

- sources.py: canonical LazyFrames, synthetic-termination-row filtering,
  the secondary view (rollout generation>0 tracks vs reference sec_*_list).
- reduce.py: streaming primitives — a single hist1d group_by pass, per-event
  scalars, edep-weighted depth/transverse profiles, species share, leakage.
- context.py/grouping.py: prep resolves fixed bin edges + energy/pdg/material
  group sets once into shared.json, so each compute job is one pass, no range
  scan (histogram efficiency).
- catalog.py: declarative PlotSpec registry — marginals x {overall,energy,pdg,
  material}, per-event totals, shower profiles, species/leakage, secondaries.
- render.py: the only plotstyle/LaTeX importer; PDFs + gallery metadata.
- condor.py + `giant analyze` CLI (prep/compute-one/list/render/submit):
  one job per plot, compute/render split (workers polars-only, no LaTeX).

Styling via ETPlot's plotstyle (added to the analysis extra). New tests cover
the reduce primitives, catalog id uniqueness + compute, condor submit, and a
guarded render smoke test. Delete the two predict-diagnostics notebooks.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/test_analysis_reduce.py

@@ -0,0 +1,171 @@
+"""Tests for the streaming compute primitives (giant.analysis.reduce/sources/grouping)."""
+
+from __future__ import annotations
+
+import numpy as np
+import polars as pl
+
+from giant.analysis import grouping as G
+from giant.analysis import reduce as R
+from giant.analysis.sources import (
+    SYNTHETIC_TERMINATION_REASONS,
+    Side,
+    physical_steps,
+    secondaries,
+)
+
+
+def _rollout_frame() -> pl.LazyFrame:
+    # event 1: primary (2 steps) + 1 secondary track + 1 escaped bookkeeping row
+    # event 2: primary (1 step)
+    return pl.DataFrame(
+        {
+            "event_id": [1, 1, 1, 1, 2],
+            "track_id": [0, 0, 1, 0, 0],
+            "parent_id": [-1, -1, 0, -1, -1],
+            "generation": [0, 0, 1, 0, 0],
+            "step_no": [0, 1, 0, 99, 0],
+            "pdg": [11, 11, 22, 11, 11],
+            "pre_x": [0.0, 0.0, 0.0, 0.0, 0.0],
+            "pre_y": [0.0, 0.0, 0.0, 0.0, 0.0],
+            "pre_z": [0.0, 1.0, 1.0, 2.0, 0.0],
+            "pre_E": [100.0, 60.0, 20.0, 30.0, 50.0],
+            "pre_dx": [0.0, 0.0, 1.0, 0.0, 0.0],
+            "pre_dy": [0.0, 0.0, 0.0, 0.0, 0.0],
+            "pre_dz": [1.0, 1.0, 0.0, 1.0, 1.0],
+            "post_x": [0.0, 0.0, 1.0, 0.0, 0.0],
+            "post_y": [0.0, 0.0, 0.0, 0.0, 0.0],
+            "post_z": [1.0, 2.0, 1.0, 2.0, 1.0],
+            "post_E": [60.0, 30.0, 0.0, 0.0, 20.0],
+            "post_dx": [0.0, 0.0, 1.0, 0.0, 0.0],
+            "post_dy": [0.0, 0.0, 0.0, 0.0, 0.0],
+            "post_dz": [1.0, 1.0, 0.0, 1.0, 1.0],
+            "edep": [40.0, 30.0, 20.0, 0.0, 30.0],
+            "step_length": [1.0, 1.0, 1.0, 0.0, 1.0],
+            "material": ["G4_PbWO4"] * 5,
+            "layer_id": [0, 1, 1, -1, 0],
+            "n_sec_pred": [1, 0, 0, 0, 0],
+            "termination_reason": [
+                "",
+                "natural_end",
+                "natural_end",
+                "escaped",
+                "natural_end",
+            ],
+        }
+    ).lazy()
+
+
+def _reference_frame() -> pl.LazyFrame:
+    return pl.DataFrame(
+        {
+            "event_id": [1, 1, 2],
+            "track_id": [0, 0, 0],
+            "step_no": [0, 1, 0],
+            "pdg": [11, 11, 11],
+            "pre_x": [0.0, 0.0, 0.0],
+            "pre_y": [0.0, 0.0, 0.0],
+            "pre_z": [0.0, 1.0, 0.0],
+            "pre_E": [100.0, 60.0, 50.0],
+            "pre_dx": [0.0, 0.0, 0.0],
+            "pre_dy": [0.0, 0.0, 0.0],
+            "pre_dz": [1.0, 1.0, 1.0],
+            "post_x": [0.0, 0.0, 0.0],
+            "post_y": [0.0, 0.0, 0.0],
+            "post_z": [1.0, 2.0, 1.0],
+            "post_E": [60.0, 30.0, 20.0],
+            "post_dx": [0.0, 0.0, 0.0],
+            "post_dy": [0.0, 0.0, 0.0],
+            "post_dz": [1.0, 1.0, 1.0],
+            "edep": [40.0, 30.0, 30.0],
+            "step_length": [1.0, 1.0, 1.0],
+            "material": ["G4_PbWO4", "G4_PbWO4", "G4_Pb"],
+            "layer_id": [0, 1, 0],
+            "sec_E_list": [[20.0], [], [10.0]],
+            "sec_pdg_list": [[22], [], [22]],
+            "sec_dx_list": [[1.0], [], [0.0]],
+            "sec_dy_list": [[0.0], [], [0.0]],
+            "sec_dz_list": [[0.0], [], [1.0]],
+        }
+    ).lazy()
+
+
+def test_hist1d_overall_and_grouped():
+    lf = _rollout_frame()
+    edges = np.linspace(0.0, 50.0, 6)  # width 10
+    h = R.hist1d(lf, pl.col("edep"), edges)
+    # edep values: 40,30,20,0,30 -> bins [0),[10),[20),[30),[40)
+    assert h[0].tolist() == [1, 0, 1, 2, 1]
+    # grouped by pdg: pdg 22 has a single edep=20
+    hg = R.hist1d(lf, pl.col("edep"), edges, group=pl.col("pdg"))
+    assert hg[22].tolist() == [0, 0, 1, 0, 0]
+    assert hg[11].sum() == 4
+
+
+def test_physical_steps_drops_synthetic_rollout_rows_only():
+    lf = _rollout_frame()
+    phys = physical_steps(lf, Side.rollout).collect()
+    assert phys.height == 4  # dropped the escaped bookkeeping row
+    assert "escaped" not in phys["termination_reason"].to_list()
+    assert SYNTHETIC_TERMINATION_REASONS  # non-empty guard
+    # reference passes through unchanged
+    ref = _reference_frame()
+    assert physical_steps(ref, Side.reference).collect().height == ref.collect().height
+
+
+def test_event_scalars_totals_include_all_rows():
+    lf = _rollout_frame()
+    es = R.event_scalars(lf).sort("event_id")
+    row1 = es.filter(pl.col("event_id") == 1).to_dicts()[0]
+    assert row1["total_edep"] == 90.0  # 40+30+20+0
+    assert row1["incident_E"] == 100.0
+    assert row1["n_steps"] == 4
+
+
+def test_secondaries_rollout_vs_reference_align():
+    r = secondaries(_rollout_frame(), Side.rollout).collect().sort("event_id")
+    assert r["energy"].to_list() == [20.0]  # only the generation>0, step_no==0 row
+    assert r["pdg"].to_list() == [22]
+    t = secondaries(_reference_frame(), Side.reference).collect().sort("event_id")
+    # two secondaries (event 1 and event 2); empty list dropped
+    assert sorted(t["energy"].to_list()) == [10.0, 20.0]
+    assert t["pdg"].to_list() == [22, 22]
+
+
+def test_leakage_fraction():
+    frac = R.leakage_fraction(_rollout_frame())
+    # event 1: escaped pre_E=30, deposited=90 -> 30/120 = 0.25; event 2: 0
+    assert sorted(round(f, 6) for f in frac) == [0.0, 0.25]
+
+
+def test_weighted_profile_matches_manual_bincount():
+    lf = _rollout_frame()
+    ea = R.entry_axis(lf)
+    lf2 = R.attach_entry_axis(lf, ea)
+    edges = np.linspace(0.0, 3.0, 4)  # depth bins along +z
+    mean, std = R.weighted_profile(lf2, R.depth_expr(), edges, pl.col("edep"))
+    assert mean.shape == (3,)
+    # totals conserved: sum over bins == mean total edep per event
+    assert np.isclose(mean.sum() * 1, (90.0 + 30.0) / 2)  # 2 events
+
+
+def test_energy_bins_edges_and_event_map():
+    incident = np.array([100.0, 100.0, 1000.0, 1000.0])
+    edges = G.energy_bin_edges(incident, n_bins=2)
+    assert len(edges) == 3 and edges[0] <= 100.0 < edges[-1]
+    ids, bins = G.event_energy_bins(_rollout_frame(), edges)
+    assert set(bins.tolist()) <= {0, 1}
+    assert len(ids) == 2
+
+
+def test_digitize_expr_matches_numpy():
+    edges = np.array([0.0, 10.0, 100.0, 1000.0])
+    df = pl.DataFrame({"v": [5.0, 50.0, 500.0, 2000.0]})
+    got = df.select(G.digitize_expr(pl.col("v"), edges).alias("b"))["b"].to_list()
+    assert got == np.digitize([5.0, 50.0, 500.0, 2000.0], edges[1:-1]).tolist()
+
+
+def test_pdg_and_material_labels():
+    assert G.pdg_label(22) == "gamma"
+    assert G.pdg_label(999999) == "999999"
+    assert G.material_label("G4_PbWO4") == "PbWO4"

tests/test_catalog.py

@@ -0,0 +1,69 @@
+"""Tests for the plot catalog: id uniqueness + every spec computes a valid Reduced."""
+
+from __future__ import annotations
+
+import pytest
+
+from giant.analysis import build_catalog, catalog_ids, get_spec
+from giant.analysis.catalog import Bundle
+from giant.analysis.context import build_context
+from tests.test_analysis_reduce import _reference_frame, _rollout_frame
+
+
+@pytest.fixture(scope="module")
+def bundle() -> Bundle:
+    r, t = _rollout_frame(), _reference_frame()
+    ctx = build_context(
+        r, t, n_energy_bins=2, n_marginal_bins=10, top_k_pdg=3, sample_rows=1000
+    )
+    return Bundle.open(r, t, ctx)
+
+
+def test_catalog_ids_unique_and_nonempty():
+    ids = catalog_ids()
+    assert ids and len(ids) == len(set(ids))
+    # the required families are all present
+    fams = {s.family for s in build_catalog()}
+    assert {"marginals", "event", "shower", "species", "secondaries"} <= fams
+
+
+def test_get_spec_roundtrip_and_unknown():
+    spec = get_spec("marginal_edep")
+    assert spec.id == "marginal_edep" and spec.family == "marginals"
+    with pytest.raises(KeyError):
+        get_spec("does_not_exist")
+
+
+def test_every_spec_computes_valid_reduced(bundle: Bundle):
+    for spec in build_catalog():
+        r = spec.compute(bundle)
+        assert r.id == spec.id
+        assert r.kind in {
+            "overlay_hist",
+            "grouped_hist",
+            "profile",
+            "bar",
+            "single_hist",
+        }
+        assert r.title and r.xlabel
+        _validate_payload(r)
+
+
+def _validate_payload(r) -> None:
+    p = r.payload
+    if r.kind == "overlay_hist":
+        n = len(p["edges"]) - 1
+        assert len(p["rollout"]) == n and len(p["reference"]) == n
+    elif r.kind == "single_hist":
+        assert len(p["rollout"]) == len(p["edges"]) - 1
+    elif r.kind == "grouped_hist":
+        n = len(p["edges"]) - 1
+        assert p["groups"], "grouped hist must have at least one group"
+        for g in p["groups"].values():
+            assert len(g["rollout"]) == n and len(g["reference"]) == n
+    elif r.kind == "profile":
+        n = len(p["edges"]) - 1
+        for k in ("rollout_mean", "rollout_std", "reference_mean", "reference_std"):
+            assert len(p[k]) == n
+    elif r.kind == "bar":
+        assert len(p["labels"]) == len(p["rollout"]) == len(p["reference"])

tests/test_condor.py

@@ -0,0 +1,86 @@
+"""Tests for the HTCondor submit description + the compute_one round-trip."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from giant.analysis import (
+    Context,
+    SubmitConfig,
+    build_context,
+    catalog_ids,
+    compute_one,
+    prep,
+    write_submit,
+)
+from giant.analysis.reduced import Reduced
+from tests.test_analysis_reduce import _reference_frame, _rollout_frame
+
+
+def test_prep_and_compute_one_roundtrip(tmp_path: Path):
+    r, t = _rollout_frame(), _reference_frame()
+    ctx = build_context(
+        r, t, n_energy_bins=2, n_marginal_bins=8, top_k_pdg=3, sample_rows=1000
+    )
+    shared = tmp_path / "shared.json"
+    ctx.save(shared)
+    assert Context.load(shared).top_pdgs == ctx.top_pdgs
+
+    out = compute_one("marginal_edep", r, t, shared, tmp_path / "marginal_edep.json")
+    reduced = Reduced.load(out)
+    assert reduced.id == "marginal_edep"
+    assert len(reduced.payload["rollout"]) == len(reduced.payload["edges"]) - 1
+
+
+def test_prep_writes_shared_json(tmp_path: Path):
+    r, t = _rollout_frame(), _reference_frame()
+    shared = prep(
+        r,
+        t,
+        tmp_path / "run",
+        n_energy_bins=2,
+        n_marginal_bins=8,
+        top_k_pdg=3,
+        sample_rows=1000,
+    )
+    assert shared.exists()
+    ctx = Context.load(shared)
+    assert set(ctx.var_ranges) == {"step_length", "edep", "delta_e", "post_E"}
+
+
+def test_write_submit_description(tmp_path: Path):
+    cfg = SubmitConfig(
+        rollout=tmp_path / "r.parquet",
+        reference=tmp_path / "t.parquet",
+        out_dir=tmp_path / "run",
+        accounting_group="cms",
+        repo_dir=tmp_path,
+    )
+    sub = write_submit(cfg)
+    txt = sub.read_text()
+    assert "universe = docker" in txt
+    assert "docker_image = mschnepf/slc7-condocker" in txt
+    assert "requirements = TARGET.ProvidesETPResources" in txt
+    assert "accounting_group = cms" in txt
+    assert "queue plotid from" in txt
+    # one queue item per catalog id
+    ids = (cfg.out_dir / "plotids.txt").read_text().split()
+    assert ids == catalog_ids()
+    # wrapper is executable and self-contained
+    wrapper = cfg.out_dir / "run_compute.sh"
+    assert wrapper.exists() and (wrapper.stat().st_mode & 0o111)
+    assert "giant analyze compute-one" in wrapper.read_text()
+
+
+def test_write_submit_remote_flag(tmp_path: Path):
+    cfg = SubmitConfig(
+        rollout=tmp_path / "r.parquet",
+        reference=tmp_path / "t.parquet",
+        out_dir=tmp_path / "run",
+        accounting_group="cms",
+        repo_dir=tmp_path,
+        remote=True,
+    )
+    txt = write_submit(cfg).read_text()
+    assert "+RemoteJob = True" in txt
+    assert "ProvidesETPResources" not in txt

tests/test_render.py

@@ -0,0 +1,92 @@
+"""Render smoke test — skipped where plotstyle / LaTeX is unavailable."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("plotstyle")
+
+from giant.analysis.reduced import Reduced  # noqa: E402
+
+
+def _try_render(reduced: list[Reduced], out: Path):
+    from giant.analysis.render import render_all
+
+    for r in reduced:
+        r.save(out / "reduced" / f"{r.id}.json")
+    return render_all(out / "reduced", out / "plots")
+
+
+def test_render_one_of_each_kind(tmp_path: Path):
+    reduced = [
+        Reduced(
+            "m",
+            "marginals",
+            "overlay_hist",
+            "Overlay",
+            "x",
+            {
+                "edges": [0, 1, 2, 3],
+                "rollout": [1, 2, 3],
+                "reference": [3, 2, 1],
+                "log_y": False,
+            },
+        ),
+        Reduced(
+            "g",
+            "marginals",
+            "grouped_hist",
+            "Grouped",
+            "x",
+            {
+                "edges": [0, 1, 2],
+                "groups": {"a": {"rollout": [1, 2], "reference": [2, 1]}},
+                "log_y": False,
+            },
+        ),
+        Reduced(
+            "p",
+            "shower",
+            "profile",
+            "Profile",
+            "depth",
+            {
+                "edges": [0, 1, 2],
+                "rollout_mean": [1, 2],
+                "rollout_std": [0.1, 0.2],
+                "reference_mean": [1.1, 1.9],
+                "reference_std": [0.1, 0.1],
+                "ylabel": "e",
+            },
+        ),
+        Reduced(
+            "b",
+            "species",
+            "bar",
+            "Bar",
+            "species",
+            {
+                "labels": ["e-", "gamma"],
+                "rollout": [0.6, 0.4],
+                "reference": [0.5, 0.5],
+                "ylabel": "frac",
+            },
+        ),
+        Reduced(
+            "s",
+            "species",
+            "single_hist",
+            "Single",
+            "x",
+            {"edges": [0, 1, 2], "rollout": [5, 1], "log_y": True},
+        ),
+    ]
+    try:
+        pdfs = _try_render(reduced, tmp_path)
+    except RuntimeError as e:  # LaTeX missing at render time
+        pytest.skip(f"LaTeX rendering unavailable: {e}")
+    assert len(pdfs) == len(reduced)
+    assert all(p.exists() for p in pdfs)
+    assert (tmp_path / "plots" / "metadata.yaml").exists()