From 8d46f27d1bc1eae6e866918dc2406dad47a02a09 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Thu, 23 Jul 2026 14:38:49 +0200 Subject: [PATCH] Reformat rollout_validation.ipynb with ruff format Co-Authored-By: Claude Sonnet 5 --- analysis/rollout_validation.ipynb | 68 +++++++++++++++++++------------ 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/analysis/rollout_validation.ipynb b/analysis/rollout_validation.ipynb index c4f5506..369c434 100644 --- a/analysis/rollout_validation.ipynb +++ b/analysis/rollout_validation.ipynb @@ -19,18 +19,18 @@ "source": [ "# GIANT rollout-vs-truth validation notebook\n", "\n", - "Diagnostics for a full autoregressive `giant rollout` shower, compared against a held-out ground-truth steps file (the same schema `giant train` consumes \u2014 see `giant.data.loader.load_steps`) rather than one-step-ahead `giant predict` output.\n", + "Diagnostics for a full autoregressive `giant rollout` shower, compared against a held-out ground-truth steps file (the same schema `giant train` consumes — see `giant.data.loader.load_steps`) rather than one-step-ahead `giant predict` output.\n", "\n", - "This is the sibling of `validation.ipynb`: that notebook checks whether one-step generation (conditioned on the *real* preceding state, every row) reproduces real marginals/correlations/shower observables. This one checks the thing that actually matters for deployment \u2014 whether a shower **rolled out autoregressively from the model's own outputs** still looks physical, which is where covariate shift (small per-step errors compounding across a track) would show up.\n", + "This is the sibling of `validation.ipynb`: that notebook checks whether one-step generation (conditioned on the *real* preceding state, every row) reproduces real marginals/correlations/shower observables. This one checks the thing that actually matters for deployment — whether a shower **rolled out autoregressively from the model's own outputs** still looks physical, which is where covariate shift (small per-step errors compounding across a track) would show up.\n", "\n", - "Built on `RolloutVsTruth`, which treats the rollout file as \"generated\" and the truth file as \"real\". Unlike the paired predict-parquet `source` (`pred_*`/`true_*` columns of the same row), the two files here are **independent, unpaired datasets** \u2014 a rollout doesn't replay real events row-for-row, so real/generated may have different lengths and there's no per-row correspondence. Everything below only ever compares real-vs-generated *distributions*, never individual paired rows, and every check still streams (no `SampleCollection`, no full-file materialization) \u2014 see `giant.analysis`'s module docstring for the `RolloutVsTruth` mechanics.\n", + "Built on `RolloutVsTruth`, which treats the rollout file as \"generated\" and the truth file as \"real\". Unlike the paired predict-parquet `source` (`pred_*`/`true_*` columns of the same row), the two files here are **independent, unpaired datasets** — a rollout doesn't replay real events row-for-row, so real/generated may have different lengths and there's no per-row correspondence. Everything below only ever compares real-vs-generated *distributions*, never individual paired rows, and every check still streams (no `SampleCollection`, no full-file materialization) — see `giant.analysis`'s module docstring for the `RolloutVsTruth` mechanics.\n", "\n", - "Same four tiers as `validation.ipynb`, all built on the same functions \u2014 pass a `RolloutVsTruth` in place of the predict-parquet path/LazyFrame everywhere:\n", + "Same four tiers as `validation.ipynb`, all built on the same functions — pass a `RolloutVsTruth` in place of the predict-parquet path/LazyFrame everywhere:\n", "\n", - "1. **stratified marginals** \u2014 per-dimension real-vs-generated, sliced by pdg/material/energy\n", - "2. **joint structure** \u2014 correlation matrices, physically-coupled pairwise plots, direction alignment\n", - "3. **physical constraints** \u2014 unit-norm directions, non-negative step_length/delta_e/edep (checked on the rollout's own output \u2014 with autoregression, a constraint violation early in a track can compound into later steps, unlike one-step-ahead validation)\n", - "4. **event-level (shower) observables** \u2014 total/mean/median energy and length per event, longitudinal/transverse profiles, shower-max depth, computed directly from the rollout shower against the truth file's own events (`compute_rollout_vs_truth_observables_pl`, the Tier 4 counterpart to `RolloutVsTruth`)" + "1. **stratified marginals** — per-dimension real-vs-generated, sliced by pdg/material/energy\n", + "2. **joint structure** — correlation matrices, physically-coupled pairwise plots, direction alignment\n", + "3. **physical constraints** — unit-norm directions, non-negative step_length/delta_e/edep (checked on the rollout's own output — with autoregression, a constraint violation early in a track can compound into later steps, unlike one-step-ahead validation)\n", + "4. **event-level (shower) observables** — total/mean/median energy and length per event, longitudinal/transverse profiles, shower-max depth, computed directly from the rollout shower against the truth file's own events (`compute_rollout_vs_truth_observables_pl`, the Tier 4 counterpart to `RolloutVsTruth`)" ] }, { @@ -47,7 +47,7 @@ " \"/ceph/lbogner/geant_steps/predictions/9e76bc2c-f4ef-4488-9f62-b6d14e1f298e.parquet\"\n", ")\n", "# Any held-out file sharing giant train's input schema (real miniCaloSim\n", - "# steps) \u2014 e.g. the val split the rollout's seed events were drawn from.\n", + "# steps) — e.g. the val split the rollout's seed events were drawn from.\n", "TRUTH_FILE = (\n", " \"/ceph/lbogner/geant_steps/processed/steps/gen3/schema2/pbwo4/shard-009.parquet\"\n", ")\n", @@ -55,7 +55,7 @@ "# sample_frac subsamples each side of the Tier 1-3 checks independently\n", "# (kept memory-bounded for large files); defaults to every row. Tier 4\n", "# (compute_rollout_vs_truth_observables_pl, below) always streams every row\n", - "# regardless \u2014 per-event sums would be silently corrupted by row subsampling.\n", + "# regardless — per-event sums would be silently corrupted by row subsampling.\n", "SOURCE = RolloutVsTruth(rollout=ROLLOUT_FILE, truth=TRUTH_FILE)" ] }, @@ -138,7 +138,7 @@ "outputs": [], "source": [ "# Real vs. generated Pearson correlation matrices (+ their difference) over\n", - "# the 9 raw target dims \u2014 catches a model that decorrelates targets that are\n", + "# the 9 raw target dims — catches a model that decorrelates targets that are\n", "# physically coupled even when every individual marginal looks clean.\n", "_ = plot_correlation_matrices(SOURCE)" ] @@ -150,7 +150,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Scatter for physically-coupled pairs (step_length/delta_e/edep) \u2014 the\n", + "# Scatter for physically-coupled pairs (step_length/delta_e/edep) — the\n", "# joint-structure check correlation matrices alone can't fully capture.\n", "_ = plot_pairwise(SOURCE, n_sample=10000)" ] @@ -162,7 +162,7 @@ "metadata": {}, "outputs": [], "source": [ - "# cos(angle) between post_dir and travel_dir \u2014 coupled through the\n", + "# cos(angle) between post_dir and travel_dir — coupled through the\n", "# scattering physics, so this is another joint-structure check.\n", "_ = plot_direction_alignment(SOURCE)" ] @@ -174,7 +174,7 @@ "source": [ "## Tier 3: physical constraints\n", "\n", - "Unit-norm direction vectors, non-negative step_length/delta_e/edep. `constraint_report_pl`/`plot_constraint_violations` only ever check the *generated* side (here the rollout output) \u2014 under autoregression a violation isn't just a one-off artifact, it can feed the next step's conditioning, so this is worth watching more closely here than in one-step-ahead validation." + "Unit-norm direction vectors, non-negative step_length/delta_e/edep. `constraint_report_pl`/`plot_constraint_violations` only ever check the *generated* side (here the rollout output) — under autoregression a violation isn't just a one-off artifact, it can feed the next step's conditioning, so this is worth watching more closely here than in one-step-ahead validation." ] }, { @@ -194,7 +194,7 @@ "source": [ "## Tier 4: event-level (shower) observables\n", "\n", - "Built on `compute_rollout_vs_truth_observables_pl`, not `compute_event_observables_pl` \u2014 the rollout file carries its own `track_id`/`termination_reason` columns the event-level aggregation needs, and the shower here already *is* a full autoregressive rollout rather than one-step generations re-aggregated by event. Entry axis/point and per-event totals are computed separately per side (rollout and truth events are unrelated), but depth/transverse bin edges are shared across both so the profiles below overlay on one binning.\n", + "Built on `compute_rollout_vs_truth_observables_pl`, not `compute_event_observables_pl` — the rollout file carries its own `track_id`/`termination_reason` columns the event-level aggregation needs, and the shower here already *is* a full autoregressive rollout rather than one-step generations re-aggregated by event. Entry axis/point and per-event totals are computed separately per side (rollout and truth events are unrelated), but depth/transverse bin edges are shared across both so the profiles below overlay on one binning.\n", "\n", "Returns the same `EventObservables` `compute_event_observables_pl` does, so every plot function from `validation.ipynb` works unchanged here too." ] @@ -292,16 +292,17 @@ "source": [ "---\n", "\n", - "For the dataset-wide breakdown of which particle species contributed how much of the total energy/length (`pdg_contribution_table_pl`), see `validation.ipynb` \u2014 it needs the paired predict schema, which this rollout-vs-truth comparison doesn't have." + "For the dataset-wide breakdown of which particle species contributed how much of the total energy/length (`pdg_contribution_table_pl`), see `validation.ipynb` — it needs the paired predict schema, which this rollout-vs-truth comparison doesn't have." ] }, { "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "source": [ "## Router gating showcase (MoE)\n", "\n", - "Every other section above is file-only \u2014 it reads `ROLLOUT_FILE` and never touches\n", + "Every other section above is file-only — it reads `ROLLOUT_FILE` and never touches\n", "a checkpoint (see `giant.analysis`'s module docstring). This section is the one\n", "deliberate exception: soft gate weights only exist inside the trained `Router`,\n", "not in the rollout parquet, so this loads the checkpoint that produced\n", @@ -316,6 +317,7 @@ { "cell_type": "code", "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", "metadata": {}, "outputs": [], "source": [ @@ -344,27 +346,38 @@ "\n", "if not hasattr(model, \"router\"):\n", " raise RuntimeError(\n", - " f\"{CHECKPOINT} has no router \u2014 it was trained with model.router.enabled=False\"\n", - " )\n" + " f\"{CHECKPOINT} has no router — it was trained with model.router.enabled=False\"\n", + " )" ] }, { "cell_type": "code", "execution_count": null, + "id": "9a63283cbaf04dbcab1f6479b197f3a8", "metadata": {}, "outputs": [], "source": [ "# Pre-step conditioning for every row of the rollout shower, reconstructed\n", "# the same way `giant predict`/`giant rollout` do (giant.data.transforms).\n", - "cols = [\"pdg\", \"pre_x\", \"pre_y\", \"pre_z\", \"pre_E\", \"pre_dx\", \"pre_dy\", \"pre_dz\",\n", - " \"material\", \"layer_id\"]\n", + "cols = [\n", + " \"pdg\",\n", + " \"pre_x\",\n", + " \"pre_y\",\n", + " \"pre_z\",\n", + " \"pre_E\",\n", + " \"pre_dx\",\n", + " \"pre_dy\",\n", + " \"pre_dz\",\n", + " \"material\",\n", + " \"layer_id\",\n", + "]\n", "df = pl.read_parquet(ROLLOUT_FILE, columns=cols)\n", "\n", "# Rows whose pdg/material fell outside the training vocab can't be encoded\n", "# (mirrors the pdg_mask filtering in `giant predict`'s CLI path).\n", - "known = df[\"pdg\"].map_elements(lambda p: int(p) in pdg_map, return_dtype=pl.Boolean) & df[\n", - " \"material\"\n", - "].map_elements(lambda m: str(m) in mat_map, return_dtype=pl.Boolean)\n", + "known = df[\"pdg\"].map_elements(\n", + " lambda p: int(p) in pdg_map, return_dtype=pl.Boolean\n", + ") & df[\"material\"].map_elements(lambda m: str(m) in mat_map, return_dtype=pl.Boolean)\n", "n_dropped = (~known).sum()\n", "if n_dropped:\n", " print(f\"dropping {n_dropped}/{len(df)} rows with unknown pdg/material\")\n", @@ -385,18 +398,19 @@ "ck = torch.from_numpy(cond_cat).long()\n", "\n", "with torch.no_grad():\n", - " gate_weights = model.router.gate(cc, ck).numpy() # (N, n_experts), rows sum to 1\n" + " gate_weights = model.router.gate(cc, ck).numpy() # (N, n_experts), rows sum to 1" ] }, { "cell_type": "code", "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", "metadata": {}, "outputs": [], "source": [ - "# EnergyRouter gates on pre-step energy, so that's the natural x-axis here \u2014\n", + "# EnergyRouter gates on pre-step energy, so that's the natural x-axis here —\n", "# swap for a categorical plot if this checkpoint used a different router type.\n", - "_ = plot_router_gating(data[\"pre_E\"], gate_weights, x_label=\"pre_E\")\n" + "_ = plot_router_gating(data[\"pre_E\"], gate_weights, x_label=\"pre_E\")" ] } ],