Prune the uv cache at the end of every CI job

The uv cache lives on a persistent volume shared by every job on the
runner (/srv/act-runner-cache/uv), so nothing trimmed it and it grew
without bound. `uv cache prune --ci` drops the entries that are not worth
keeping between runs (pre-built wheels for local sources) while leaving
the downloaded-wheel cache that makes `uv sync` fast.

In bump-version the step carries the same is_merge guard as the rest of
that job, since uv is only set up on a merge push.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.3.6"
+current_version = "0.3.10"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 search = "{current_version}"

.gitea/workflows/ci.yml

@@ -28,6 +28,9 @@ jobs:
           echo "UV_LINK_MODE=copy" >> "$GITHUB_ENV"
       - run: uv sync --extra cpu --extra dev
       - run: uv run ruff check .
+      # The uv cache is a persistent volume shared by every job on this
+      # runner, so each job trims what it no longer needs before exiting.
+      - run: uv cache prune --ci
 
   ruff-format:
     name: Format (ruff format)
@@ -46,6 +49,7 @@ jobs:
           echo "UV_LINK_MODE=copy" >> "$GITHUB_ENV"
       - run: uv sync --extra cpu --extra dev
       - run: uv run ruff format --check .
+      - run: uv cache prune --ci
 
   type-check:
     name: Type check (ty)
@@ -64,6 +68,7 @@ jobs:
           echo "UV_LINK_MODE=copy" >> "$GITHUB_ENV"
       - run: uv sync --extra cpu --extra dev
       - run: uv run ty check .
+      - run: uv cache prune --ci
 
   test:
     name: Tests
@@ -87,6 +92,7 @@ jobs:
         with:
           name: coverage-report
           path: coverage.xml
+      - run: uv cache prune --ci
 
   bump-version:
     name: Bump version, tag, and update changelog on merge to master
@@ -173,6 +179,10 @@ jobs:
             git tag -a "$TAG" -m "$TAG"
             git push origin "refs/tags/$TAG"
           fi
+      # Same guard as every other step here: on a non-merge push uv was never
+      # set up, so there is nothing to prune.
+      - run: uv cache prune --ci
+        if: steps.merge_check.outputs.is_merge == 'true'
 
   sync-version-on-tag:
     name: Sync project version with tag
@@ -201,3 +211,4 @@ jobs:
           else
             echo "Tag version matches project version ($CURRENT_VERSION)"
           fi
+      - run: uv cache prune --ci

CHANGELOG.md

@@ -1,5 +1,41 @@
 # Changelog
 
+## [0.3.10] - 2026-08-26
+
+### Changed
+
+- Backfill CHANGELOG.md for v0.2.0-v0.3.2
+
+- Docs: bring README and CLAUDE.md in line with v0.3.9
+
+## [0.3.9] - 2026-08-24
+
+### Added
+
+- Add multi-rollout support to giant analyze [gitea #77](https://git.larsbogner.de/lars/giant/issues/77)
+
+
+### Changed
+
+- Escape LaTeX-special characters in plot titles/xlabels [gitea #81](https://git.larsbogner.de/lars/giant/issues/81)
+
+## [0.3.8] - 2026-08-24
+
+### Added
+
+- Add giant analyze metrics plots for training progress [gitea #75](https://git.larsbogner.de/lars/giant/issues/75)
+
+
+### Fixed
+
+- Fix LaTeX-unavailable skip check in analyze metrics smoke test
+
+## [0.3.7] - 2026-08-24
+
+### Added
+
+- Add rollout-quality distance, confusion, containment and router plots [gitea #76](https://git.larsbogner.de/lars/giant/issues/76)
+
 ## [0.3.6] - 2026-08-24
 
 ### Changed
@@ -37,4 +73,479 @@
 
 - Document CI_TOKEN's write:repository scope requirement [gitea #50](https://git.larsbogner.de/lars/giant/issues/50)
 
-# Changelog
+## [0.3.2] - 2026-08-17
+
+### Added
+
+- Add configs/baseline.toml as the kept reference model
+
+
+### Fixed
+
+- Clamp analysis histogram bins before the i32 cast, not after [gitea #61](https://git.larsbogner.de/lars/giant/issues/61)
+
+- Clip raw predicted log_mass in decode_secondaries [gitea #54](https://git.larsbogner.de/lars/giant/issues/54)
+
+
+### Changed
+
+- Let dwarf warm-cache take --config so it can't under-warm a config's cache keys [gitea #59](https://git.larsbogner.de/lars/giant/issues/59)
+
+- Implement n_sec.mode = "stop_token" for the AR secondary decoder [gitea #40](https://git.larsbogner.de/lars/giant/issues/40)
+
+- Bump patch version to 0.3.2
+
+## [0.3.1] - 2026-08-14
+
+### Added
+
+- Add an Objective registry for the flow/ddpm/wgan generator choice [gitea #32](https://git.larsbogner.de/lars/giant/issues/32)
+
+
+### Changed
+
+- Make trunk architecture selectable via a registry [gitea #33](https://git.larsbogner.de/lars/giant/issues/33)
+
+- Make ResBlock's conditioning-injection mechanism selectable [gitea #34](https://git.larsbogner.de/lars/giant/issues/34)
+
+- Make HistoryEncoder a pluggable registry, like Router/Objective [gitea #35](https://git.larsbogner.de/lars/giant/issues/35)
+
+- Deduplicate n_sec_head/type_head MLPs into build_mlp_head [gitea #36](https://git.larsbogner.de/lars/giant/issues/36)
+
+- Give the cond_cat/cond_cont column layout one owner [gitea #37](https://git.larsbogner.de/lars/giant/issues/37)
+
+- Give Stage1Model/Stage2OneShot/Stage2Autoregressive a shared StageModel base [gitea #39](https://git.larsbogner.de/lars/giant/issues/39)
+
+- Pass ConditioningAxisConfig/ParticleTypeConfig themselves instead of raw dicts [gitea #38](https://git.larsbogner.de/lars/giant/issues/38)
+
+- Bump patch version to 0.3.1
+
+## [0.3.0] - 2026-08-13
+
+### Added
+
+- Add v0.3.0 design doc: Stage-2 autoregressive redesign
+
+- Add pytest-cov to dev deps and run coverage in CI
+
+- Add coverage for router-center seeding, geometry batch reader, material topN cache, and setup-cache corruption paths
+
+- Add render.py coverage: figure params, router diagnostics plots, gallery/condor glue
+
+- Add unknown-key validation to config.toml merge (issues.md Issue 2)
+
+- Add consumed-keys audit test (issues.md Issue 5)
+
+
+### Fixed
+
+- Fix test_render_all_run_gallery_invokes_subprocess clobbering LaTeX's own subprocess.run
+
+
+### Removed
+
+- Remove issues.md
+
+
+### Changed
+
+- Refine v0.3.0 design: defaults, deferred scope, open questions
+
+- Document the differentiability position and its validation obligation
+
+- V0.3.0 step 1: new nested config schema, v0.2 migration shim
+
+- V0.3.0 step 2: network.py refactor to composable stage models
+
+- V0.3.0 step 3: per-stage train.py trainers + pipeline.py/cli.py rewrite
+
+- V0.3.0 step 4: type map + particle_type.target = "onehot"/"embedding"
+
+- V0.3.0 step 5: Stage2Autoregressive (history=markov) + §11.4 grad instrumentation
+
+- V0.3.0 step 6: sample.py/rollout.py AR generation + class->PDG decode
+
+- V0.3.0 step 7: AttentionHistory (KV-cached) + scheduled/never teacher forcing
+
+- V0.3.0 post-implementation audit: resolve all 9 tracked discrepancies
+
+- Refactor train.py into giant/training/ around a metrics collector
+
+- Silence the fork-safety warning from num_workers>0 pipeline tests
+
+- Deduplicate giant/training/trainers.py shared per-stage logic
+
+- Rewrite README for v0.3.0 architecture, quick start, and data columns
+
+- Bump version to 0.3.0
+
+- Delete docs/v0.3.0-design.md and strip all references to it
+
+- Apply ruff format
+
+- Downgrade coverage-report upload to actions/upload-artifact@v3
+
+- Bump ruff line-length to 120 and reformat
+
+- Make config dataclasses the single source of truth for DEFAULT_CONFIG
+
+- Extract giant train/new-run's CLI override mapping into a table-driven function (issues.md Issues 3 & 4)
+
+- Mark issues.md Issues 3 & 4 as fixed
+
+- Extract predict/rollout's duplicated inference bootstrap into giant.checkpoint_io (issues.md Issue 5)
+
+- Mark issues.md Issue 5 as fixed
+
+- Unify the two v0.2->v0.3 migration surfaces (issues.md Issue 6)
+
+- Type the data/model/training batch contracts with NamedTuples (issues.md Issue 7)
+
+- Split giant/model/network.py into giant/model/ (issues.md Issue 8)
+
+- Move scripts/ to giant/tools/ (issues.md Issue 9)
+
+- Reject stage2_model.stage1_context = 'sampled' as unimplemented (issues.md Issue 1)
+
+- Honour wgan.critic_hidden_dim/critic_n_res_blocks in build_critics [gitea #28](https://git.larsbogner.de/lars/giant/issues/28)
+
+- Validate stage2_model.autoregressive.order in validate_config [gitea #30](https://git.larsbogner.de/lars/giant/issues/30)
+
+- Decouple secondary-species vocabulary from conditioning.particle.emb_dim [gitea #29](https://git.larsbogner.de/lars/giant/issues/29)
+
+- Skip router auxiliary loss compute when their lambda is 0 [gitea #31](https://git.larsbogner.de/lars/giant/issues/31)
+
+## [0.2.0] - 2026-08-04
+
+### Added
+
+- Add CLAUDE.md with architecture overview and dev commands
+
+- Add streaming data pipeline and giant CLI entry point
+
+- Add giant predict command
+
+- Add ROOT-to-parquet conversion script with convert dependency group
+
+- Add post_pos as a model target via travel_dir decomposition
+
+- Add --coord local mode to predict for raw-space prediction debugging
+
+- Add KL divergence to marginal validation and hook it into the training loop
+
+- Add graceful shutdown on SIGINT/SIGTERM
+
+- Add configurable dropout to ResBlocks
+
+- Add giant.analysis module for notebook-based model quality diagnostics
+
+- Add lazy polars I/O and duplicate KL/constraint checks for giant.analysis
+
+- Add ruff and ty as dev dependencies, fix lint/type findings
+
+- Add linear warmup before cosine LR decay
+
+- Add --batch-size auto to estimate batch size from free GPU memory
+
+- Add hyperparameter scan
+
+- Add --batch-size auto to predict, matching train
+
+- Add tqdm progress bar to predict
+
+- Add KL bar plots and sample_frac to load_predicted_local; ignore root parquet scratch files
+
+- Add event-level shower observables to giant.analysis
+
+- Add total length traveled per event to event observables
+
+- Add pdg energy/length contribution pie plots
+
+- Add export script for Tier 4 event-level/pdg-share plots
+
+- Add mean/median deposited energy and step length plots per event
+
+- Add export script for ETP group-update presentation plots
+
+- Add photon edep export scripts and per-step presentation plots
+
+- Add tooling for a versioned geant_steps dataset layout
+
+- Add --copy mode to migrate_geant_steps.py
+
+- Add update-manifest and create-manifest subcommands to bump_dataset_version
+
+- Add --to flag for bump-gen/bump-schema and --gen flag for update-manifest
+
+- Add disk usage summary to dwarf status
+
+- Add file counts and reference tracking to dwarf status
+
+- Add --comment option to predict, recorded in YAML sidecar
+
+- Add energy-conservation PoC ODE-step comparison scripts
+
+- Add autoregressive shower rollout driver
+
+- Add fast slab lookup for the GeometryOracle, replacing knn as the default
+
+- Add load_rollout_vs_truth to compare rollouts against held-out truth data
+
+- Add mixture-of-experts routing prototype for Stage 1 and Stage 2
+
+- Add ProcessRouter for physics-process-based expert gating
+
+- Add PdgRouter for particle-type-based expert gating
+
+- Add ComposedRouter for multi-axis MoE gating
+
+- Add EMA weights, weight decay, step-based LR schedule, and grad-norm logging to training
+
+- Add WGAN-GP mode as a throwaway fast-eval experiment
+
+- Add router gating diagnostic for MoE checkpoints
+
+- Add Gitea Actions CI pipeline
+
+- Add configs for router energy (embedding/physical) and WGAN baseline runs
+
+- Add opt-in Weights & Biases logging for the training loop
+
+- Add test coverage for resolve_expert_dims
+
+- Add regression coverage for vocab/process index-map builders
+
+- Add dwarf warm-cache to precompute the setup-stage sidecar
+
+- Add giant new-run to scaffold a config.toml + run dir ahead of training
+
+- Add learnable per-expert width and shared temperature to EnergyRouter
+
+- Add opt-in straight-through Gumbel-softmax combine weights to MoE router
+
+- Add gumbel router configs sweeping learnable-knob combinations
+
+- Add gumbel/learn_centers/learn_width/learn_temperature to out-dir naming
+
+- Add bigger WGAN config (hidden_dim=512, n_blocks=6)
+
+- Add data-integrity guards against silent NaN/Inf propagation and races
+
+
+### Fixed
+
+- Fix column names to match actual parquet schema
+
+- Fix installed torch version to be compatible with cuda drivers
+
+- Fix miniCaloSim link in README
+
+- Fix giant.analysis import after Phase 2 dataset API changes
+
+- Fix silent failure modes surfaced by extensive code review
+
+- Fix ruff, ty, and pytest failures; apply ruff format
+
+- Clamp n_sec classification label to K_MAX
+
+- Fix rollout edep mismatch and add truth overlay to Tier 4 observables
+
+- Fix crashes in physical-property conditioning edge cases
+
+- Fix router experts silently ignoring --hidden-dim/--n-blocks
+
+- Fix conditioning="physical" so it can actually generalize past training vocab
+
+- Fix training-loop checkpoint/resume and WGAN bugs
+
+- Fix stale-partial reuse and n_chunks mismatch in analysis condor pipeline
+
+- Fix CLI/tooling robustness gaps and dedupe the Conditioning enum
+
+- Fix test_write_submit_requires_synced_venv for active-venv resolution
+
+
+### Removed
+
+- Remove scripts/train.py in favor of the giant train CLI
+
+- Drop orphaned child tracks instead of nulling secondary targets
+
+
+### Changed
+
+- Initial commit: giant surrogate model with two-phase roadmap in README
+
+- Implement Phase 1: full data pipeline, model, training, and config support
+
+- Handle material column as string type
+
+- Rename pre_energy/post_energy columns to pre_E/post_E
+
+- Rename direction columns from pre_dir_x/y/z to pre_dx/dy/dz
+
+- Batch StreamingStepsDataset internally instead of per-row collate
+
+- Dedup training pipeline, add seeding/resume and per-epoch metrics logging
+
+- Split torch into cpu/cuda extras and pin dependency version bounds
+
+- Apply ruff format and document lint/type tooling in CLAUDE.md
+
+- Update README to match current architecture and tooling
+
+- Make sampler step count configurable for validation
+
+- Calibrate auto batch size separately for inference vs training
+
+- Skip rows with unknown PDG codes during predict
+
+- Buffer predict rows across row-group boundaries before inference
+
+- Export plots for knowledge base
+
+- Rework validation notebook with markdown sections and Tier 4 plots
+
+- Allow steps_to_parquet.py to accept multiple ROOT input files
+
+- Encode edep/secondary/post energy as a conservation-constrained simplex
+
+- Expose dataset/conversion scripts as uv entry points
+
+- Restrict holdout overlap check to holdout vs dev/full only
+
+- Route predict output to UUID-named parquet with YAML reference sidecar
+
+- Implement Phase 2: secondary particle prediction
+
+- Unify dataset/tooling scripts into a single `dwarf` Typer CLI
+
+- Fold --to/--gen dataset-versioning flags into the dwarf CLI
+
+- Prefix default train output dir with current date
+
+- Color-code dwarf status output by tree level
+
+- Show VERSIONS.md reason extracts in dwarf status
+
+- Wire up predict CLI to load and run the Stage-2 sec_decoder
+
+- Wire up n_sec/species/energy-fraction validation for Stage 2
+
+- Detach Stage-2 type-embedding target to stop self-referential collapse
+
+- Weight Stage-2 secondary loss equally between direction and type-embedding dims
+
+- Recalibrate batch-size estimate for the post-Phase-2 model size
+
+- Update CLAUDE.md and README for the implemented Phase 2 model
+
+- Error on missing secondary lists instead of silently zeroing Stage-2 targets
+
+- Derive a unique per-job seed for minicalosim shard generation
+
+- Rescale secondary energies to exactly consume the e_sec budget
+
+- Support --energy-gev in dwarf make-root for the new minicalosim energy arg
+
+- Stream giant rollout output instead of buffering the whole run
+
+- Scale auto batch-size estimate by MoE expert count during training
+
+- Rewrite analysis module as a lean, fully-streaming pipeline
+
+- Reimplement rollout-vs-truth comparison on the streaming analysis module
+
+- Condition on material/particle physical properties instead of learned embeddings
+
+- Ignore the scratchpad working directory
+
+- Quote the on: key in the CI workflow
+
+- Split CI lint stage into parallel jobs
+
+- Rewrite analysis as streaming rollout-vs-reference plotting pipeline
+
+- Analyze: drive prep/submit from the rollout YAML sidecar
+
+- Analyze: show model/training params on rendered figures
+
+- Deps: install plotstyle from git.larsbogner.de package index
+
+- Analyze: drop stale ty:ignore on plotstyle import
+
+- Test: replace prep(**_CTX) splat with a typed _prep helper
+
+- Analyze: add MoE router gating/share diagnostic plots
+
+- Chore: remove stray CUDA sanity script and stale Phase 2 planning doc
+
+- Docs: document compute environment, WGAN/MoE status, and condor-gpu-train-rollout
+
+- Analyze: normalize pdg dtype in open_side to fix rollout/reference concat
+
+- Analyze: chunk per-plot aggregation across HTCondor jobs
+
+- Analyze: expose bin/pdg options on `analyze submit`
+
+- Analyze: estimate per-job HTCondor walltime from chunk row count
+
+- Analyze: run condor compute jobs via .venv/bin/giant, not uv run
+
+- Analyze: default condor docker image to alma9-gridjob
+
+- Analyze: raise default condor job memory request to 8192 MB
+
+- Analyze: recalibrate condor walltime model from real cluster timings
+
+- Transforms: pad legacy cond normalizers for pre-physical-conditioning checkpoints
+
+- Analyze: default run directory to <repo>/analysis_runs, gitignored
+
+- Docs: record first MoE router rollout benchmark result in the roadmap
+
+- Router: seed EnergyRouter centers from data quantiles instead of a fixed linspace
+
+- Docs: note the EnergyRouter centers_init fix in the roadmap
+
+- Analyze: thread full model/training/rollout/dataset params to plots
+
+- Ci: share one uv sync across jobs, gate tests on lint+type-check, sync tag/version on release tags
+
+- Ci: replace unsupported artifact sharing with a bind-mounted uv cache
+
+- Ci: stop setup-uv from overriding UV_CACHE_DIR
+
+- Ci: re-pin UV_CACHE_DIR after setup-uv, which exports its own value regardless of enable-cache
+
+- Ci: set UV_LINK_MODE=copy to silence the cross-filesystem hardlink warning
+
+- Log batch-level metrics to W&B, not just per-epoch summaries
+
+- Log router health, WGAN grad-norm split, n_sec accuracy, GPU/throughput to W&B
+
+- Persist global_step across --resume so W&B step stays monotonic
+
+- Timestamp default checkpoint dir to avoid W&B run-id collisions
+
+- Skip empty-slice mean/std in sec phys validation print
+
+- Speed up giant train's setup stage
+
+- Speed up _WelfordAccumulator's per-chunk update
+
+- Make default checkpoint out_dir name reflect only non-default hyperparams
+
+- Cache giant train's setup stage in a sidecar file
+
+- Pass --seed through to the train/val event split
+
+- Offset event_id per file to avoid cross-file collisions
+
+- Store a quantile grid instead of a raw reservoir sample in the setup cache
+
+- Scope wandb run config to only-active hyperparameters
+
+- Resolve giant condor wrapper from the active venv, not a hardcoded path
+
+- Bump version to 0.2.0

CLAUDE.md

@@ -7,18 +7,22 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 ```bash
 uv sync --extra cpu                # install dependencies with CPU-only torch (standard/default)
 uv sync --extra cuda               # install dependencies with CUDA 11.8 torch
-uv sync --extra cpu --extra dev    # add dev extras (pytest, etc.)
+uv sync --extra cpu --extra dev    # add dev extras (pytest, ruff, ty, bump-my-version, git-cliff, + all runtime extras)
 uv sync --extra cpu --extra geometry  # add scikit-learn for the geometry oracle (giant rollout)
+uv sync --extra cpu --extra workflow  # add b2luigi for `giant workflow` pipeline orchestration
 pytest               # run tests
 giant new-run --hidden-dim 512 --lr 3e-4        # scaffold a config.toml + run dir ahead of training
-giant train path/to/steps.parquet --mode flow   # train (flow matching)
-giant train path/to/steps.parquet --mode ddpm   # train (DDPM baseline)
-giant train path/to/steps.parquet --mode wgan   # train (WGAN-GP, single-pass eval; implemented, not yet tested)
-giant train path/to/steps.parquet --router --router-type energy  # MoE routing trunk (implemented; first rollout benchmark failed with lambda_balance=0, retrain needed — see Roadmap)
+giant train path/to/steps.parquet               # train (defaults: stage 1 flow, stage 2 wgan + autoregressive)
+giant train path/to/steps.parquet --mode flow   # set both stages' generative objective at once
+giant train path/to/steps.parquet --stage1-generator flow --stage2-generator wgan   # per-stage override
+giant train path/to/steps.parquet --router --router-type energy  # MoE routing trunk (see Roadmap for status)
+giant model summary --config config.toml         # build-only: parameter counts + which config keys actually bite
 giant predict path/to/steps.parquet --checkpoint ckpt/best.pt   # per-step predictions
 giant rollout path/to/steps.parquet --checkpoint ckpt/best.pt --geometry oracle.pkl  # full showers
-giant analyze submit rollout.yaml --accounting-group cms   # parallel rollout-vs-reference analysis on HTCondor
-giant analyze render <run_dir> --gallery                   # render PDFs + HTML gallery (run_dir from prep/submit)
+giant workflow run spec.toml --batch --workers 20          # whole pipeline (cache-warm -> train -> rollout -> analysis)
+giant analyze prep rollout.yaml --chunks 32                # lay out an analysis run dir (compute jobs come from the workflow)
+giant analyze render <run_dir> --gallery                   # render PDFs + HTML gallery (run_dir from prep)
+giant analyze metrics <train_run_dir>                      # training-progress plots from metrics.csv
 dwarf --help                                     # dataset/tooling CLI: convert, migrate, bump-gen,
                                                   # bump-schema, status, update-manifest, create-manifest,
                                                   # make-root, build-geometry-oracle, warm-cache, hparam-scan
@@ -27,6 +31,8 @@ dwarf --help                                     # dataset/tooling CLI: convert,
 
 `cpu` and `cuda` are mutually exclusive — pick one to select the torch build (pinned to 2.3.x; newer torch requires newer NVIDIA drivers). Plain `uv sync` with no extra will not install torch at all; uv has no concept of a "default extra", so `--extra cpu` should always be included unless you need GPU support.
 
+`configs/` holds kept reference configs (`baseline.toml`, `default.toml`, the router/WGAN scan configs) — pass them with `--config`.
+
 ### Lint and type checking
 
 ```bash
@@ -37,60 +43,86 @@ uv run ty check .       # type check
 
 Part of the `dev` extra. Run these periodically (not just at commit time) to catch drift early.
 
+### Release tooling
+
+Merges to `master` auto-bump the patch version, tag, and update `CHANGELOG.md` via the Gitea workflow in `.gitea/workflows/ci.yml` (bump-my-version + git-cliff). Don't hand-edit the version in `pyproject.toml` or write changelog entries by hand.
+
 ## Compute environment
 
 Work on this repo happens across three kinds of machine:
 
 - **Local dev machines** (laptop + desktop, identical): repo at `~/Programming/giant`, no access to `/ceph` — datasets, training results, and models aren't reachable here.
 - **Portal machines** (`portal1`, `deepthought`, `deepthought2`, `bms1`, `bms2`, `bms3`): repo lives under `/work`, and `/ceph` holds ROOT/parquet files and trained models. **These are shared with other users** — stay strictly within `/work/lbogner` and `/ceph/lbogner`, and keep resource usage to roughly a quarter of CPU/RAM and a single GPU so as not to disturb other users' jobs.
-- **HTCondor worker nodes**: never run or SSH onto these directly — the only sanctioned path is submitting jobs through condor (`giant analyze submit`, and the in-progress remote-GPU train/rollout submission on `condor-gpu-train-rollout`). `/ceph` is available there; `/work` is only sometimes mounted, depending on the node.
+- **HTCondor worker nodes**: never run or SSH onto these directly — the only sanctioned path is `giant workflow run <spec.toml> --batch` (b2luigi, see the Workflow section), which submits and polls every job. `/ceph` is available there; `/work` is only sometimes mounted, depending on the node.
 
 ## Architecture
 
-GIANT is a conditional generative surrogate for the Geant4 step function. It replaces the stochastic physics engine: given a pre-step particle state (conditioning), it samples a post-step outcome — now including the variable-length list of secondary particles the step produces (Phase 2, see Roadmap).
+GIANT is a conditional generative surrogate for the Geant4 step function. It replaces the stochastic physics engine: given a pre-step particle state (conditioning), it samples a post-step outcome — including the variable-length list of secondary particles the step produces.
 
-**Data pipeline** (`giant/data/`): parquet files from miniCaloSim are loaded into numpy arrays (`loader.py`), then log-transformed and rotated into a local coordinate frame where `pre_dir = ẑ` (`transforms.py`), before being wrapped in a PyTorch `Dataset` (`dataset.py`). Train/val split is by `event_id` to avoid leaking correlated steps from the same shower.
+**Data pipeline** (`giant/data/`): parquet files from miniCaloSim are loaded into numpy arrays (`loader.py`), then log-transformed and rotated into a local coordinate frame where `pre_dir = ẑ` (`transforms.py`), before being wrapped in a PyTorch `Dataset` (`dataset.py`, streaming variant included). Train/val split is by `event_id` (`--seed`-controlled) to avoid leaking correlated steps from the same shower. Loading a directory or `.manifest` of several parquet files offsets each file's `event_id`s by a per-file stride so ids stay globally unique. `setup_cache.py` persists the pre-epoch setup scan (vocab maps, event split, process maps, normalizer stats) as a sidecar so repeated runs over the same `data` path don't rescan (`--cache-setup`/`--rebuild-setup-cache`, precomputable with `dwarf warm-cache --config ...`).
 
 **Stage-1 output space (9D, `giant/constants.py:LOCAL_TARGET_NAMES`):** `log_step_length`, two additive-log-ratio (ALR) coordinates `edep_logit`/`sec_logit` of a **deposit / secondary / post-energy simplex**, `post_dir` (post-scattering momentum direction, unit vector in the local frame), and `travel_dir` (direction of `post_pos - pre_pos`, unit vector in the local frame). The energy simplex decodes via softmax over `[edep_logit, sec_logit, 0]` × `pre_E` so `edep + e_sec + post_E == pre_E` holds by construction — energy conservation is architectural, not learned (see `energy_simplex_decode`). `post_pos` is not a raw target — it's reconstructed at inference as `pre_pos + step_length * world_frame(travel_dir)`, since `step_length` already encodes that displacement's magnitude and duplicating it would let the two become inconsistent.
 
-**Conditioning vector (15D continuous, `COND_DIM`):** pre-step position, log(pre-energy), pre-step direction, layer ID (`COND_DIM_BASE=8`) — plus, since particle/material physical-property conditioning (`model.conditioning`, see below), 7 more columns: particle `log(mass)`/`charge` (`PARTICLE_PHYS_DIM=2`, `giant/particles.py`) and material `Z_eff`/`A_eff`/`log(density)`/`log(X0)`/`log(λ_int)` (`MATERIAL_PHYS_DIM=5`, `giant/materials.py`). `n_sec` and `e_sec` are **not conditioning inputs** (that was Phase 1 / the energy-conservation PoC); the model predicts them.
+**Conditioning vector (15D continuous, `COND_DIM`):** pre-step position, log(pre-energy), pre-step direction, layer ID (`COND_DIM_BASE=8`) — plus 7 physical-property columns: particle `log(mass)`/`charge` (`PARTICLE_PHYS_DIM=2`, `giant/particles.py`) and material `Z_eff`/`A_eff`/`log(density)`/`log(X0)`/`log(λ_int)` (`MATERIAL_PHYS_DIM=5`, `giant/materials.py`). `n_sec` and `e_sec` are **not conditioning inputs** — the model predicts them. `giant/cond_layout.py` is the single source of truth for the `cond_cont`/`cond_cat` column layout shared by `giant.data.transforms`, `giant.model.encoders`, and `giant.model.routers`.
 
-`ConditionEncoder`/`SecondaryConditionEncoder` (`giant/model/network.py`) support two mutually exclusive `conditioning` modes, selected per-checkpoint (`model_config["conditioning"]`, defaulting to `"embedding"` for old checkpoints without the key, `"physical"` for new `giant train` runs — see `--conditioning`):
-- **`"embedding"`** (original Phase 2 design): a learned `nn.Embedding` per PDG code / material name, indexed by a dataset-scoped dense vocab (`pdg_map`/`mat_map`). Memorizes the training menu.
-- **`"physical"`** (default): the 7 physical-property columns above are each routed through a small MLP (`particle_mlp`/`material_mlp`) to the same `emb_dim` width the embedding tables would have produced — a drop-in replacement computable for any PDG code / material name, not just ones seen in training, which is what lets the surrogate generalize to a held-out material or species. `giant/particles.py` decodes nuclear/ion PDG codes (the `10LZZZAAAI` scheme) via the scikit-HEP `particle` package with a Z/A-digit-decode fallback for isomer codes the package's ground-state-only table misses. `giant/materials.py` ships real Geant4-11.4.1-derived `z_eff`/`a_eff`/`density`/`x0`/`lambda_int` values for every material the detector geometry actually produces; the sole exception is `G4_LYSO` (not a stock Geant4 NIST material, never actually constructed by the geometry — see the module docstring), which stays `MaterialProperties(None, ...)` and raises loudly (`MaterialPropertiesNotFilledError`) rather than silently defaulting if it's ever requested.
+`ConditionEncoder` (`giant/model/encoders.py`) configures the particle and material identity axes **independently** (`conditioning.particle` / `conditioning.material`, each a `ConditioningAxisConfig` with `type`/`emb_dim`/`n_layers`), so they may mix freely. Three per-axis modes:
+- **`"physical"`** (default): the axis's raw physical properties routed through a small MLP — computable for any PDG code / material name, which is what lets the surrogate generalize beyond the training menu. `giant/particles.py` decodes nuclear/ion PDG codes (the `10LZZZAAAI` scheme) via the scikit-HEP `particle` package with a Z/A-digit-decode fallback for isomer codes the package's ground-state-only table misses. `giant/materials.py` ships real Geant4-11.4.1-derived values for every material the detector geometry actually produces; the sole exception is `G4_LYSO` (not a stock Geant4 NIST material, never actually constructed by the geometry — see the module docstring), which stays `MaterialProperties(None, ...)` and raises loudly (`MaterialPropertiesNotFilledError`) rather than silently defaulting.
+- **`"embedding"`**: a learned `nn.Embedding` per PDG code / material name, indexed by a dataset-scoped dense vocab. Memorizes the training menu; the generalization-comparison baseline, and the only mode compatible with `stage2_model.particle_type.target = "embedding"`.
+- **`"onehot"`**: a fixed, unlearned vector over the top `emb_dim - 1` codes by training-set count plus one "other" bin. Not a reparameterization of `"embedding"` — the vocabulary cap is the real difference.
 
-**Model** (`giant/model/network.py`): a two-stage model, both checkpointed together.
-- **Stage 1 — `DenoisingMLP`:** `ResBlock` stack with a `SinusoidalEmbedding` for the flow/diffusion time variable and a `ConditionEncoder` fusing the conditioning. Predicts the 9D primary vector field, plus an `n_sec_head` classifier over `{0..K_MAX}` (`K_MAX=15`) that runs on the condition encoding alone (no diffusion noise), callable via `predict_n_sec`.
-- **Stage 2 — `SecondaryDecoder`:** a second flow-matching net (`SecondaryConditionEncoder` fuses the pre-step conditioning with the Stage-1 outcome) that generates all `K_MAX` secondary slots at once. Each slot is `(stick-breaking energy logit, local-frame direction 3D, log-mass, charge)` = `SEC_SLOT_DIM=6`, ordered by descending energy; slots beyond the predicted `n_sec` are masked. Secondary energies are a **stick-breaking partition of the `e_sec` budget** from Stage 1 (they sum to it), so the whole chain conserves energy. A secondary's mass/charge are regressed directly against a fixed physics-derived target (its ground-truth PDG code's `giant.particles.particle_mass_charge`) — not a learned/moving embedding target, so nothing needs detaching. **No snapping at inference**: the predicted (mass, charge) are used as-is as the secondary's physical identity, including for its own future conditioning if it goes on to take further steps in a rollout. A separate, reporting-only nearest-known-PDG lookup (`giant.particles.nearest_known_pdg`) is used purely to populate a nominal `pdg` label for output rows / `"embedding"`-mode fallback conditioning — it never feeds back into the model.
+`conditioning.share_stages` decides whether the two stages get one shared encoder instance or two identically-configured independent ones.
 
-`schedule.py` provides both a `CosineSchedule` for DDPM and the flow matching loss utilities (Lipman et al. 2022 conditional flow matching).
+**Model** (`giant/model/`, both stages checkpointed together). `network.py` is only a re-export shim now; the real code is split by concern:
+- `layers.py` — `ResBlock`/`AdaLNResBlock` + `BLOCK_REGISTRY` (conditioning-injection mechanism is selectable), `SinusoidalEmbedding`, `ContextAdapter`, `build_mlp_head`.
+- `encoders.py` — `ConditionEncoder` (above).
+- `trunks.py` — `TRUNK_REGISTRY`/`build_trunk`: everything downstream of the fused conditioning vector, as a registrable expert *body* (`resmlp` default, plus a `none` variant). `RoutedTrunk` builds `router.n_experts` instances of whichever body is named, so mixing is orthogonal to which body is mixed.
+- `routers.py` — `Router` base + `ROUTER_REGISTRY`: `energy`/`pdg`/`process`/`composed`/`none`. Soft-mixed at train time, **top-1 dispatched at eval time** (each row runs exactly one small expert), which is the actual inference-speed win. `EnergyRouter`/`PdgRouter` gate on a quantity known at inference; `ProcessRouter` runs its own small classifier (process isn't known upfront); `ComposedRouter` gates jointly over outer-product expert cells via repeated `--router-axis "type:key=val,..."`. The `--router*`/`--n-experts` CLI flags target `stage1_model.router` only; stage 2's router is config-file-only (`stage2_model.router`). `EnergyRouter` accepts `centers_init`, which `giant/pipeline.py` auto-populates from real data quantiles via a reservoir sample collected during the normalizer-fitting pass.
+- `history.py` — `HISTORY_REGISTRY`/`build_history`: `markov` (previous token only), `attention` (causal self-attention, KV-cached at inference via `init_cache`/`step`), `none`. Stage-2 autoregressive only.
+- `objectives.py` — `Objective` base + registry for `flow`/`ddpm`/`wgan`: answers in one place whether a stage needs a time embedding, is adversarial, folds the secondary type slice into its trunk output, what its trunk input is, and which loss it trains against.
+- `models.py` — the composed stage models: `Stage1Model`, `Stage2OneShot`, `Stage2Autoregressive`, `CriticModel`, all on a shared `StageModel` base.
+- `builders.py` — `build_models`/`build_critics`, assembling the above from a config dict.
+- `schedule.py` (`CosineSchedule` for DDPM + conditional-flow-matching losses), `wgan.py` (gradient penalty / critic / generator losses, Gulrajani et al. 2017), `summary.py` (`giant model summary`), `_legacy.py` (v0.2 checkpoint migration).
 
-**Samplers** (`giant/sample.py`): DDPM, DDIM, and flow matching (ODE integration, ~10 steps). Flow matching is the primary mode.
+**Stage 1 — primary step.** Trunk (routed or not) over the fused conditioning, plus a `SinusoidalEmbedding` of the flow/diffusion time for non-adversarial objectives, predicting the 9D vector field. An `n_sec` classifier head over `{0..k_max}` runs on the condition encoding alone; `stage2_model.n_sec.owner` decides whether it lives on stage 1 (v0.2 checkpoints) or stage 2 (default).
 
-**WGAN-GP mode (`--mode wgan`, implemented, not yet tested):** a throwaway fast-eval alternative to the flow/DDPM samplers above — single forward pass instead of ~10 ODE steps. Dedicated noise-conditioned generators (`WGANGenerator`/`WGANSecondaryGenerator`, `giant/model/network.py`) stand in for `DenoisingMLP`/`SecondaryDecoder`, trained against `Critic`/`SecondaryCritic` discriminators with the gradient-penalty loss in `giant/model/wgan.py` (Gulrajani et al. 2017); `sample_wgan` (`giant/sample.py`) does the single-pass draw at inference. Not yet validated against the flow-matching baseline.
+**Stage 2 — secondaries.** Conditioned on the pre-step state plus a projected stage-1 outcome (`stage2_model.context_dim`; `stage1_context` selects ground-truth vs sampled context, annealable via `ctx_p_start`/`ctx_p_end`). Two decoders (`stage2_model.decoder`):
+- **`autoregressive`** (default): one secondary at a time in descending-energy order, each token conditioned on a `HistoryEncoder` summary of prior tokens, with teacher forcing (`always`/`scheduled`/`never`, `tf_p_start`/`tf_p_end`). `n_sec.mode = "stop_token"` lets the length be emitted by the sequence itself instead of the classifier head.
+- **`one_shot`**: all `k_max` slots in one pass, masked past the predicted `n_sec` (the v0.2 behaviour).
 
-**MoE routing trunk (`--router`, implemented; first rollout benchmark shows the experts don't specialize — see Roadmap):** an alternative to `DenoisingMLP`'s monolithic `ResBlock` trunk — a `Router` (`giant/model/network.py`, `ROUTER_REGISTRY`/`build_router`) gates between small per-expert `ResBlock` stacks (`Expert`), soft-mixed over all experts at train time but **top-1 dispatched at eval time** (each row runs exactly one small expert), which is the actual inference-speed win. Router types gate on different conditioning axes: `EnergyRouter`/`PdgRouter` read a quantity already known at inference time, `ProcessRouter` runs its own small classifier over pre-step conditioning (since process isn't known upfront); `ComposedRouter` gates jointly over multiple axes (outer-product expert cells) via repeated `--router-axis "type:key=val,..."` flags. Config lives under `model.router` (`giant/config.py`), deep-merged one level so `router.enabled` alone doesn't drop the rest of the defaults.
+Secondary energies are a **stick-breaking partition of the `e_sec` budget** from Stage 1 (they sum to it), so the whole chain conserves energy. Particle identity is set by `stage2_model.particle_type.target`: `"onehot"` (default — categorical over the top `n_classes - 1` PDG codes by training count plus "other", with configurable `other_policy` and `class_weighting`), `"physical"` (continuous `(log-mass, charge)` regressed against `giant.particles.particle_mass_charge`), or `"embedding"` (nearest-row snap into the conditioning embedding table; requires `conditioning.particle.type = "embedding"`).
 
-**Validation** (`giant/validate.py`): step-level marginal comparisons.
+**Samplers** (`giant/sample.py`): DDPM, DDIM, flow matching (ODE integration, ~10 steps), and single-pass WGAN, plus the stage-2 secondary sampling loop (one-shot and autoregressive).
 
-**Analysis** (`giant/analysis/`, `giant analyze` CLI): a lean, streaming rollout-vs-reference plotting pipeline that compares one autoregressive `giant rollout` (for a given checkpoint) against a held-out miniCaloSim reference steps file, and produces publication-styled PDFs assembled into an HTML gallery. It exploits the fact that rollout output and a raw reference file share a world-frame physical column subset under identical names (`pre_*`/`post_*`/`edep`/`step_length`/`pdg`/`material`/`event_id`), so no ALR/local-frame decode is needed — everything is world-frame mm/MeV. Structure: `sources.py` (canonical LazyFrames + synthetic-termination-row filtering + the secondary view, which is `generation>0 & step_no==0` rollout tracks vs exploded `sec_*_list` reference columns), `reduce.py` (the streaming primitives — a single `hist1d` `group_by([group,bin]).len()` pass, per-event scalars, edep-weighted depth/transverse profiles, species share, leakage), `grouping.py`/`context.py` (fixed bin edges + energy-quantile/pdg/material group sets resolved once by `prep` into `shared.json`, so every compute job is one pass with no range scan), `catalog.py` (the declarative `PlotSpec` registry — marginals × {overall,energy,pdg,material}, per-event totals, shower profiles, species/leakage, secondaries), and `render.py` (the only module importing ETPlot's `plotstyle`/LaTeX; dispatches on `Reduced.kind`, writes PDFs + `metadata.yaml`). **Input is a `giant rollout` YAML sidecar** (`condor.py:load_rollout_yaml`): its `output`/`dataset` keys name the rollout parquet and the seed file (= the reference truth), and the rest of the YAML (checkpoint, geometry oracle, cutoffs) flows into each plot's gallery metadata. `prep` derives its own **run directory** next to the rollout parquet (`<...>/analysis_<id>/`) holding `shared.json`, `run_meta.json`, `reduced_partial/`, `reduced/`, `plots/`. **Compute/merge/render split:** `giant analyze submit rollout.yaml --chunks N` runs `prep` (recording the run's chunk count `N` in `run_meta.json`) then submits one HTCondor job per (plot, chunk) pair (`compute-one --id --chunk --run-dir`, polars/numpy only — no LaTeX on workers), each streaming over an `event_id`-disjoint slice (`event_id % N == chunk`) and writing a small `reduced_partial/<id>__<chunk>.json`; every `PlotSpec` (`catalog.py`) splits into a `compute_partial`/`finalize` pair so a plot's chunks can be summed/concatenated back together correctly (`chunkable=False` specs — the router diagnostics, already bounded/subsampled — always run as a single chunk regardless of `N`). The local `giant analyze render <run_dir>` first joins every plot's chunk partials into `reduced/<id>.json` (`merge_all`, a no-op join when `N=1`), then turns those into the styled PDF/gallery tree. See `giant/analysis/__init__.py`.
+**Training** (`giant/training/`): `loop.py` (epoch loop, graceful shutdown, best-checkpoint selection), `trainers.py` (`StageSpec` + per-stage flow/ddpm and WGAN-GP trainers, and the `MetricSpec` declarations that define `metrics.csv`'s columns), `stage2_inputs.py` (ground-truth stage-2 targets + teacher-forcing inputs), `metrics.py` (`MetricsCollector`: `metrics.csv`, W&B logging, progress/summary), `checkpoint.py`, `amp.py` (`train.precision = fp32|bf16` autocast), `plots.py` (`giant analyze metrics`). Per-stage `init_from`/`freeze` lets one stage be retrained against a fixed, known-good other stage while still producing a complete rollout-capable checkpoint.
 
-**Shower rollout** (`giant/rollout.py`, `giant rollout` CLI): autoregressively steps the two-stage model into a full shower — each primary post-step becomes the next pre-step, secondaries are pushed as new tracks, and per-step `material`/`layer_id` come from a `GeometryOracle` (`giant/geometry.py`, built via `dwarf build-geometry-oracle`) that learns position → (material, layer_id) from data and flags detector escape by nearest-neighbour distance. Tracks terminate on energy cutoff, per-track max steps, escape, or natural end; energy is deposited locally on every stop except escape (leakage), so showers conserve energy by construction.
+**Config** (`giant/config.py`): frozen dataclasses are the single source of truth; `DEFAULT_CONFIG` is *generated* from `GiantConfig().to_dict()` rather than hand-maintained. Blocks: `[conditioning]`, `[stage1_model]`, `[stage2_model]`, `[train]`, `[meta]`. Unknown keys are rejected on merge (with a did-you-mean suggestion), and `tests/test_config_consumed_keys.py` audits that every key is actually read somewhere.
+
+**Validation** (`giant/validate.py`): step-level marginal + KL-divergence comparisons during training (`--validate-every`).
+
+**Analysis** (`giant/analysis/`, `giant analyze` CLI): a lean, streaming rollout-vs-reference plotting pipeline that compares one or more autoregressive `giant rollout` runs against a single held-out miniCaloSim reference steps file shared by all of them, and produces publication-styled PDFs assembled into an HTML gallery — one distinctly colored series per rollout, one reference line/panel. It exploits the fact that rollout output and a raw reference file share a world-frame physical column subset under identical names (`pre_*`/`post_*`/`edep`/`step_length`/`pdg`/`material`/`event_id`), so no ALR/local-frame decode is needed — everything is world-frame mm/MeV. Structure: `sources.py` (canonical LazyFrames + `RolloutSpec`/`Side` — a rollout's opened frames + per-checkpoint diagnostic inputs — + synthetic-termination-row filtering + the secondary view, which is `generation>0 & step_no==0` rollout tracks vs exploded `sec_*_list` reference columns), `variables.py` (the per-step value expressions shared by range sizing and the plot registry), `reduce.py` (the streaming primitives — a single `hist1d` `group_by([group,bin]).len()` pass, per-event scalars, edep-weighted depth/transverse profiles, species share, leakage), `grouping.py`/`context.py` (fixed bin edges + energy-quantile/pdg/material group sets resolved once by `prep` into `shared.json` over the union of the reference and every rollout, so every compute job is one pass with no range scan), `reduced.py` (`Partial`/`Reduced` — the compact self-describing JSON a compute job emits), `catalog.py` (the declarative `PlotSpec` registry — marginals × {overall,energy,pdg,material}, per-event totals, shower profiles/containment, species/leakage, secondaries, distance/confusion summaries, router and type-embedding diagnostics; `giant analyze list` prints every id), `runtime_estimate.py` (per-(plot, chunk) walltime estimates for the submit description), and `render.py` (the only module importing ETPlot's `plotstyle`/LaTeX; dispatches on `Reduced.kind`, writes PDFs + `metadata.yaml`; each rollout gets a stable `ps.get_color(i)` slot by its position in `series`, the reference always draws in one fixed dashed-ink style). `Bundle.rollouts` is a name-keyed dict of `Side`, and every `compute_partial`/`finalize` builds a `Reduced.payload["series"]` dict keyed the same way, with `payload["reference"]` as the one distinguished non-rollout entry. The heatmap-shaped specs (`marginal_distance_summary`, `n_sec_confusion`) and the checkpoint-bound diagnostics (`router_gating.py`, `type_embedding_distance.py`) are inherently one-matrix/one-checkpoint per rollout, so they render as one panel per rollout instead of one line/bar per rollout.
+
+**Input is one or more `giant rollout` YAML sidecars** (`run.py:load_rollout_yamls`): each YAML's `output`/`dataset` keys name its rollout parquet and seed file (= the reference truth); every supplied YAML must resolve to the same `dataset`, checked up front with a clear error otherwise (the premise is "N candidates vs one ground truth"). Each rollout's series name comes from a repeated `--label` CLI flag, else the YAML stem (N>1), else `"rollout"` (a single YAML). `prep` creates a **run directory** (`<cwd>/analysis_runs/analysis_<id>/` by default, `--run-dir` to override) holding `shared.json`, `run_meta.json` (`RunMeta.rollouts: list[{name,path,plot_meta}]`, insertion order = CLI order = every plot's series order), `reduced_partial/`, `reduced/`, `plots/`. **Compute/merge/render split:** `giant analyze prep a.yaml [b.yaml ...] --chunks N` records `N` in `run_meta.json`, and the workflow's `AnalysisComputeTask` runs one HTCondor job per (plot, chunk) pair (`compute-one --id --chunk --run-dir`, polars/numpy only — no LaTeX on workers), each streaming over an `event_id`-disjoint slice (`event_id % N == chunk`) of the reference **and every rollout** and writing a small `reduced_partial/<id>__<chunk>.json`; every `PlotSpec` splits into a `compute_partial`/`finalize` pair so chunks can be summed/concatenated back per rollout (`chunkable=False` specs — the checkpoint-bound diagnostics, already bounded/subsampled — always run as a single chunk). The local `giant analyze render <run_dir>` first joins every plot's chunk partials into `reduced/<id>.json` (`merge_all`, a no-op join when `N=1`; `merge-one` does a single plot for debugging), then turns those into the styled PDF/gallery tree. `giant analyze metrics <train_run_dir>` is a separate, unrelated entry point: training-progress plots straight from a run's `metrics.csv`.
+
+**Workflow orchestration** (`giant/workflow/`, `giant workflow run` CLI): b2luigi is the **only sanctioned way to run a multi-step pipeline**; `giant`/`dwarf` are single-step primitives the tasks invoke. One workflow TOML (`configs/workflow_example.toml`) parameterises a whole experiment — `[workflow]`/`[condor]`/`[dataset]`/`[geometry]` plus repeated `[[train]]`/`[[rollout]]`/`[[analysis]]` tables, each cross-referenced by name — and `spec.py` parses it into frozen dataclasses, rejecting unknown keys and dangling references. Every task's output directory is `<result_dir>/<kind>/name=<name>/spec_hash=<hash>/…`, where the 8-hex `spec_hash` covers that task's resolved sub-spec **and its transitive parents**, so an edited spec re-runs exactly the affected subtree instead of silently reusing stale outputs. The DAG (`tasks.py`): `DatasetTask` (external, fails fast if `/ceph` isn't mounted) → `WarmCacheTask` / `GeometryOracleTask` → `TrainEpochTask(name, milestone)` → `TrainTask` → `RolloutTask` → `AnalysisPrepTask` → `AnalysisComputeTask(name, plot_id, chunk)` → `AnalysisRenderTask` → `WorkflowTask`. Training is fanned out into **one short GPU job per epoch** (`epochs_per_job` trades queue waits back), chained by `--resume` on the previous job's `last.pt` — the loop already handles that unchanged — and `TrainTask` republishes `best.pt`/`last.pt`/a concatenated `metrics.csv` so nothing downstream sees the fan-out. `StreamingStepsDataset.set_epoch` and `config.epoch_seed` (both applied per epoch by `training/loop.py`) derive the batch order and the global RNG state from `(seed, epoch)`, so epoch *k* is bit-identical either way — verified by diffing a chained run's `metrics.csv` against a single 3-epoch `giant train`. `AnalysisRenderTask` is always local (the only step importing plotstyle/LaTeX); `htcondor.py` holds the CPU/GPU submit settings, with the GPU requirement strings (`TARGET.ProvidesEtpCeph` + device/memory pins) ported from the `condor-gpu-train-rollout` branch. `run.py` is the script b2luigi re-executes on workers (`--spec` forwarded via `task_cmd_additional_args`, so a worker resolves the identical graph); `giant workflow run` is a thin exec of it. Needs `uv sync --extra cpu --extra workflow`.
+
+**Shower rollout** (`giant/rollout.py`, `giant rollout` CLI): autoregressively steps the two-stage model into a full shower, advancing tracks breadth-first (every sweep steps all active tracks once, in `batch_size` chunks, so many tracks share each forward pass). Each primary post-step becomes the next pre-step, secondaries are pushed as new tracks, and per-step `material`/`layer_id` come from a `GeometryOracle` (`giant/geometry.py`, built via `dwarf build-geometry-oracle`) that learns position → (material, layer_id) from data and flags detector escape by nearest-neighbour distance. Tracks terminate on one of the `TERM_*` reasons in `constants.py` (energy cutoff, max steps, escape, natural end, unknown pdg, max tracks); energy is deposited locally on every stop except escape (leakage), so showers conserve energy by construction. `giant/checkpoint_io.py` is the shared checkpoint → ready-to-run-models path used by both `predict` and `rollout`.
 
 ## Roadmap
 
 **Phase 1 (done):** number of secondaries and their total energy were conditioning inputs; the model predicted only the 9D primary post-step (energy-conservation PoC).
 
-**Phase 2 (implemented — baseline):** the two-stage model above jointly predicts `n_sec`, the energy simplex (`e_sec` falls out of it), and each secondary's energy/direction/species, so a rollout is self-contained (no ground-truth secondary counts injected). This is the "get a baseline out" track agreed with Jan & Tobias (2026-07-07).
+**Phase 2 (done):** the two-stage model jointly predicts `n_sec`, the energy simplex (`e_sec` falls out of it), and each secondary's energy/direction/species, so a rollout is self-contained (no ground-truth secondary counts injected).
 
-**Physical-property conditioning (implemented):** `model.conditioning = "physical" | "embedding"` (see above) replaces the learned PDG/material embeddings with a small MLP over particle mass/charge and material Z_eff/A_eff/density/X0/λ_int, and Stage 2 predicts a secondary's mass/charge directly instead of a snapped species embedding. `"embedding"` stays available as the generalization-comparison baseline. `giant/materials.py`'s table is already filled with real values for every material the geometry produces. **Not yet done:** the actual held-out-material/species generalization comparison against the `"embedding"` baseline is unrun — the 34GB multi-material dataset at the repo root (6 materials, 237 PDG codes including nuclear/ion codes) is the natural dataset for that experiment.
+**Physical-property conditioning (implemented, default):** `conditioning.particle.type` / `conditioning.material.type` = `physical | embedding | onehot`. **Not yet done:** the actual held-out-material/species generalization comparison against the `"embedding"` baseline is unrun — the 34GB multi-material dataset (6 materials, 237 PDG codes including nuclear/ion codes) is the natural dataset for that experiment.
 
-**Faster-eval architectures (implemented, validation in progress):** both tracks below target a ~10× native-Geant4 eval budget and are now wired into `giant train`/`giant/model/network.py`, but neither has a validated result yet — treat both as unproven until the corresponding analysis run says otherwise:
-- **WGAN-GP** (`--mode wgan`, see Architecture above): implemented, **not yet tested** — no rollout-vs-reference analysis run against it yet.
-- **MoE routing trunk** (`--router`, see Architecture above): implemented, **first rollout benchmark done (2026-07-22), result: needs retraining with a different router config, not abandoned.** A 10-expert `EnergyRouter` run (`n_experts=10`, `temperature=0.5`, `learn_centers=true`, **`lambda_balance=0.0`**, only 20 fine-tuning epochs resumed from a non-routed checkpoint) diverged badly from Geant4 on step granularity, secondary species, and shower shape, despite roughly matching bulk total deposited energy. The `router_gating` diagnostic plot points at the likely cause: the ten experts overlap heavily across ~5 decades of pre-step energy instead of partitioning it — even the top-energy expert only reaches ~60–65% gate weight at the highest energies plotted — so eval-time top-1 (Voronoi) dispatch is choosing among near-ties rather than real specialists. Two contributors were identified: the missing load-balancing loss (`lambda_balance=0.0`), and `EnergyRouter`'s center init (`torch.linspace(-2, 2, n_experts)`) assuming a roughly uniform z-normalized energy distribution, which real energy spectra don't match. **Fixed (2026-07-27):** `EnergyRouter` now accepts an optional `centers_init` (backward compatible — omitting it keeps the old linspace), and `giant train` auto-populates it from real data quantiles via a reservoir sample collected during the existing normalizer-fitting pass in `giant/pipeline.py` (no extra file scan), for `--router-type energy` only. The routing *strategy* itself may still be sound, but the specific benchmarked config wasn't. **Next step before further evaluation: retrain with `lambda_balance > 0` and the new quantile-seeded centers (and consider more epochs / a from-scratch run rather than a short fine-tune), then re-check whether `router_gating` sharpens up.** Full writeup: `/home/lars/knowledge-base/experiments/giant-router-energy-rollout-validation.md`.
+**v0.3.0 — Stage-2 autoregressive redesign (implemented, released; on `master` since 2026-08-13):** motivated by the 2026-08-03 WGAN rollout benchmark, which failed specifically at the secondary-species level (zero photon secondaries, ~4M hallucinated `-14` muon antineutrinos). Stage 2 became autoregressive in descending-energy order with teacher forcing, and the particle-type representation went back to **categorical** (`particle_type.target = "onehot"`), reversing the 2026-07-17 continuous `(log-mass, charge)` target. The config break (`[conditioning]`/`[stage1_model]`/`[stage2_model]`/`[train]` replacing the flat `train.mode` + `[model]`) makes per-stage generators, stage-2-only training, and one-shot-vs-autoregressive comparison all expressible, and the `network.py` refactor into composable parts (encoder × trunk × objective) also makes routed WGAN work for the first time.
 
-A sampling-calorimeter (multi-material) dataset is still a planned future direction, not yet built. See the knowledge base (`/home/lars/knowledge-base/meta/roadmap.md`).
+v0.2 configs and checkpoints are auto-migrated (`config.migrate_config`, `model._legacy._migrate_legacy_model_config`, both drawing on shared facts in `giant/_migration.py`). **v0.2 checkpoint-loading support has no expiry decided yet**: `/ceph` still holds pre-v0.3.0 checkpoints and analysis runs referencing them, so don't delete or substantially alter either migration function or `tests/legacy/network_v02_snapshot.py` (the frozen v0.2 snapshot they're tested against) without an explicit decision to do so first.
 
-**v0.3.0 — Stage-2 autoregressive redesign (designed, not implemented; branch `v0.3.0-stage2-autoregressive`):** the 2026-08-03 WGAN rollout benchmark failed specifically at the secondary-species level (zero photon secondaries, ~4M hallucinated `-14` muon antineutrinos). The agreed response pivots Stage 2 to **autoregressive generation** in descending-energy order with teacher forcing, and switches the particle-type representation back to **categorical** (top N−1 by training-set count + an "other" bucket), reversing the 2026-07-17 continuous `(log-mass, charge)` target. This requires a config break: `[conditioning]` / `[stage1_model]` / `[stage2_model]` / `[train]` blocks replace the single global `train.mode` + `[model]`, so per-stage generators (`stage1 = flow` + `stage2 = wgan`), stage-2-only training, and one-shot-vs-autoregressive comparison are all expressible. `network.py` is refactored from ten permutation classes into composable parts (encoder × trunk × objective), which also makes routed WGAN work for the first time. This config break is why v0.2-shaped configs/checkpoints need migrating at all (`config.migrate_config`, `model.network._migrate_legacy_model_config`, both drawing on shared facts in `giant/_migration.py`) — v0.2 checkpoint-loading support has **no expiry decided yet**: `/ceph` still holds pre-v0.3.0 checkpoints and analysis runs referencing them, so don't delete or substantially alter either migration function or `tests/legacy/network_v02_snapshot.py` (the frozen v0.2 snapshot they're tested against) without an explicit decision to do so first.
+**Faster-eval architectures — both implemented, neither validated.** Target is a ~10× native-Geant4 eval budget; no eval-latency number exists for any configuration yet, so that budget is unverified across the board.
+- **WGAN-GP** (`--stage2-generator wgan`, now the stage-2 default): first rollout benchmark 2026-08-03 failed with secondary-species mode collapse — the failure v0.3.0 was designed to address. **No post-v0.3.0 benchmark has been run.** Writeup: `/home/lars/knowledge-base/experiments/giant-wgan-physical-rollout-validation.md`.
+- **MoE routing trunk** (`--router`): first rollout benchmark 2026-07-22 diverged badly from Geant4 on step granularity, secondary species, and shower shape, despite roughly matching bulk total deposited energy. Cause identified as a bad config, not a bad idea: `lambda_balance=0.0` (no load-balancing loss) plus `EnergyRouter`'s `torch.linspace(-2, 2, n_experts)` center init assuming a roughly uniform z-normalized energy distribution — so the ten experts overlapped across ~5 decades of energy instead of partitioning it, and eval-time top-1 dispatch chose among near-ties rather than real specialists. Both prerequisites are fixed in code (quantile-seeded `centers_init` from `pipeline.py`, `lambda_balance` exposed). **Next step: retrain with `lambda_balance > 0` and quantile-seeded centers (consider a from-scratch run rather than a short fine-tune), then re-check whether `router_gating` sharpens up.** Writeup: `/home/lars/knowledge-base/experiments/giant-router-energy-rollout-validation.md`.
 
-**Condor-submitted GPU training/rollout (in progress, `condor-gpu-train-rollout` branch, not yet merged):** moves `giant train`/`giant rollout` off the shared portal GPU dev machines (see Compute environment) onto remote-GPU HTCondor submission on TOpAS/NEMO2 (`giant/condor.py`). Partway between "needs major features" and feature-complete — not ready to merge yet.
+A sampling-calorimeter (multi-material) dataset track is still open and unblocked, not yet started. See the knowledge base (`/home/lars/knowledge-base/meta/roadmap.md`).
+
+**Condor-submitted GPU training/rollout (`condor-gpu-train-rollout` branch, superseded):** its goal — moving `giant train`/`giant rollout` off the shared portal GPU dev machines onto remote-GPU HTCondor submission — is now met by the b2luigi workflow above. Its `train-submit`/`rollout-submit` commands are deliberately **not** ported and must not be revived when that branch is eventually merged; the only part that survived is `_gpu_requirements`, which moved into `giant/workflow/htcondor.py`.

README.md

@@ -10,6 +10,7 @@ A conditional generative model that replaces the Geant4 step function: given a p
 uv sync --extra cpu               # install deps (CPU torch; use --extra cuda for GPU)
 
 giant new-run --hidden-dim 512 --lr 3e-4          # scaffold config.toml + run dir
+giant model summary --config config.toml          # parameter counts + which config keys actually bite
 giant train path/to/steps.parquet                 # train (flow + wgan by default)
 giant predict path/to/steps.parquet --checkpoint checkpoints/.../best.pt
 
@@ -42,9 +43,9 @@ A **two-stage model**, checkpointed together. Either stage's outcome can be prod
 
 Either way, secondary energies stick-break the `e_sec` budget handed down from Stage 1, so the full chain conserves energy. A secondary's particle identity is represented as `onehot` (categorical, top-N PDG codes + "other"), `physical` (continuous log-mass/charge), or `embedding` (nearest-neighbour lookup).
 
-**Conditioning.** Pre-step position/energy/direction/layer, plus particle mass/charge and material Z_eff/A_eff/density/X0/λ_int, encoded the same three ways as particle identity above (`--conditioning`) — the `physical` representation generalizes to species/materials outside the training menu since it's computed rather than looked up. `n_sec`/`e_sec` are always model outputs, never conditioning inputs.
+**Conditioning.** Pre-step position/energy/direction/layer, plus particle mass/charge and material Z_eff/A_eff/density/X0/λ_int, encoded the same three ways as particle identity above. The particle and material axes are configured independently (`conditioning.particle.type` / `conditioning.material.type`; `--conditioning` sets both at once) and may mix — the `physical` representation generalizes to species/materials outside the training menu since it's computed rather than looked up. `n_sec`/`e_sec` are always model outputs, never conditioning inputs.
 
-**MoE routing** (`--router`, either stage): a pluggable `Router` (`energy`/`pdg`/`process`/`composed` axes) top-1-dispatches each row to one of several small expert trunks at eval time, instead of running one monolithic trunk.
+**MoE routing** (`--router`): a pluggable `Router` (`energy`/`pdg`/`process`/`composed` axes) top-1-dispatches each row to one of several small expert trunks at eval time, instead of running one monolithic trunk. The CLI flags configure Stage 1's router; Stage 2 has its own `stage2_model.router` block, config-file only.
 
 ## Data
 
@@ -64,12 +65,24 @@ giant/
 │   ├── data/
 │   │   ├── loader.py       # parquet → numpy arrays (incl. streaming/chunked reads)
 │   │   ├── transforms.py   # log transforms, local-frame rotation, energy simplex, secondary encode/decode
-│   │   └── dataset.py      # StepsDataset / StreamingStepsDataset (PyTorch)
+│   │   ├── dataset.py      # StepsDataset / StreamingStepsDataset (PyTorch)
+│   │   └── setup_cache.py  # sidecar cache for the pre-epoch setup scan (vocab/split/normalizers)
 │   ├── model/
-│   │   ├── network.py      # ConditionEncoder, Stage1Model, Stage2OneShot/Stage2Autoregressive, Router/MoE, CriticModel
+│   │   ├── models.py       # Stage1Model, Stage2OneShot, Stage2Autoregressive, CriticModel
+│   │   ├── builders.py     # build_models / build_critics — config dict → assembled stage models
+│   │   ├── encoders.py     # ConditionEncoder (physical / embedding / onehot, per axis)
+│   │   ├── layers.py       # ResBlock/AdaLNResBlock registry, SinusoidalEmbedding, MLP heads
+│   │   ├── trunks.py       # trunk registry (resmlp, none) + RoutedTrunk (MoE expert bodies)
+│   │   ├── routers.py      # Router registry: energy / pdg / process / composed / none
+│   │   ├── history.py      # stage-2 AR history encoders: markov / attention (KV-cached) / none
+│   │   ├── objectives.py   # flow / ddpm / wgan objective registry
 │   │   ├── schedule.py     # CosineSchedule (DDPM) and flow matching utilities
-│   │   └── wgan.py         # WGAN-GP gradient penalty / critic / generator losses
+│   │   ├── wgan.py         # WGAN-GP gradient penalty / critic / generator losses
+│   │   ├── summary.py      # build-only introspection behind `giant model summary`
+│   │   ├── _legacy.py      # v0.2 checkpoint model_config/state-dict migration
+│   │   └── network.py      # re-export shim over all of the above
 │   ├── constants.py        # output/conditioning dims, K_MAX, secondary slot layout, schema keys
+│   ├── cond_layout.py      # single source of truth for the cond_cont/cond_cat column layout
 │   ├── particles.py        # PDG → (mass, charge) decode, incl. nuclear/ion codes; onehot/embedding secondary-identity decode
 │   ├── materials.py        # material name → (Z_eff, A_eff, density, X0, λ_int)
 │   ├── config.py           # default hyperparameters, TOML config merging, device autodetect
@@ -79,20 +92,33 @@ giant/
 │   │   ├── trainers.py     #   StageSpec + flow/ddpm and WGAN-GP per-stage trainers
 │   │   ├── stage2_inputs.py#   ground-truth stage-2 targets + autoregressive/teacher-forcing inputs
 │   │   ├── metrics.py      #   MetricsCollector: metrics.csv columns, W&B logging, progress/summary
+│   │   ├── amp.py          #   bf16 autocast (`train.precision`)
+│   │   ├── plots.py        #   training-progress plots (`giant analyze metrics`)
 │   │   └── checkpoint.py   #   checkpoint assembly/restore (format unchanged since v0.2)
 │   ├── sample.py           # DDPM / DDIM / flow matching / WGAN samplers + secondary sampling
+│   ├── checkpoint_io.py    # checkpoint → ready-to-run models/normalizers (predict + rollout)
 │   ├── geometry.py         # GeometryOracle: position → (material, layer_id, escaped) for rollout
 │   ├── rollout.py          # autoregressive shower rollout driver
 │   ├── validate.py         # step-level marginal + KL-divergence validation
+│   ├── _migration.py       # shared v0.2 → v0.3 facts used by both migration surfaces
 │   ├── analysis/           # rollout-vs-reference analysis pipeline (see `giant analyze` below)
 │   │   ├── sources.py      #   canonical LazyFrames + secondary view
+│   │   ├── variables.py    #   per-step value expressions shared by range sizing and the catalog
 │   │   ├── reduce.py       #   streaming reduction primitives (hist1d, per-event scalars, profiles, ...)
 │   │   ├── grouping.py     #   fixed bin edges + energy/pdg/material group sets
 │   │   ├── context.py      #   resolves grouping into `shared.json` once per run
-│   │   ├── catalog.py      #   declarative PlotSpec registry
-│   │   ├── condor.py       #   prep / compute-one / submit-description plumbing
+│   │   ├── reduced.py      #   Partial/Reduced — the compact JSON a compute job emits
+│   │   ├── catalog.py      #   declarative PlotSpec registry (`giant analyze list`)
+│   │   ├── router_gating.py / type_embedding_distance.py   # checkpoint-bound diagnostics
+│   │   ├── runtime_estimate.py  #   per-(plot, chunk) walltime estimates for the job requests
+│   │   ├── run.py          #   prep / compute-one / merge plumbing
 │   │   └── render.py       #   PDFs + HTML gallery (only module importing plotstyle/LaTeX)
-│   └── cli.py              # `giant train` / `new-run` / `predict` / `rollout` / `analyze` Typer app
+│   ├── workflow/           # b2luigi pipeline orchestration (`giant workflow run spec.toml`)
+│   │   ├── spec.py         #   workflow TOML -> frozen dataclasses, validation, per-task spec hashes
+│   │   ├── htcondor.py     #   CPU/GPU submit settings (docker image, +RemoteJob, GPU requirements)
+│   │   ├── tasks.py        #   the task graph: cache-warm -> train (one job/epoch) -> rollout -> analysis
+│   │   └── run.py          #   the script b2luigi re-executes on every worker
+│   └── cli.py              # `giant train` / `new-run` / `model summary` / `predict` / `rollout` / `analyze` / `workflow`
 ├── giant/tools/                        # dataset/tooling logic, unified under the `dwarf` CLI (`dwarf --help`)
 │   ├── dwarf.py                        # Typer app: convert, migrate, bump-gen, bump-schema, status,
 │   │                                    #   update-manifest, create-manifest, make-root,
@@ -116,8 +142,14 @@ giant/
 uv sync --extra cpu                   # CPU-only torch (use --extra cuda for CUDA 11.8 instead)
 uv sync --extra cpu --extra dev       # add dev tools (pytest, ruff, ty)
 uv sync --extra cpu --extra geometry  # add scikit-learn, for `dwarf build-geometry-oracle` / rollout
+uv sync --extra cpu --extra analysis  # matplotlib/polars/plotstyle, for `giant analyze render`
+uv sync --extra cpu --extra convert   # uproot/awkward/polars, for `dwarf convert`
+uv sync --extra cpu --extra wandb     # W&B logging (`giant train --wandb`)
+uv sync --extra cpu --extra workflow  # b2luigi, for `giant workflow run`
 ```
 
+The `dev` extra pulls in `convert`, `analysis`, `geometry`, `wandb` and `workflow` as well.
+
 `cpu` and `cuda` are mutually exclusive — pick one to select the torch build (pinned to 2.3.x). Plain `uv sync` installs no torch at all. See `CLAUDE.md` for details.
 
 ## Training, prediction, rollout
@@ -137,11 +169,13 @@ Useful flags on `giant train`:
 - `--stage2-decoder {autoregressive,one_shot}` — Stage 2 decoding strategy (see Architecture)
 - `--conditioning {physical,embedding,onehot}` — conditioning representation
 - `--router` / `--router-type` / `--n-experts` / `--router-axis` — MoE routing
+- `--stage2-stage1-context {truth,sampled}` — feed Stage 2 the ground-truth or the model's own sampled Stage-1 outcome (annealable via `stage2_model.ctx_p_start`/`ctx_p_end`)
+- `--precision {fp32,bf16}` — bf16 autocast in the training loop
 - `--wandb` — log per-epoch metrics to Weights & Biases (needs `uv sync --extra wandb`); metric names are `<stage>/<split>/<metric>` plus an unprefixed run-level tail, all derived from `giant/training/trainers.py` `MetricSpec`s
 - `--no-cache-setup` / `--rebuild-setup-cache` — control the setup-stage sidecar cache (vocab maps, event split, normalizer stats); `dwarf warm-cache` precomputes it
 - `--stage1-init-from`/`--stage2-init-from` (checkpoint `.pt`) + `--stage1-freeze`/`--stage2-freeze` — load a stage's weights from another checkpoint and never update them, so the other stage can be retrained alone against a fixed, known-good one while still producing a complete, rollout-capable checkpoint
 
-Config-file-only knobs (no CLI flag — use `--config config.toml`): `stage2_model.autoregressive.teacher_forcing`/`.history`, `stage2_model.particle_type.target`. v0.2 flat-schema configs and checkpoints load fine (auto-migrated).
+Config-file-only knobs (no CLI flag — use `--config config.toml`): `stage2_model.autoregressive.teacher_forcing`/`.history`, `stage2_model.particle_type.target`/`.class_weighting`, `stage2_model.n_sec.mode`/`.owner`, `conditioning.share_stages`, `stage*_model.trunk.*` and the finer `router` knobs (`lambda_balance`, `gumbel`, `learn_width`, …). `configs/` holds kept reference configs. v0.2 flat-schema configs and checkpoints load fine (auto-migrated).
 
 `giant rollout` seeds showers from each event's highest-energy entry step, then autoregressively steps the model to completion, pushing secondaries as new tracks and looking up `material`/`layer_id` from the geometry oracle each step. Tracks terminate on energy cutoff, max steps, detector escape, or natural end; energy is deposited locally on every stop except escape, so showers conserve energy by construction.
 
@@ -151,11 +185,31 @@ Config-file-only knobs (no CLI flag — use `--config config.toml`): `stage2_mod
 - `giant analyze` — deeper rollout-vs-reference diagnostics (marginals by energy/pdg/material, per-event totals, shower profiles, species share, leakage, secondaries):
 
 ```bash
-giant analyze submit rollout.yaml --accounting-group cms   # prep + one HTCondor job per plot (compute only)
-giant analyze render <run_dir> --gallery                   # local: styled PDFs + HTML gallery (needs LaTeX)
+giant analyze prep rollout.yaml --chunks 8                 # lay out the run directory
+giant analyze prep a.yaml b.yaml --label flow --label wgan  # N rollouts vs one shared reference
+giant analyze render <run_dir> --gallery                   # local: merge chunks, then styled PDFs + HTML gallery (needs LaTeX)
+
+giant analyze list                                         # every catalog plot id
+giant analyze compute-one --id marginal_edep --run-dir <run_dir> --chunk 0   # what a condor job runs
+giant analyze merge-one --id marginal_edep --run-dir <run_dir>               # merge one plot's chunks (debugging)
 ```
 
-`<run_dir>` is derived next to the rollout parquet (`analyze prep`/`submit` print it). Compute jobs are polars/numpy only; only `render` needs LaTeX, so it always runs locally.
+The per-(plot, chunk) compute jobs themselves are submitted by the workflow (below), not by `giant analyze` — these commands are the single-step primitives it calls. `<run_dir>` defaults to `<cwd>/analysis_runs/analysis_<id>` (`--run-dir` overrides it; `prep` prints it). Multiple rollout YAMLs must all name the same reference (`dataset`) file; each renders as its own colored series against one reference line/panel. Compute jobs are polars/numpy only; only `render` needs LaTeX, so it always runs locally.
+
+## Workflow orchestration
+
+Multi-step pipelines run through [b2luigi](https://github.com/belle2/b2luigi) — one spec file describes a whole experiment, and every step's outputs are files on `/ceph` that are only recomputed when their spec (or an upstream one) changes:
+
+```bash
+uv sync --extra cpu --extra workflow
+giant workflow run configs/workflow_example.toml --mode dry-run   # what would run
+giant workflow run configs/workflow_example.toml --mode show-output  # where every output goes
+giant workflow run configs/workflow_example.toml --batch --workers 20  # submit to HTCondor and wait
+```
+
+The spec holds `[workflow]`/`[condor]`/`[dataset]`/`[geometry]` plus repeated `[[train]]`, `[[rollout]]` and `[[analysis]]` tables cross-referenced by name (see `configs/workflow_example.toml`). The task graph is `DatasetTask → WarmCacheTask/GeometryOracleTask → TrainEpochTask… → TrainTask → RolloutTask → AnalysisPrepTask → AnalysisComputeTask(plot, chunk) → AnalysisRenderTask`. Training is split into one short GPU job per epoch (chained by `--resume`), which schedules better on a busy farm and survives preemption; `TrainTask` then publishes one `best.pt`/`last.pt`/`metrics.csv` for everything downstream. Rendering always runs locally, since it is the only step that needs LaTeX.
+
+Separately, `giant analyze metrics <train_run_dir>` renders training-progress plots (loss/lr/accuracy/grad-norm/router/wgan/throughput) straight from a training run's `metrics.csv`.
 
 ## Development
 

configs/workflow_example.toml

@@ -0,0 +1,68 @@
+# Example GIANT workflow spec — `giant workflow run configs/workflow_example.toml`.
+#
+# One file parameterises a whole experiment: the datasets, the geometry oracle,
+# N trainings, N rollouts, and the analyses comparing them. Every task's output
+# directory carries a hash of its resolved sub-spec (plus its parents), so
+# editing anything here re-runs exactly the affected subtree and nothing else.
+#
+# result_dir/log_dir must be visible from both the submit host and the workers
+# (i.e. on /ceph) — there is deliberately no HTCondor file transfer.
+
+[workflow]
+name       = "baseline-vs-router"
+result_dir = "/ceph/lbogner/workflows/baseline-vs-router"
+log_dir    = "/ceph/lbogner/workflows/baseline-vs-router/logs"
+
+[condor]
+accounting_group = "cms"
+repo_dir         = "/work/lbogner/giant"   # also b2luigi's working_dir
+env_script       = "/work/lbogner/giant/condor_env.sh"
+docker_image_cpu = "cverstege/alma9-gridjob"
+docker_image_gpu = "mschnepf/slc7-condocker"
+remote           = true
+
+[dataset]
+steps     = "/ceph/lbogner/geant_steps/train/"     # training data
+reference = "/ceph/lbogner/geant_steps/holdout/"   # rollout seeds + analysis truth
+
+[geometry]
+method    = "slab"
+subsample = 500_000
+
+[[train]]
+name           = "baseline"
+config         = "configs/baseline.toml"
+epochs         = 200
+epochs_per_job = 1            # one short GPU job per epoch, chained
+request_gpus   = 1
+gpu_memory_mb  = 20000
+overrides      = { lr = 3e-4 }   # `giant train` flag names
+
+[[train]]
+name           = "router-balanced"
+config         = "configs/router.toml"
+epochs         = 200
+epochs_per_job = 1
+request_gpus   = 1
+gpu_memory_mb  = 20000
+
+[[rollout]]
+name          = "baseline"
+train         = "baseline"     # -> [[train]].name
+n_events      = 2000
+energy_cutoff = 0.1
+
+[[rollout]]
+name          = "router-balanced"
+train         = "router-balanced"
+n_events      = 2000
+energy_cutoff = 0.1
+
+[[analysis]]
+name        = "baseline-vs-router"
+rollouts    = ["baseline", "router-balanced"]
+chunks      = 32
+energy_bins = 4
+bins        = 50
+top_pdg     = 6
+gallery     = true

giant/analysis/__init__.py

@@ -1,9 +1,10 @@
 """Rollout-vs-reference analysis: streaming compute + plotstyle rendering.
 
-Compares one autoregressive ``giant rollout`` against a held-out miniCaloSim
-reference file, producing publication-styled comparison plots generated in
-parallel on HTCondor (one job per plot x data chunk, compute/merge/render
-split).
+Compares one or more autoregressive ``giant rollout`` runs against a single
+held-out miniCaloSim reference file shared by all of them, producing
+publication-styled comparison plots (one colored series per rollout, one
+reference line) generated in parallel on HTCondor (one job per plot x data
+chunk, compute/merge/render split) — orchestrated by ``giant/workflow``.
 
 Only ``render`` (and the ``render`` CLI path) imports plotstyle/LaTeX; everything
 re-exported here is plotstyle-free so it runs on a compute worker. Import
@@ -11,41 +12,42 @@ re-exported here is plotstyle-free so it runs on a compute worker. Import
 """
 
 from giant.analysis.catalog import build_catalog, catalog_ids, get_spec
-from giant.analysis.condor import (
+from giant.analysis.run import (
+    LoadedRollout,
     RunMeta,
-    SubmitConfig,
     compute_one,
     compute_reduced,
     derive_run_dir,
     load_rollout_yaml,
+    load_rollout_yamls,
     merge_all,
     merge_one,
     prep,
-    write_submit,
 )
 from giant.analysis.context import Context, build_context
 from giant.analysis.reduced import Partial, Reduced
 from giant.analysis.runtime_estimate import RUNTIME_SAFETY_MARGIN, estimate_runtime_s
-from giant.analysis.sources import Side
+from giant.analysis.sources import RolloutSpec, Side
 
 __all__ = [
     "build_catalog",
     "catalog_ids",
     "get_spec",
+    "LoadedRollout",
     "RunMeta",
-    "SubmitConfig",
     "compute_one",
     "compute_reduced",
     "derive_run_dir",
     "load_rollout_yaml",
+    "load_rollout_yamls",
     "merge_all",
     "merge_one",
     "prep",
-    "write_submit",
     "Context",
     "build_context",
     "Partial",
     "Reduced",
+    "RolloutSpec",
     "Side",
     "RUNTIME_SAFETY_MARGIN",
     "estimate_runtime_s",

giant/analysis/catalog.py

@@ -4,7 +4,7 @@ Each spec knows its stable ``id`` (used for the reduced-data filename, the PDF
 stem and the condor queue item), its gallery ``family`` (subdirectory), and a
 ``compute_partial(bundle) -> dict`` / ``finalize(parts, ctx) -> Reduced`` pair
 that together run the streaming reduction. ``compute_partial`` runs once per
-``(plot, chunk)`` condor job against a ``Bundle`` whose four LazyFrames are
+``(plot, chunk)`` condor job against a ``Bundle`` whose LazyFrames are
 already filtered to that chunk (see ``Bundle.open``'s ``chunk`` argument); it
 returns a small JSON-safe partial artifact — either a raw sum-mergeable count
 dict (histograms/species sums against fixed edges) or a raw per-event/
@@ -16,6 +16,19 @@ exactly what a single unchunked pass would produce. Specs marked
 ``chunkable=False`` (the router ones) always run as a single chunk regardless
 of the configured chunk count.
 
+Every ``compute_partial`` here returns ``{"r": {rollout_name: <shape>}, "t":
+<shape>}`` — one entry per rollout in ``Bundle.rollouts`` (insertion order,
+which is the order rollouts were given on the CLI) plus the single reference.
+``finalize`` merges each rollout's chunks independently and assembles a
+``Reduced.payload`` keyed the same way: ``"series": {name: ...}`` for the
+rollouts, ``"reference": ...`` as one distinguished entry (omitted on
+rollout-only plots like ``leakage_fraction``). The two heatmap-shaped specs
+(``marginal_distance_summary``, ``n_sec_confusion``) and the router
+diagnostics are inherently one-matrix/one-checkpoint per rollout, so their
+``"series"`` entries are whole per-rollout artifacts (a matrix, a gating
+dict) rather than a single number/array — ``render.py`` draws those as one
+panel per rollout instead of one line/bar per rollout.
+
 Rendering lives in ``render.py`` and dispatches on ``Reduced.kind`` — the
 catalog itself never imports plotstyle, so ``compute-one`` jobs stay LaTeX-free.
 
@@ -47,6 +60,7 @@ from giant.analysis.reduce import (
     leakage_fraction,
     profile_finalize,
     profile_partial,
+    sec_count_by_event,
     species_share,
     sum_merge,
     transverse_expr,
@@ -56,8 +70,9 @@ from giant.analysis.router_gating import (
     compute_router_gating,
     compute_router_share_by_pdg,
     compute_router_share_by_process,
+    compute_router_specialization,
 )
-from giant.analysis.sources import Side, open_side, physical_steps, secondaries
+from giant.analysis.sources import RolloutSide, RolloutSpec, Side, open_side, physical_steps, secondaries
 from giant.analysis.type_embedding_distance import compute_type_embedding_l1_distance
 from giant.analysis.variables import RANGED_VARS, cos_scatter_expr
 
@@ -67,51 +82,44 @@ class Bundle:
     """Everything a compute runs against — built once per ``compute-one`` job."""
 
     ctx: Context
-    r_all: pl.LazyFrame  # rollout, all rows (incl. synthetic termination rows)
+    rollouts: dict[str, RolloutSide]  # name -> frames, insertion order = CLI order
     t_all: pl.LazyFrame  # reference, all rows
-    r_phys: pl.LazyFrame  # rollout, physical steps only
     t_phys: pl.LazyFrame  # reference, physical steps only
-    checkpoint: str | None = None  # from the rollout YAML; router_gating only
-    # Diagnostic pre-aggregated at rollout time (giant.rollout.
-    # L1DistCollector.summary()) — from the rollout YAML, type_embedding_l1_distance
-    # only. Unlike checkpoint/router_gating, this needs no live model: it's
-    # already a finished histogram, just passed through.
-    type_embedding_l1_dist: dict | None = None
 
     @classmethod
     def open(
         cls,
-        rollout,
+        rollouts: list[RolloutSpec],
         reference,
         ctx: Context,
-        checkpoint=None,
         chunk: tuple[int, int] | None = None,
-        type_embedding_l1_dist: dict | None = None,
     ) -> "Bundle":
-        """Open both sides, optionally restricted to one event-disjoint chunk.
+        """Open the reference + every rollout, optionally restricted to one event-disjoint chunk.
 
-        ``chunk = (chunk_index, n_chunks)`` filters both sides to
+        ``chunk = (chunk_index, n_chunks)`` filters every side to
         ``event_id % n_chunks == chunk_index`` *before* deriving the physical/
         secondary views, so every downstream reduction (which is either
         row-local or a ``group_by("event_id")``) sees a self-contained,
         event-disjoint slice — no cross-chunk lookups are ever needed.
         """
-        r_all = open_side(rollout, Side.rollout)
         t_all = open_side(reference, Side.reference)
+        pred = None
         if chunk is not None:
             idx, n = chunk
             pred = pl.col("event_id") % n == idx
-            r_all = r_all.filter(pred)
             t_all = t_all.filter(pred)
-        return cls(
-            ctx=ctx,
-            r_all=r_all,
-            t_all=t_all,
-            r_phys=physical_steps(r_all, Side.rollout),
-            t_phys=physical_steps(t_all, Side.reference),
-            checkpoint=checkpoint,
-            type_embedding_l1_dist=type_embedding_l1_dist,
-        )
+        sides: dict[str, RolloutSide] = {}
+        for rs in rollouts:
+            r_all = open_side(rs.source, Side.rollout)
+            if pred is not None:
+                r_all = r_all.filter(pred)
+            sides[rs.name] = RolloutSide(
+                all=r_all,
+                phys=physical_steps(r_all, Side.rollout),
+                checkpoint=rs.checkpoint,
+                type_embedding_l1_dist=rs.type_embedding_l1_dist,
+            )
+        return cls(ctx=ctx, rollouts=sides, t_all=t_all, t_phys=physical_steps(t_all, Side.reference))
 
 
 @dataclass
@@ -144,8 +152,10 @@ def _unchunkable(
 # small numpy/hist helpers
 # ---------------------------------------------------------------------------
 
-_ROLL = "rollout"
-_REF = "reference"
+
+def _per_rollout(b: Bundle, fn: Callable[[RolloutSide], object]) -> dict[str, object]:
+    """``{name: fn(rollout_side)}`` over every rollout, preserving CLI order."""
+    return {name: fn(rs) for name, rs in b.rollouts.items()}
 
 
 def _counts(h: dict, key, nbins: int) -> list[int]:
@@ -166,14 +176,89 @@ def _finalize_counts(merged: dict[str, list], key, nbins: int) -> list[int]:
     return list(merged.get(str(key), [0] * nbins))
 
 
-def _np_hist_pair(r: np.ndarray, t: np.ndarray, nbins: int) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
-    """Shared-edge histogram of two small per-event arrays (robust range)."""
-    both = np.concatenate([r, t]) if (len(r) or len(t)) else np.array([0.0, 1.0])
+def _np_hist_shared_edges(arrays: list[np.ndarray], nbins: int) -> tuple[np.ndarray, list[np.ndarray]]:
+    """Shared-edge histogram of several small per-event arrays (robust range).
+
+    The edges are sized from the union of every array (reference + all
+    rollouts), so every series in the resulting overlay is directly
+    comparable on one axis.
+    """
+    non_empty = [a for a in arrays if len(a)]
+    both = np.concatenate(non_empty) if non_empty else np.array([0.0, 1.0])
     lo, hi = float(np.quantile(both, 0.001)), float(np.quantile(both, 0.999))
     if not (hi - lo > 1e-6 * max(abs(hi), 1.0)):
         lo, hi = lo - 0.5, hi + 0.5
     edges = np.linspace(lo, hi, nbins + 1)
-    return edges, np.histogram(r, edges)[0], np.histogram(t, edges)[0]
+    return edges, [np.histogram(a, edges)[0] for a in arrays]
+
+
+def _ks_statistic(r_counts, t_counts) -> float:
+    """KS statistic (max |CDF diff|) between two same-edge binned histograms.
+
+    ``nan`` when neither side has any mass (nothing to compare); 1.0 (maximal
+    mismatch) when exactly one side is entirely empty and the other isn't —
+    correctly the worst score rather than an undefined one.
+    """
+    r_counts = np.asarray(r_counts, dtype=np.float64)
+    t_counts = np.asarray(t_counts, dtype=np.float64)
+    r_tot, t_tot = r_counts.sum(), t_counts.sum()
+    if r_tot == 0 and t_tot == 0:
+        return float("nan")
+    if r_tot == 0 or t_tot == 0:
+        return 1.0
+    r_cdf = np.cumsum(r_counts) / r_tot
+    t_cdf = np.cumsum(t_counts) / t_tot
+    return float(np.max(np.abs(r_cdf - t_cdf)))
+
+
+def _integer_confusion(
+    t: np.ndarray, r: np.ndarray, max_bins: int = 21, cap: int | None = None
+) -> tuple[list[str], np.ndarray]:
+    """Confusion matrix of two paired small-integer arrays (e.g. secondary counts).
+
+    Bins are consecutive integers ``0..cap``, with the last bin an overflow
+    ``"cap+"`` bucket, so an occasional pathological count doesn't blow up the
+    heatmap. Returns ``(labels, matrix)`` with ``matrix[i, j]`` counting pairs
+    with ``t == i`` and ``r == j`` (both clipped into ``[0, cap]``).
+
+    ``cap``, if given, is used as-is instead of being derived from ``t``/``r``
+    — lets a multi-rollout caller fix one shared cap (and so one shared label
+    set) across every rollout's matrix rather than each panel picking its own.
+    """
+    if cap is None:
+        cap = min(max(int(t.max()) if len(t) else 0, int(r.max()) if len(r) else 0, 1), max_bins - 1)
+    t_c = np.clip(t.astype(np.int64), 0, cap)
+    r_c = np.clip(r.astype(np.int64), 0, cap)
+    n = cap + 1
+    mat = np.zeros((n, n), dtype=np.int64)
+    np.add.at(mat, (t_c, r_c), 1)
+    labels = [str(i) for i in range(cap)] + [f"{cap}+"]
+    return labels, mat
+
+
+def _containment_depths(mat: np.ndarray, edges: np.ndarray, quantile: float) -> np.ndarray:
+    """Per-event depth containing ``quantile`` of that event's deposited energy.
+
+    ``mat`` is a ``(n_events, n_bins)`` edep-per-depth-bin sum matrix (see
+    ``reduce.profile_partial``); bins are ordered by increasing depth (matching
+    ``edges``, monotonic). Zero-energy events are dropped — containment depth is
+    undefined for them.
+    """
+    totals = mat.sum(axis=1)
+    valid = totals > 0
+    mat, totals = mat[valid], totals[valid]
+    cum = np.cumsum(mat, axis=1) / totals[:, None]
+    idx = (cum >= quantile).argmax(axis=1)  # first bin whose cumulative fraction reaches quantile
+    return edges[1:][idx]
+
+
+def _group_keys(ctx: Context, axis: str) -> list:
+    """The group keys ``_marginal_grouped_finalize`` iterates for ``axis``."""
+    if axis == "pdg":
+        return list(ctx.top_pdgs)
+    if axis == "material":
+        return list(ctx.materials)
+    return list(range(len(ctx.energy_edges) - 1))  # energy
 
 
 # Human-readable figure titles per marginal variable (the axis labels carry units;
@@ -210,7 +295,7 @@ def _marginal_overall_partial(b: Bundle, var: str) -> dict:
     _, expr = _var(var)
     edges = _marginal_edges(b.ctx, var)
     return {
-        "r": _partial_hist(b.r_phys, expr, edges),
+        "r": _per_rollout(b, lambda rs: _partial_hist(rs.phys, expr, edges)),
         "t": _partial_hist(b.t_phys, expr, edges),
     }
 
@@ -219,7 +304,8 @@ def _marginal_overall_finalize(parts: list[dict], ctx: Context, var: str) -> Red
     label, _ = _var(var)
     edges = _marginal_edges(ctx, var)
     nb = len(edges) - 1
-    r = sum_merge([p["r"] for p in parts])
+    names = list(parts[0]["r"])
+    series = {name: _finalize_counts(sum_merge([p["r"][name] for p in parts]), 0, nb) for name in names}
     t = sum_merge([p["t"] for p in parts])
     return Reduced(
         id=f"marginal_{var}",
@@ -229,8 +315,8 @@ def _marginal_overall_finalize(parts: list[dict], ctx: Context, var: str) -> Red
         xlabel=label,
         payload={
             "edges": edges.tolist(),
-            _ROLL: _finalize_counts(r, 0, nb),
-            _REF: _finalize_counts(t, 0, nb),
+            "series": series,
+            "reference": _finalize_counts(t, 0, nb),
             "log_y": True,
         },
     )
@@ -241,23 +327,24 @@ def _energy_group_expr(lf: pl.LazyFrame, edges: np.ndarray) -> pl.Expr:
     return pl.col("event_id").replace_strict(ids, bins, default=-1, return_dtype=pl.Int64)
 
 
+def _grouped_hist_dict(lf: pl.LazyFrame, expr: pl.Expr, edges: np.ndarray, axis: str, ctx: Context, nb: int) -> dict:
+    if axis == "pdg":
+        h = hist1d(lf, expr, edges, group=pl.col("pdg"))
+    elif axis == "material":
+        h = hist1d(lf, expr, edges, group=pl.col("material"))
+    else:  # energy
+        e_edges = np.asarray(ctx.energy_edges)
+        h = hist1d(lf, expr, edges, group=_energy_group_expr(lf, e_edges))
+    return {str(k): _counts(h, k, nb) for k in h}
+
+
 def _marginal_grouped_partial(b: Bundle, var: str, axis: str) -> dict:
     _, expr = _var(var)
     edges = _marginal_edges(b.ctx, var)
-    if axis == "pdg":
-        r = hist1d(b.r_phys, expr, edges, group=pl.col("pdg"))
-        t = hist1d(b.t_phys, expr, edges, group=pl.col("pdg"))
-    elif axis == "material":
-        r = hist1d(b.r_phys, expr, edges, group=pl.col("material"))
-        t = hist1d(b.t_phys, expr, edges, group=pl.col("material"))
-    else:  # energy
-        e_edges = np.asarray(b.ctx.energy_edges)
-        r = hist1d(b.r_phys, expr, edges, group=_energy_group_expr(b.r_phys, e_edges))
-        t = hist1d(b.t_phys, expr, edges, group=_energy_group_expr(b.t_phys, e_edges))
     nb = len(edges) - 1
     return {
-        "r": {str(k): _counts(r, k, nb) for k in r},
-        "t": {str(k): _counts(t, k, nb) for k in t},
+        "r": _per_rollout(b, lambda rs: _grouped_hist_dict(rs.phys, expr, edges, axis, b.ctx, nb)),
+        "t": _grouped_hist_dict(b.t_phys, expr, edges, axis, b.ctx, nb),
     }
 
 
@@ -265,29 +352,24 @@ def _marginal_grouped_finalize(parts: list[dict], ctx: Context, var: str, axis:
     label, _ = _var(var)
     edges = _marginal_edges(ctx, var)
     nb = len(edges) - 1
-    r = sum_merge([p["r"] for p in parts])
-    t = sum_merge([p["t"] for p in parts])
-    groups: dict[str, dict] = {}
+    names = list(parts[0]["r"])
+    r_merged = {name: sum_merge([p["r"][name] for p in parts]) for name in names}
+    t_merged = sum_merge([p["t"] for p in parts])
 
     if axis == "pdg":
-        for k in ctx.top_pdgs:
-            groups[pdg_label(k)] = {
-                _ROLL: _finalize_counts(r, k, nb),
-                _REF: _finalize_counts(t, k, nb),
-            }
+        keys, labels = ctx.top_pdgs, [pdg_label(k) for k in ctx.top_pdgs]
     elif axis == "material":
-        for m in ctx.materials:
-            groups[material_label(m)] = {
-                _ROLL: _finalize_counts(r, m, nb),
-                _REF: _finalize_counts(t, m, nb),
-            }
+        keys, labels = ctx.materials, [material_label(m) for m in ctx.materials]
     else:  # energy
         e_edges = np.asarray(ctx.energy_edges)
-        for bi, lbl in enumerate(energy_bin_labels(e_edges)):
-            groups[lbl] = {
-                _ROLL: _finalize_counts(r, bi, nb),
-                _REF: _finalize_counts(t, bi, nb),
-            }
+        keys, labels = list(range(len(e_edges) - 1)), energy_bin_labels(e_edges)
+
+    groups: dict[str, dict] = {}
+    for k, lbl in zip(keys, labels):
+        groups[lbl] = {
+            "series": {name: _finalize_counts(r_merged[name], k, nb) for name in names},
+            "reference": _finalize_counts(t_merged, k, nb),
+        }
 
     return Reduced(
         id=f"marginal_{var}_by_{axis}",
@@ -299,22 +381,96 @@ def _marginal_grouped_finalize(parts: list[dict], ctx: Context, var: str, axis:
     )
 
 
+# ---------------------------------------------------------------------------
+# distance summary: a var x group-axis scorecard per rollout, reusing the marginal hists
+# ---------------------------------------------------------------------------
+
+
+def _distance_summary_partial(b: Bundle) -> dict:
+    out: dict[str, dict] = {}
+    for var in MARGINAL_VARS:
+        out[var] = {"overall": _marginal_overall_partial(b, var)}
+        for axis in GROUPING_AXES:
+            out[var][axis] = _marginal_grouped_partial(b, var, axis)
+    return out
+
+
+def _distance_summary_finalize(parts: list[dict], ctx: Context) -> Reduced:
+    col_labels = ["overall", *GROUPING_AXES]
+    names = list(parts[0][MARGINAL_VARS[0]]["overall"]["r"])
+    matrices: dict[str, list[list[float]]] = {name: [] for name in names}
+
+    for var in MARGINAL_VARS:
+        edges = _marginal_edges(ctx, var)
+        nb = len(edges) - 1
+
+        t_overall = sum_merge([p[var]["overall"]["t"] for p in parts])
+        r_overall = {name: sum_merge([p[var]["overall"]["r"][name] for p in parts]) for name in names}
+        row: dict[str, list[float]] = {name: [] for name in names}
+        for name in names:
+            row[name].append(
+                _ks_statistic(_finalize_counts(r_overall[name], 0, nb), _finalize_counts(t_overall, 0, nb))
+            )
+
+        for axis in GROUPING_AXES:
+            t_grp = sum_merge([p[var][axis]["t"] for p in parts])
+            r_grp = {name: sum_merge([p[var][axis]["r"][name] for p in parts]) for name in names}
+            for name in names:
+                dists, weights = [], []
+                for k in _group_keys(ctx, axis):
+                    rc, tc = _finalize_counts(r_grp[name], k, nb), _finalize_counts(t_grp, k, nb)
+                    w = sum(rc) + sum(tc)
+                    if w == 0:
+                        continue
+                    dists.append(_ks_statistic(rc, tc))
+                    weights.append(w)
+                row[name].append(float(np.average(dists, weights=weights)) if dists else float("nan"))
+
+        for name in names:
+            matrices[name].append(row[name])
+
+    return Reduced(
+        id="marginal_distance_summary",
+        family="quality",
+        kind="heatmap",
+        title="Marginal distance summary (KS statistic, rollout vs reference)",
+        xlabel="grouping axis",
+        payload={
+            "series": matrices,
+            "row_labels": [_TITLE_NAMES[v] for v in MARGINAL_VARS],
+            "col_labels": col_labels,
+            "ylabel": "marginal variable",
+            "cbar_label": "KS statistic (0 = identical, 1 = maximal mismatch)",
+            "vmin": 0.0,
+            "vmax": 1.0,
+        },
+    )
+
+
 # ---------------------------------------------------------------------------
 # per-event scalar observables
 # ---------------------------------------------------------------------------
 
 
 def _event_scalar_partial(b: Bundle, col: str, use_all: bool) -> dict:
-    r_lf, t_lf = (b.r_all, b.t_all) if use_all else (b.r_phys, b.t_phys)
-    r = event_scalars(r_lf)[col].to_numpy()
-    t = event_scalars(t_lf)[col].to_numpy()
-    return {"r": r.tolist(), "t": t.tolist()}
+    t_lf = b.t_all if use_all else b.t_phys
+
+    def _vals(rs: RolloutSide) -> list[float]:
+        lf = rs.all if use_all else rs.phys
+        return event_scalars(lf)[col].to_numpy().tolist()
+
+    return {
+        "r": _per_rollout(b, _vals),
+        "t": event_scalars(t_lf)[col].to_numpy().tolist(),
+    }
 
 
 def _event_scalar_finalize(parts: list[dict], ctx: Context, spec_id: str, title: str, xlabel: str) -> Reduced:
-    r = np.concatenate([np.asarray(p["r"], dtype=float) for p in parts])
+    names = list(parts[0]["r"])
+    r_arrays = {name: np.concatenate([np.asarray(p["r"][name], dtype=float) for p in parts]) for name in names}
     t = np.concatenate([np.asarray(p["t"], dtype=float) for p in parts])
-    edges, rc, tc = _np_hist_pair(r, t, ctx.n_marginal_bins)
+    edges, counts = _np_hist_shared_edges([t, *(r_arrays[n] for n in names)], ctx.n_marginal_bins)
+    t_counts, *r_counts = counts
     return Reduced(
         id=spec_id,
         family="event",
@@ -323,40 +479,44 @@ def _event_scalar_finalize(parts: list[dict], ctx: Context, spec_id: str, title:
         xlabel=xlabel,
         payload={
             "edges": edges.tolist(),
-            _ROLL: rc.astype(np.int64).tolist(),
-            _REF: tc.astype(np.int64).tolist(),
+            "series": {name: c.astype(np.int64).tolist() for name, c in zip(names, r_counts)},
+            "reference": t_counts.astype(np.int64).tolist(),
             "log_y": False,
         },
     )
 
 
 def _event_total_edep_by_energy_partial(b: Bundle) -> dict:
-    r = event_scalars(b.r_all)
     t = event_scalars(b.t_all)
+
+    def _vals(rs: RolloutSide) -> dict:
+        r = event_scalars(rs.all)
+        return {"incident": r["incident_E"].to_list(), "edep": r["total_edep"].to_list()}
+
     return {
-        "r_incident": r["incident_E"].to_list(),
-        "r_edep": r["total_edep"].to_list(),
-        "t_incident": t["incident_E"].to_list(),
-        "t_edep": t["total_edep"].to_list(),
+        "r": _per_rollout(b, _vals),
+        "t": {"incident": t["incident_E"].to_list(), "edep": t["total_edep"].to_list()},
     }
 
 
 def _event_total_edep_by_energy_finalize(parts: list[dict], ctx: Context) -> Reduced:
     e_edges = np.asarray(ctx.energy_edges)
-    r_inc = np.concatenate([np.asarray(p["r_incident"], dtype=float) for p in parts])
-    r_val = np.concatenate([np.asarray(p["r_edep"], dtype=float) for p in parts])
-    t_inc = np.concatenate([np.asarray(p["t_incident"], dtype=float) for p in parts])
-    t_val = np.concatenate([np.asarray(p["t_edep"], dtype=float) for p in parts])
-    r_bin = np.clip(np.digitize(r_inc, e_edges[1:-1]), 0, len(e_edges) - 2)
+    names = list(parts[0]["r"])
+    t_inc = np.concatenate([np.asarray(p["t"]["incident"], dtype=float) for p in parts])
+    t_val = np.concatenate([np.asarray(p["t"]["edep"], dtype=float) for p in parts])
+    r_inc = {n: np.concatenate([np.asarray(p["r"][n]["incident"], dtype=float) for p in parts]) for n in names}
+    r_val = {n: np.concatenate([np.asarray(p["r"][n]["edep"], dtype=float) for p in parts]) for n in names}
+
+    edges, _ = _np_hist_shared_edges([t_val, *(r_val[n] for n in names)], ctx.n_marginal_bins)
     t_bin = np.clip(np.digitize(t_inc, e_edges[1:-1]), 0, len(e_edges) - 2)
-    edges, _, _ = _np_hist_pair(r_val, t_val, ctx.n_marginal_bins)
+    r_bin = {n: np.clip(np.digitize(r_inc[n], e_edges[1:-1]), 0, len(e_edges) - 2) for n in names}
+
     groups: dict[str, dict] = {}
     for bi, lbl in enumerate(energy_bin_labels(e_edges)):
-        rc = np.histogram(r_val[r_bin == bi], edges)[0]
         tc = np.histogram(t_val[t_bin == bi], edges)[0]
         groups[lbl] = {
-            _ROLL: rc.astype(np.int64).tolist(),
-            _REF: tc.astype(np.int64).tolist(),
+            "series": {n: np.histogram(r_val[n][r_bin[n] == bi], edges)[0].astype(np.int64).tolist() for n in names},
+            "reference": tc.astype(np.int64).tolist(),
         }
     return Reduced(
         id="event_total_edep_by_energy",
@@ -375,15 +535,15 @@ def _event_total_edep_by_energy_finalize(parts: list[dict], ctx: Context) -> Red
 
 def _profile_partial(b: Bundle, coord_fn, edges_key: str) -> dict:
     edges = np.asarray(getattr(b.ctx, edges_key))
-    r_lf = attach_entry_axis(b.r_all, entry_axis(b.r_all))
-    t_lf = attach_entry_axis(b.t_all, entry_axis(b.t_all))
-    r_ids, r_mat = profile_partial(r_lf, coord_fn(), edges, pl.col("edep"))
-    t_ids, t_mat = profile_partial(t_lf, coord_fn(), edges, pl.col("edep"))
+
+    def _mat(lf: pl.LazyFrame) -> dict:
+        lf2 = attach_entry_axis(lf, entry_axis(lf))
+        ids, mat = profile_partial(lf2, coord_fn(), edges, pl.col("edep"))
+        return {"ids": ids.tolist(), "mat": mat.tolist()}
+
     return {
-        "r_ids": r_ids.tolist(),
-        "r_mat": r_mat.tolist(),
-        "t_ids": t_ids.tolist(),
-        "t_mat": t_mat.tolist(),
+        "r": _per_rollout(b, lambda rs: _mat(rs.all)),
+        "t": _mat(b.t_all),
     }
 
 
@@ -415,12 +575,19 @@ def _profile_finalize(
 ) -> Reduced:
     edges = np.asarray(getattr(ctx, edges_key))
     nb = len(edges) - 1
-    _assert_event_disjoint([p["r_ids"] for p in parts], spec_id, "rollout")
-    _assert_event_disjoint([p["t_ids"] for p in parts], spec_id, "reference")
-    r_mats = [np.asarray(p["r_mat"], dtype=float).reshape(-1, nb) for p in parts]
-    t_mats = [np.asarray(p["t_mat"], dtype=float).reshape(-1, nb) for p in parts]
-    r_mean, r_std = profile_finalize(r_mats)
+    names = list(parts[0]["r"])
+
+    _assert_event_disjoint([p["t"]["ids"] for p in parts], spec_id, "reference")
+    t_mats = [np.asarray(p["t"]["mat"], dtype=float).reshape(-1, nb) for p in parts]
     t_mean, t_std = profile_finalize(t_mats)
+
+    series: dict[str, dict] = {}
+    for name in names:
+        _assert_event_disjoint([p["r"][name]["ids"] for p in parts], spec_id, name)
+        mats = [np.asarray(p["r"][name]["mat"], dtype=float).reshape(-1, nb) for p in parts]
+        mean, std = profile_finalize(mats)
+        series[name] = {"mean": mean.tolist(), "std": std.tolist()}
+
     return Reduced(
         id=spec_id,
         family="shower",
@@ -429,35 +596,85 @@ def _profile_finalize(
         xlabel=xlabel,
         payload={
             "edges": edges.tolist(),
-            "rollout_mean": r_mean.tolist(),
-            "rollout_std": r_std.tolist(),
-            "reference_mean": t_mean.tolist(),
-            "reference_std": t_std.tolist(),
+            "series": series,
+            "reference": {"mean": t_mean.tolist(), "std": t_std.tolist()},
             "ylabel": "mean deposited energy per event [MeV]",
         },
     )
 
 
+# ---------------------------------------------------------------------------
+# shower containment depth (reuses the longitudinal profile's per-event matrix)
+# ---------------------------------------------------------------------------
+
+_CONTAINMENT_QUANTILES: list[tuple[float, str]] = [
+    (0.90, "shower_containment_depth_90"),
+    (0.95, "shower_containment_depth_95"),
+]
+
+
+def _containment_finalize(parts: list[dict], ctx: Context, spec_id: str, quantile: float) -> Reduced:
+    edges = np.asarray(ctx.depth_edges)
+    nb = len(edges) - 1
+    names = list(parts[0]["r"])
+
+    _assert_event_disjoint([p["t"]["ids"] for p in parts], spec_id, "reference")
+    t_full = np.concatenate([np.asarray(p["t"]["mat"], dtype=float).reshape(-1, nb) for p in parts], axis=0)
+    t_depth = _containment_depths(t_full, edges, quantile)
+
+    r_depths: dict[str, np.ndarray] = {}
+    for name in names:
+        _assert_event_disjoint([p["r"][name]["ids"] for p in parts], spec_id, name)
+        full = np.concatenate([np.asarray(p["r"][name]["mat"], dtype=float).reshape(-1, nb) for p in parts], axis=0)
+        r_depths[name] = _containment_depths(full, edges, quantile)
+
+    hedges, counts = _np_hist_shared_edges([t_depth, *(r_depths[n] for n in names)], ctx.n_marginal_bins)
+    t_counts, *r_counts = counts
+    return Reduced(
+        id=spec_id,
+        family="shower",
+        kind="overlay_hist",
+        title=f"Shower containment depth ({quantile:.0%} of deposited energy)",
+        xlabel=f"depth containing {quantile:.0%} of deposited energy [mm]",
+        payload={
+            "edges": hedges.tolist(),
+            "series": {name: c.astype(np.int64).tolist() for name, c in zip(names, r_counts)},
+            "reference": t_counts.astype(np.int64).tolist(),
+            "log_y": False,
+        },
+    )
+
+
 # ---------------------------------------------------------------------------
 # species share + leakage
 # ---------------------------------------------------------------------------
 
 
 def _species_share_partial(b: Bundle) -> dict:
-    r = species_share(b.r_all)
     t = species_share(b.t_all)
+
+    def _map(rs: RolloutSide) -> dict[str, float]:
+        r = species_share(rs.all)
+        return {str(k): v for k, v in zip(r["pdg"].to_list(), r["total_edep"].to_list())}
+
     return {
-        "r": {str(k): v for k, v in zip(r["pdg"].to_list(), r["total_edep"].to_list())},
+        "r": _per_rollout(b, _map),
         "t": {str(k): v for k, v in zip(t["pdg"].to_list(), t["total_edep"].to_list())},
     }
 
 
 def _species_share_finalize(parts: list[dict], ctx: Context) -> Reduced:
-    r_map = sum_merge([p["r"] for p in parts])
+    names = list(parts[0]["r"])
+    r_maps = {n: sum_merge([p["r"][n] for p in parts]) for n in names}
     t_map = sum_merge([p["t"] for p in parts])
-    r_tot = sum(r_map.values()) or 1.0
     t_tot = sum(t_map.values()) or 1.0
     labels = [pdg_label(k) for k in ctx.top_pdgs]
+
+    series: dict[str, list[float]] = {}
+    for n in names:
+        r_tot = sum(r_maps[n].values()) or 1.0
+        series[n] = [r_maps[n].get(str(k), 0.0) / r_tot for k in ctx.top_pdgs]
+
     return Reduced(
         id="species_edep_share",
         family="species",
@@ -466,22 +683,23 @@ def _species_share_finalize(parts: list[dict], ctx: Context) -> Reduced:
         xlabel="species",
         payload={
             "labels": labels,
-            _ROLL: [r_map.get(str(k), 0.0) / r_tot for k in ctx.top_pdgs],
-            _REF: [t_map.get(str(k), 0.0) / t_tot for k in ctx.top_pdgs],
+            "series": series,
+            "reference": [t_map.get(str(k), 0.0) / t_tot for k in ctx.top_pdgs],
             "ylabel": "fraction of total deposited energy",
         },
     )
 
 
 def _leakage_partial(b: Bundle) -> dict:
-    frac = leakage_fraction(b.r_all)
-    return {"frac": frac.tolist()}
+    return {"r": _per_rollout(b, lambda rs: leakage_fraction(rs.all).tolist())}
 
 
 def _leakage_finalize(parts: list[dict], ctx: Context) -> Reduced:
-    frac = np.concatenate([np.asarray(p["frac"], dtype=float) for p in parts])
-    edges = np.linspace(0.0, max(float(frac.max()) if len(frac) else 1.0, 1e-3), ctx.n_marginal_bins + 1)
-    counts = np.histogram(frac, edges)[0]
+    names = list(parts[0]["r"])
+    arrays = {n: np.concatenate([np.asarray(p["r"][n], dtype=float) for p in parts]) for n in names}
+    max_val = max((float(a.max()) for a in arrays.values() if len(a)), default=1e-3)
+    edges = np.linspace(0.0, max(max_val, 1e-3), ctx.n_marginal_bins + 1)
+    series = {n: np.histogram(arrays[n], edges)[0].astype(np.int64).tolist() for n in names}
     return Reduced(
         id="leakage_fraction",
         family="species",
@@ -490,7 +708,7 @@ def _leakage_finalize(parts: list[dict], ctx: Context) -> Reduced:
         xlabel="escaped energy fraction",
         payload={
             "edges": edges.tolist(),
-            _ROLL: counts.astype(np.int64).tolist(),
+            "series": series,
             "log_y": True,
             "note": "rollout only; the reference has no detector-escape concept",
         },
@@ -502,24 +720,36 @@ def _leakage_finalize(parts: list[dict], ctx: Context) -> Reduced:
 # ---------------------------------------------------------------------------
 
 
-def _sec_frames(b: Bundle):
-    return (
-        secondaries(b.r_phys, Side.rollout),
-        secondaries(b.t_all, Side.reference),
-    )
+def _t_sec(b: Bundle) -> pl.LazyFrame:
+    return secondaries(b.t_all, Side.reference)
+
+
+def _r_sec(rs: RolloutSide) -> pl.LazyFrame:
+    return secondaries(rs.phys, Side.rollout)
 
 
 def _sec_count_per_event_partial(b: Bundle) -> dict:
-    r_sec, t_sec = _sec_frames(b)
-    r = r_sec.group_by("event_id").agg(pl.len().alias("n")).collect(engine="streaming")["n"].to_numpy()
-    t = t_sec.group_by("event_id").agg(pl.len().alias("n")).collect(engine="streaming")["n"].to_numpy()
-    return {"r": r.tolist(), "t": t.tolist()}
+    t = _t_sec(b).group_by("event_id").agg(pl.len().alias("n")).collect(engine="streaming")["n"].to_numpy()
+
+    def _r(rs: RolloutSide) -> list[float]:
+        return (
+            _r_sec(rs)
+            .group_by("event_id")
+            .agg(pl.len().alias("n"))
+            .collect(engine="streaming")["n"]
+            .to_numpy()
+            .tolist()
+        )
+
+    return {"r": _per_rollout(b, _r), "t": t.tolist()}
 
 
 def _sec_count_per_event_finalize(parts: list[dict], ctx: Context) -> Reduced:
-    r = np.concatenate([np.asarray(p["r"], dtype=float) for p in parts])
+    names = list(parts[0]["r"])
     t = np.concatenate([np.asarray(p["t"], dtype=float) for p in parts])
-    edges, rc, tc = _np_hist_pair(r, t, min(ctx.n_marginal_bins, 40))
+    r = {n: np.concatenate([np.asarray(p["r"][n], dtype=float) for p in parts]) for n in names}
+    edges, counts = _np_hist_shared_edges([t, *(r[n] for n in names)], min(ctx.n_marginal_bins, 40))
+    t_c, *r_cs = counts
     return Reduced(
         id="sec_count_per_event",
         family="secondaries",
@@ -528,8 +758,8 @@ def _sec_count_per_event_finalize(parts: list[dict], ctx: Context) -> Reduced:
         xlabel="secondaries per event",
         payload={
             "edges": edges.tolist(),
-            _ROLL: rc.astype(np.int64).tolist(),
-            _REF: tc.astype(np.int64).tolist(),
+            "series": {n: c.astype(np.int64).tolist() for n, c in zip(names, r_cs)},
+            "reference": t_c.astype(np.int64).tolist(),
             "log_y": False,
         },
     )
@@ -541,14 +771,22 @@ def _counts_by_pdg(sec_lf: pl.LazyFrame) -> dict[str, int]:
 
 
 def _sec_count_per_species_partial(b: Bundle) -> dict:
-    r_sec, t_sec = _sec_frames(b)
-    return {"r": _counts_by_pdg(r_sec), "t": _counts_by_pdg(t_sec)}
+    return {"r": _per_rollout(b, lambda rs: _counts_by_pdg(_r_sec(rs))), "t": _counts_by_pdg(_t_sec(b))}
 
 
 def _sec_count_per_species_finalize(parts: list[dict], ctx: Context) -> Reduced:
-    r = sum_merge([p["r"] for p in parts])
+    names = list(parts[0]["r"])
+    r_maps = {n: sum_merge([p["r"][n] for p in parts]) for n in names}
     t = sum_merge([p["t"] for p in parts])
-    keys = sorted(set(r) | set(t), key=lambda k: -(r.get(k, 0) + t.get(k, 0)))[: len(ctx.top_pdgs)]
+
+    all_keys = set(t)
+    for m in r_maps.values():
+        all_keys |= set(m)
+
+    def _total(k: str) -> float:
+        return t.get(k, 0) + sum(m.get(k, 0) for m in r_maps.values())
+
+    keys = sorted(all_keys, key=lambda k: -_total(k))[: len(ctx.top_pdgs)]
     return Reduced(
         id="sec_count_per_species",
         family="secondaries",
@@ -557,39 +795,34 @@ def _sec_count_per_species_finalize(parts: list[dict], ctx: Context) -> Reduced:
         xlabel="species",
         payload={
             "labels": [pdg_label(int(k)) for k in keys],
-            _ROLL: [float(r.get(k, 0)) for k in keys],
-            _REF: [float(t.get(k, 0)) for k in keys],
+            "series": {n: [float(r_maps[n].get(k, 0)) for k in keys] for n in names},
+            "reference": [float(t.get(k, 0)) for k in keys],
             "ylabel": "secondary count",
         },
     )
 
 
 def _sec_energy_partial(b: Bundle) -> dict:
-    r_sec, t_sec = _sec_frames(b)
     edges = np.linspace(*b.ctx.sec_energy_range, b.ctx.n_sec_bins + 1)
     return {
-        "r": _partial_hist(r_sec, pl.col("energy"), edges),
-        "t": _partial_hist(t_sec, pl.col("energy"), edges),
+        "r": _per_rollout(b, lambda rs: _partial_hist(_r_sec(rs), pl.col("energy"), edges)),
+        "t": _partial_hist(_t_sec(b), pl.col("energy"), edges),
     }
 
 
 def _sec_energy_finalize(parts: list[dict], ctx: Context) -> Reduced:
     edges = np.linspace(*ctx.sec_energy_range, ctx.n_sec_bins + 1)
     nb = len(edges) - 1
-    r = sum_merge([p["r"] for p in parts])
+    names = list(parts[0]["r"])
     t = sum_merge([p["t"] for p in parts])
+    series = {name: _finalize_counts(sum_merge([p["r"][name] for p in parts]), 0, nb) for name in names}
     return Reduced(
         id="sec_energy",
         family="secondaries",
         kind="overlay_hist",
         title="Secondary birth energy",
         xlabel="secondary energy [MeV]",
-        payload={
-            "edges": edges.tolist(),
-            _ROLL: _finalize_counts(r, 0, nb),
-            _REF: _finalize_counts(t, 0, nb),
-            "log_y": True,
-        },
+        payload={"edges": edges.tolist(), "series": series, "reference": _finalize_counts(t, 0, nb), "log_y": True},
     )
 
 
@@ -603,26 +836,80 @@ def _sec_cos_angle_partial(b: Bundle) -> dict:
         ea = entry_axis(steps_lf)
         return _partial_hist(attach_entry_axis(sec_lf, ea), cos, edges)
 
-    r_sec, t_sec = _sec_frames(b)
-    return {"r": _side(r_sec, b.r_phys), "t": _side(t_sec, b.t_all)}
+    return {
+        "r": _per_rollout(b, lambda rs: _side(_r_sec(rs), rs.phys)),
+        "t": _side(_t_sec(b), b.t_all),
+    }
 
 
 def _sec_cos_angle_finalize(parts: list[dict], ctx: Context) -> Reduced:
     edges = np.linspace(-1.0, 1.0, ctx.n_sec_bins + 1)
     nb = len(edges) - 1
-    r = sum_merge([p["r"] for p in parts])
+    names = list(parts[0]["r"])
     t = sum_merge([p["t"] for p in parts])
+    series = {name: _finalize_counts(sum_merge([p["r"][name] for p in parts]), 0, nb) for name in names}
     return Reduced(
         id="sec_cos_angle",
         family="secondaries",
         kind="overlay_hist",
         title="Secondary emission angle relative to the shower axis",
         xlabel="cos of emission angle",
+        payload={"edges": edges.tolist(), "series": series, "reference": _finalize_counts(t, 0, nb), "log_y": False},
+    )
+
+
+def _n_sec_confusion_partial(b: Bundle) -> dict:
+    t_ids, t_n = sec_count_by_event(b.t_all, _t_sec(b))
+
+    def _r(rs: RolloutSide) -> dict:
+        ids, n = sec_count_by_event(rs.phys, _r_sec(rs))
+        return {"ids": ids.tolist(), "n": n.tolist()}
+
+    return {"r": _per_rollout(b, _r), "t": {"ids": t_ids.tolist(), "n": t_n.tolist()}}
+
+
+def _n_sec_confusion_finalize(parts: list[dict], ctx: Context) -> Reduced:
+    names = list(parts[0]["r"])
+    # event-disjoint chunking (see Bundle.open) means each event_id appears in
+    # exactly one part on each side, so a plain dict build is a safe merge.
+    t_ids = np.concatenate([np.asarray(p["t"]["ids"], dtype=np.int64) for p in parts])
+    t_n = np.concatenate([np.asarray(p["t"]["n"], dtype=np.int64) for p in parts])
+    t_map = dict(zip(t_ids.tolist(), t_n.tolist()))
+
+    pairs: dict[str, tuple[np.ndarray, np.ndarray]] = {}
+    max_val = 0
+    for name in names:
+        r_ids = np.concatenate([np.asarray(p["r"][name]["ids"], dtype=np.int64) for p in parts])
+        r_n = np.concatenate([np.asarray(p["r"][name]["n"], dtype=np.int64) for p in parts])
+        r_map = dict(zip(r_ids.tolist(), r_n.tolist()))
+        common = sorted(set(r_map) & set(t_map))
+        true_n = np.array([t_map[e] for e in common], dtype=np.int64)
+        pred_n = np.array([r_map[e] for e in common], dtype=np.int64)
+        pairs[name] = (true_n, pred_n)
+        if len(true_n):
+            max_val = max(max_val, int(true_n.max()), int(pred_n.max()))
+
+    cap = min(max(max_val, 1), 20)
+    matrices: dict[str, list[list[int]]] = {}
+    labels: list[str] = []
+    for name in names:
+        true_n, pred_n = pairs[name]
+        labels, mat = _integer_confusion(true_n, pred_n, cap=cap)
+        matrices[name] = mat.tolist()
+
+    return Reduced(
+        id="n_sec_confusion",
+        family="secondaries",
+        kind="heatmap",
+        title="Predicted vs true secondary count per event",
+        xlabel="predicted secondaries (rollout)",
         payload={
-            "edges": edges.tolist(),
-            _ROLL: _finalize_counts(r, 0, nb),
-            _REF: _finalize_counts(t, 0, nb),
-            "log_y": False,
+            "series": matrices,
+            "row_labels": labels,
+            "col_labels": labels,
+            "ylabel": "true secondaries (reference)",
+            "cbar_label": "event count",
+            "vmin": 0.0,
         },
     )
 
@@ -631,17 +918,18 @@ def _sec_cos_angle_finalize(parts: list[dict], ctx: Context) -> Reduced:
 # router diagnostics (not chunked — already bounded/subsampled)
 # ---------------------------------------------------------------------------
 
-_router_gating_partial, _router_gating_finalize = _unchunkable(
-    lambda b: compute_router_gating(b.checkpoint, b.r_phys, b.t_phys)
-)
+_router_gating_partial, _router_gating_finalize = _unchunkable(lambda b: compute_router_gating(b.rollouts, b.t_phys))
 _router_share_pdg_partial, _router_share_pdg_finalize = _unchunkable(
-    lambda b: compute_router_share_by_pdg(b.checkpoint, b.r_phys, b.t_phys, b.ctx.top_pdgs)
+    lambda b: compute_router_share_by_pdg(b.rollouts, b.t_phys, b.ctx.top_pdgs)
 )
 _router_share_process_partial, _router_share_process_finalize = _unchunkable(
-    lambda b: compute_router_share_by_process(b.checkpoint, b.t_phys)
+    lambda b: compute_router_share_by_process(b.rollouts, b.t_phys)
+)
+_router_specialization_partial, _router_specialization_finalize = _unchunkable(
+    lambda b: compute_router_specialization(b.rollouts, b.t_phys)
 )
 _type_embedding_l1_distance_partial, _type_embedding_l1_distance_finalize = _unchunkable(
-    lambda b: compute_type_embedding_l1_distance(b.type_embedding_l1_dist)
+    lambda b: compute_type_embedding_l1_distance(b.rollouts)
 )
 
 
@@ -676,6 +964,15 @@ def build_catalog() -> list[PlotSpec]:
                 )
             )
 
+    specs.append(
+        PlotSpec(
+            "marginal_distance_summary",
+            "quality",
+            compute_partial=_distance_summary_partial,
+            finalize=_distance_summary_finalize,
+        )
+    )
+
     specs += [
         PlotSpec(
             "event_total_edep",
@@ -745,6 +1042,17 @@ def build_catalog() -> list[PlotSpec]:
                 "transverse_edges",
             ),
         ),
+    ]
+    for quantile, spec_id in _CONTAINMENT_QUANTILES:
+        specs.append(
+            PlotSpec(
+                spec_id,
+                "shower",
+                compute_partial=lambda b: _profile_partial(b, depth_expr, "depth_edges"),
+                finalize=lambda parts, ctx, q=quantile, sid=spec_id: _containment_finalize(parts, ctx, sid, q),
+            )
+        )
+    specs += [
         PlotSpec(
             "species_edep_share",
             "species",
@@ -781,6 +1089,12 @@ def build_catalog() -> list[PlotSpec]:
             compute_partial=_sec_cos_angle_partial,
             finalize=_sec_cos_angle_finalize,
         ),
+        PlotSpec(
+            "n_sec_confusion",
+            "secondaries",
+            compute_partial=_n_sec_confusion_partial,
+            finalize=_n_sec_confusion_finalize,
+        ),
         PlotSpec(
             "router_gating",
             "model",
@@ -802,6 +1116,13 @@ def build_catalog() -> list[PlotSpec]:
             finalize=_router_share_process_finalize,
             chunkable=False,
         ),
+        PlotSpec(
+            "router_specialization",
+            "model",
+            compute_partial=_router_specialization_partial,
+            finalize=_router_specialization_finalize,
+            chunkable=False,
+        ),
         PlotSpec(
             "type_embedding_l1_distance",
             "model",

giant/analysis/context.py

@@ -26,7 +26,7 @@ from giant.analysis.reduce import (
     entry_axis,
     transverse_expr,
 )
-from giant.analysis.sources import Side, open_side, physical_steps, secondaries
+from giant.analysis.sources import RolloutSpec, Side, open_side, physical_steps, secondaries
 from giant.analysis.variables import RANGED_VARS
 
 
@@ -74,9 +74,9 @@ def _row_subsample(lf: pl.LazyFrame, sample_rows: int, seed: int) -> pl.LazyFram
     return lf.filter((pl.col("pre_E").hash(seed=seed) % 2**32) < threshold)
 
 
-def _combined_quantiles(r_vals: np.ndarray, t_vals: np.ndarray, lo_q: float, hi_q: float) -> tuple[float, float]:
-    """Robust (lo_q, hi_q) range over the union of two value samples."""
-    both = np.concatenate([r_vals, t_vals])
+def _combined_quantiles(vals: list[np.ndarray], lo_q: float, hi_q: float) -> tuple[float, float]:
+    """Robust (lo_q, hi_q) range over the union of several value samples."""
+    both = np.concatenate(vals)
     lo, hi = float(np.quantile(both, lo_q)), float(np.quantile(both, hi_q))
     if not (hi - lo > 1e-6 * max(abs(hi), 1.0)):
         lo, hi = lo - 0.5, hi + 0.5
@@ -84,7 +84,7 @@ def _combined_quantiles(r_vals: np.ndarray, t_vals: np.ndarray, lo_q: float, hi_
 
 
 def build_context(
-    rollout: str | Path | pl.LazyFrame,
+    rollouts: list[RolloutSpec],
     reference: str | Path | pl.LazyFrame,
     *,
     n_energy_bins: int = 4,
@@ -94,41 +94,51 @@ def build_context(
     sample_rows: int = 1_000_000,
     seed: int = 0,
 ) -> Context:
-    """Resolve the shared context from the two files (the ``prep`` step)."""
-    r_all = open_side(rollout, Side.rollout)
+    """Resolve the shared context from the reference + every rollout (the ``prep`` step).
+
+    Every range/quantile below is the union of the reference and *all*
+    rollouts, so a single set of fixed bin edges/group sets is valid for
+    every series a compute job streams over.
+    """
     t_all = open_side(reference, Side.reference)
-    r_lf = physical_steps(r_all, Side.rollout)
     t_lf = physical_steps(t_all, Side.reference)
+    r_lfs = {rs.name: physical_steps(open_side(rs.source, Side.rollout), Side.rollout) for rs in rollouts}
 
     # Ranged marginal variables: robust ranges over a shared row subsample.
     exprs = [e.alias(n) for n, (_, e) in RANGED_VARS.items()]
-    r_s = _row_subsample(r_lf, sample_rows, seed).select(exprs).collect(engine="streaming")
     t_s = _row_subsample(t_lf, sample_rows, seed).select(exprs).collect(engine="streaming")
+    r_s = {
+        name: _row_subsample(lf, sample_rows, seed).select(exprs).collect(engine="streaming")
+        for name, lf in r_lfs.items()
+    }
     var_ranges = {
-        name: _combined_quantiles(r_s[name].to_numpy(), t_s[name].to_numpy(), _LO_Q, _HI_Q) for name in RANGED_VARS
+        name: _combined_quantiles([t_s[name].to_numpy(), *(df[name].to_numpy() for df in r_s.values())], _LO_Q, _HI_Q)
+        for name in RANGED_VARS
     }
 
     # Energy-bin edges from exact per-event incident energies (cheap group_by).
     def _incident(lf: pl.LazyFrame) -> np.ndarray:
         return lf.group_by("event_id").agg(pl.col("pre_E").max()).collect(engine="streaming")["pre_E"].to_numpy()
 
-    r_inc, t_inc = _incident(r_lf), _incident(t_lf)
-    energy_edges = energy_bin_edges(np.concatenate([r_inc, t_inc]), n_energy_bins)
+    t_inc = _incident(t_lf)
+    r_inc = {name: _incident(lf) for name, lf in r_lfs.items()}
+    energy_edges = energy_bin_edges(np.concatenate([t_inc, *r_inc.values()]), n_energy_bins)
 
     # Top PDG species and material list (cheap single-column group_bys).
     def _counts(lf: pl.LazyFrame, col: str) -> pl.DataFrame:
         return lf.group_by(col).agg(pl.len().alias("n")).collect(engine="streaming")
 
     pdg_counts = (
-        pl.concat([_counts(r_lf, "pdg"), _counts(t_lf, "pdg")])
+        pl.concat([_counts(t_lf, "pdg"), *(_counts(lf, "pdg") for lf in r_lfs.values())])
         .group_by("pdg")
         .agg(pl.col("n").sum())
         .sort("n", descending=True)
     )
     top_pdgs = [int(x) for x in pdg_counts["pdg"].to_list()[:top_k_pdg]]
-    materials = sorted(
-        set(_counts(r_lf, "material")["material"].to_list()) | set(_counts(t_lf, "material")["material"].to_list())
-    )
+    material_set: set[str] = set(_counts(t_lf, "material")["material"].to_list())
+    for lf in r_lfs.values():
+        material_set |= set(_counts(lf, "material")["material"].to_list())
+    materials = sorted(material_set)
 
     # Shower depth / transverse ranges from a subsampled proxy.
     def _proxy(lf: pl.LazyFrame) -> tuple[np.ndarray, np.ndarray]:
@@ -140,19 +150,20 @@ def build_context(
         )
         return sub["d"].to_numpy(), sub["t"].to_numpy()
 
-    r_d, r_t = _proxy(r_lf)
     t_d, t_t = _proxy(t_lf)
-    d_lo, d_hi = _combined_quantiles(r_d, t_d, _LO_Q, _HI_Q)
+    r_proxy = {name: _proxy(lf) for name, lf in r_lfs.items()}
+    d_lo, d_hi = _combined_quantiles([t_d, *(p[0] for p in r_proxy.values())], _LO_Q, _HI_Q)
     depth_edges = np.linspace(d_lo, d_hi, n_marginal_bins + 1)
-    t_hi = max(float(np.quantile(np.concatenate([r_t, t_t]), _HI_Q)), 1e-6)
+    t_hi = max(float(np.quantile(np.concatenate([t_t, *(p[1] for p in r_proxy.values())]), _HI_Q)), 1e-6)
     transverse_edges = np.linspace(0.0, t_hi, n_marginal_bins + 1)
 
     # Secondary energy range.
-    r_se = secondaries(r_lf, Side.rollout).select("energy")
-    t_se = secondaries(t_all, Side.reference).select("energy")
-    r_se = _row_sample_col(r_se, sample_rows, seed)
-    t_se = _row_sample_col(t_se, sample_rows, seed)
-    sec_energy_range = _combined_quantiles(r_se, t_se, _LO_Q, _HI_Q)
+    t_se = _row_sample_col(secondaries(t_all, Side.reference).select("energy"), sample_rows, seed)
+    r_se = {
+        name: _row_sample_col(secondaries(lf, Side.rollout).select("energy"), sample_rows, seed)
+        for name, lf in r_lfs.items()
+    }
+    sec_energy_range = _combined_quantiles([t_se, *r_se.values()], _LO_Q, _HI_Q)
 
     return Context(
         n_marginal_bins=n_marginal_bins,
@@ -165,8 +176,8 @@ def build_context(
         sec_energy_range=sec_energy_range,
         n_sec_bins=n_sec_bins,
         n_events={
-            "rollout": len(r_inc),
             "reference": len(t_inc),
+            **{name: len(arr) for name, arr in r_inc.items()},
         },
     )
 

giant/analysis/reduce.py

@@ -271,3 +271,20 @@ def leakage_fraction(lf: pl.LazyFrame) -> np.ndarray:
     escaped = per_event["escaped"].fill_null(0.0).to_numpy()
     total = deposited + escaped
     return np.where(total > 0, escaped / total, 0.0)
+
+
+def sec_count_by_event(lf_all: pl.LazyFrame, sec_lf: pl.LazyFrame) -> tuple[np.ndarray, np.ndarray]:
+    """Per-event secondary count, zero-filled for events that produced none.
+
+    Two bounded per-event ``group_by``s — the full event set (from ``lf_all``)
+    and the secondary counts (from ``sec_lf``, see ``sources.secondaries``) —
+    merged in Python via a dict. Both results are event-granularity (not
+    per-row), so this stays in the same bounded-memory budget as
+    ``event_scalars``; a plain ``group_by`` on ``sec_lf`` alone would silently
+    drop zero-secondary events instead of zero-filling them.
+    """
+    ev = lf_all.select("event_id").unique().collect(engine="streaming")["event_id"].to_numpy()
+    cnt_df = sec_lf.group_by("event_id").agg(pl.len().alias("n")).collect(engine="streaming")
+    cnt = dict(zip(cnt_df["event_id"].to_list(), cnt_df["n"].to_list()))
+    counts = np.array([cnt.get(int(e), 0) for e in ev], dtype=np.int64)
+    return ev, counts

giant/analysis/reduced.py

@@ -11,15 +11,24 @@ import json
 from dataclasses import asdict, dataclass, field
 from pathlib import Path
 
-# Reduced.kind values:
-#   "overlay_hist"   rollout vs reference density histogram over shared edges
+# Reduced.kind values (payload keys a rollout series by name under
+# payload["series"], with the reference — where one exists — kept as one
+# distinguished payload["reference"] entry; see catalog.py's module
+# docstring for the full per-kind payload shape):
+#   "overlay_hist"   N-rollout-series vs reference density histogram over shared edges
 #   "grouped_hist"   one panel per group (energy/pdg/material), each an overlay
-#   "profile"        edep-weighted mean +/- event-RMS vs depth/radius, two series
-#   "bar"            per-category rollout vs reference bars (share / counts)
-#   "single_hist"    one series only (e.g. rollout leakage; reference has none)
-#   "router_gating"  stacked mean MoE gate weight vs energy, rollout + reference
-#   "router_share"   stacked bar of MoE top-1 dispatch share by category
-#   "unavailable"    plot not applicable to this run (e.g. non-MoE checkpoint)
+#   "profile"        edep-weighted mean +/- event-RMS vs depth/radius, N series + reference
+#   "bar"            per-category N-rollout-series vs reference bars (share / counts)
+#   "single_hist"    rollout-only series (e.g. leakage; reference has none)
+#   "router_gating"  stacked mean MoE gate weight vs energy, one rollout+reference
+#                    panel-pair per rollout with an enabled MoE router
+#   "router_share"   stacked bar of MoE top-1 dispatch share by category, one
+#                    panel per rollout with an enabled MoE router
+#   "router_specialization"  max gate weight vs energy (one scalar trend line
+#                    summarizing "router_gating"), per rollout with an enabled router
+#   "heatmap"        row x col matrix + colorbar, one panel per rollout (a
+#                    distance scorecard or a predicted-vs-true confusion matrix)
+#   "unavailable"    plot not applicable to this run (e.g. no MoE checkpoint)
 
 
 @dataclass

giant/analysis/render.py

@@ -9,10 +9,19 @@ streaming compute.
 For each reduced artifact it writes ``<out>/<family>/<id>.pdf`` plus a sibling
 ``<id>.yaml`` (per-plot gallery metadata) and a per-family ``metadata.yaml``.
 Optionally runs ``gallery generate`` to build the static HTML site.
+
+Every rollout series gets a stable color via ``ps.get_color(i)``, ``i`` being
+its position in ``payload["series"]`` — that position is fixed by the run's
+YAML/``--label`` order (threaded unchanged from ``condor.RunMeta.rollouts``
+through every ``PlotSpec``), so a given rollout keeps the same color across
+every plot in a run. The reference, where a plot has one, always draws in one
+fixed, distinct style (dark ink, dashed) instead of taking a slot in that
+cycle.
 """
 
 from __future__ import annotations
 
+import dataclasses
 import subprocess
 from pathlib import Path
 
@@ -22,7 +31,32 @@ import yaml
 
 from giant.analysis.reduced import Reduced
 
-_SERIES_LABELS = {"rollout": "rollout", "reference": "reference (Geant4)"}
+_REFERENCE_LABEL = "reference (Geant4)"
+
+_TEX_ESCAPE_MAP = {
+    "\\": r"\textbackslash{}",
+    "%": r"\%",
+    "&": r"\&",
+    "#": r"\#",
+    "$": r"\$",
+    "_": r"\_",
+    "{": r"\{",
+    "}": r"\}",
+}
+
+
+def _tex_escape(text: str) -> str:
+    """Escape characters LaTeX treats specially in catalog-authored title/xlabel
+    text (e.g. a literal ``%`` in a "90% of deposited energy" title, which
+    ``usetex`` otherwise reads as a comment marker and aborts the whole figure —
+    see gitea #81). A single pass over the *original* characters, so the
+    backslashes an escape itself introduces (e.g. ``\textbackslash{}``) are
+    never re-escaped."""
+    return "".join(_TEX_ESCAPE_MAP.get(c, c) for c in text)
+
+
+def _ref_color() -> str:
+    return ps.colors.INK["primary"]
 
 
 def _density(counts: list[int] | np.ndarray, edges: np.ndarray) -> np.ndarray:
@@ -33,10 +67,13 @@ def _density(counts: list[int] | np.ndarray, edges: np.ndarray) -> np.ndarray:
     return counts / (total * (edges[1] - edges[0]))
 
 
-def _overlay(ax, edges: np.ndarray, series: dict[str, list], log_y: bool) -> None:
-    for key in ("reference", "rollout"):
-        if key in series:
-            ax.stairs(_density(series[key], edges), edges, label=_SERIES_LABELS[key])
+def _overlay(ax, edges: np.ndarray, payload: dict, log_y: bool) -> None:
+    if "reference" in payload:
+        ax.stairs(
+            _density(payload["reference"], edges), edges, label=_REFERENCE_LABEL, color=_ref_color(), linestyle="--"
+        )
+    for i, (name, counts) in enumerate(payload.get("series", {}).items()):
+        ax.stairs(_density(counts, edges), edges, label=name, color=ps.get_color(i))
     if log_y:
         ax.set_yscale("log")
 
@@ -47,8 +84,8 @@ def _router_summary(router_cfg: dict) -> str:
     return f"{router_cfg.get('type', '?')}×{router_cfg.get('n_experts', '?')}"
 
 
-def _figure_params_v2(mc: dict, run_meta: dict) -> dict:
-    """`_figure_params` for a new-shape (nested) `model_config` — has a
+def _figure_params_v2(mc: dict, meta: dict) -> dict:
+    """`_figure_params_single` for a new-shape (nested) `model_config` — has a
     `stage1_model` key. Reports stage 1's architecture (the headline
     generator); stage 2's generator is only added (`mode_s2`) when it
     differs from stage 1's, since a mixed run (the `stage1=flow` +
@@ -71,23 +108,24 @@ def _figure_params_v2(mc: dict, run_meta: dict) -> dict:
     if particle_type is not None:
         params["conditioning"] = particle_type
     params["router"] = _router_summary(s1.get("router") or {})
-    if run_meta.get("training_epoch") is not None:
-        params["epoch"] = run_meta["training_epoch"]
-    if run_meta.get("best_val_loss") is not None:
-        params["best_val_loss"] = round(run_meta["best_val_loss"], 4)
+    if meta.get("training_epoch") is not None:
+        params["epoch"] = meta["training_epoch"]
+    if meta.get("best_val_loss") is not None:
+        params["best_val_loss"] = round(meta["best_val_loss"], 4)
     if mode == "wgan":
         noise_dim = (s1.get("wgan") or {}).get("noise_dim")
         if noise_dim is not None:
             params["noise_dim"] = noise_dim
-    elif run_meta.get("steps") is not None:
-        params["steps"] = run_meta["steps"]
+    elif meta.get("steps") is not None:
+        params["steps"] = meta["steps"]
     return params
 
 
-def _figure_params(run_meta: dict) -> dict:
-    """Curated run identity for the figure subtitle (``new_figure(params=...)``).
+def _figure_params_single(meta: dict) -> dict:
+    """Curated run identity for the figure subtitle (``new_figure(params=...)``),
+    for exactly one rollout's ``plot_meta``.
 
-    ``run_meta``/each plot's own ``<id>.yaml`` (see ``_plot_metadata``) already
+    ``meta``/each plot's own ``<id>.yaml`` (see ``_plot_metadata``) already
     carry every threaded model/training/rollout/dataset parameter for
     after-the-fact lookup — this picks only the handful that matter for
     telling figures apart at a glance while flipping through a gallery, since
@@ -99,9 +137,9 @@ def _figure_params(run_meta: dict) -> dict:
     Handles both a v0.2 checkpoint's flat ``model_config`` and a v0.3.0
     nested one (has a ``stage1_model`` key — see ``_figure_params_v2``).
     """
-    mc = run_meta.get("model_config") or {}
+    mc = meta.get("model_config") or {}
     if "stage1_model" in mc:
-        return _figure_params_v2(mc, run_meta)
+        return _figure_params_v2(mc, meta)
 
     mode = mc.get("mode")
     params: dict = {}
@@ -114,18 +152,35 @@ def _figure_params(run_meta: dict) -> dict:
     if mc.get("conditioning") is not None:
         params["conditioning"] = mc["conditioning"]
     params["router"] = _router_summary(mc.get("router") or {})
-    if run_meta.get("training_epoch") is not None:
-        params["epoch"] = run_meta["training_epoch"]
-    if run_meta.get("best_val_loss") is not None:
-        params["best_val_loss"] = round(run_meta["best_val_loss"], 4)
+    if meta.get("training_epoch") is not None:
+        params["epoch"] = meta["training_epoch"]
+    if meta.get("best_val_loss") is not None:
+        params["best_val_loss"] = round(meta["best_val_loss"], 4)
     if mode == "wgan":
         if mc.get("noise_dim") is not None:
             params["noise_dim"] = mc["noise_dim"]
-    elif run_meta.get("steps") is not None:
-        params["steps"] = run_meta["steps"]
+    elif meta.get("steps") is not None:
+        params["steps"] = meta["steps"]
     return params
 
 
+def _figure_params(run_meta: dict) -> dict:
+    """Curated run identity for the figure subtitle.
+
+    A single-rollout run reuses that rollout's ``plot_meta`` (same curated
+    model/training/rollout subset as always — see ``_figure_params_single``);
+    a multi-rollout run instead names the series being compared, since no
+    single ``model_config`` applies to the figure as a whole (each plot's own
+    gallery YAML still carries every rollout's full ``plot_meta`` for
+    after-the-fact lookup, via ``_plot_metadata``).
+    """
+    rollouts = run_meta.get("rollouts") or {}
+    if len(rollouts) == 1:
+        ((_, meta),) = rollouts.items()
+        return _figure_params_single(meta)
+    return {"rollouts": ", ".join(rollouts)} if rollouts else {}
+
+
 def _render_overlay(r: Reduced, params: dict):
     edges = np.asarray(r.payload["edges"])
     fig, ax = ps.new_figure("thesis-single", title=r.title, params=params)
@@ -139,7 +194,8 @@ def _render_overlay(r: Reduced, params: dict):
 def _render_single(r: Reduced, params: dict):
     edges = np.asarray(r.payload["edges"])
     fig, ax = ps.new_figure("thesis-single", title=r.title, params=params)
-    ax.stairs(_density(r.payload["rollout"], edges), edges, label=_SERIES_LABELS["rollout"])
+    for i, (name, counts) in enumerate(r.payload.get("series", {}).items()):
+        ax.stairs(_density(counts, edges), edges, label=name, color=ps.get_color(i))
     if r.payload.get("log_y"):
         ax.set_yscale("log")
     if r.payload.get("log_x"):
@@ -181,11 +237,17 @@ def _render_profile(r: Reduced, params: dict):
     edges = np.asarray(r.payload["edges"])
     centers = 0.5 * (edges[:-1] + edges[1:])
     fig, ax = ps.new_figure("thesis-single", title=r.title, params=params)
-    for key in ("reference", "rollout"):
-        mean = np.asarray(r.payload[f"{key}_mean"])
-        std = np.asarray(r.payload[f"{key}_std"])
-        (line,) = ax.plot(centers, mean, label=_SERIES_LABELS[key])
-        ax.fill_between(centers, mean - std, mean + std, alpha=0.2, color=line.get_color())
+    if "reference" in r.payload:
+        ref = r.payload["reference"]
+        mean, std = np.asarray(ref["mean"]), np.asarray(ref["std"])
+        color = _ref_color()
+        ax.plot(centers, mean, label=_REFERENCE_LABEL, color=color, linestyle="--")
+        ax.fill_between(centers, mean - std, mean + std, alpha=0.2, color=color)
+    for i, (name, side) in enumerate(r.payload.get("series", {}).items()):
+        mean, std = np.asarray(side["mean"]), np.asarray(side["std"])
+        color = ps.get_color(i)
+        ax.plot(centers, mean, label=name, color=color)
+        ax.fill_between(centers, mean - std, mean + std, alpha=0.2, color=color)
     ax.set_xlabel(r.xlabel)
     ax.set_ylabel(r.payload.get("ylabel", "mean deposited energy [MeV]"))
     ps.style_legend(ax, title="source")
@@ -195,10 +257,19 @@ def _render_profile(r: Reduced, params: dict):
 def _render_bar(r: Reduced, params: dict):
     labels = r.payload["labels"]
     x = np.arange(len(labels))
-    width = 0.4
+    series = r.payload.get("series", {})
+    has_ref = "reference" in r.payload
+    n_bars = len(series) + (1 if has_ref else 0)
+    width = 0.8 / max(n_bars, 1)
+    offsets = np.linspace(-0.4 + width / 2, 0.4 - width / 2, n_bars)
     fig, ax = ps.new_figure("thesis-single", title=r.title, params=params)
-    ax.bar(x - width / 2, r.payload["reference"], width, label=_SERIES_LABELS["reference"])
-    ax.bar(x + width / 2, r.payload["rollout"], width, label=_SERIES_LABELS["rollout"])
+    idx = 0
+    if has_ref:
+        ax.bar(x + offsets[idx], r.payload["reference"], width, label=_REFERENCE_LABEL, color=_ref_color())
+        idx += 1
+    for i, (name, vals) in enumerate(series.items()):
+        ax.bar(x + offsets[idx], vals, width, label=name, color=ps.get_color(i))
+        idx += 1
     ax.set_xticks(x)
     ax.set_xticklabels(labels, rotation=45, ha="right")
     ax.set_ylabel(r.payload.get("ylabel", "value"))
@@ -207,56 +278,137 @@ def _render_bar(r: Reduced, params: dict):
 
 
 def _render_router_gating(r: Reduced, params: dict):
-    n_experts = r.payload["n_experts"]
+    series = r.payload.get("series", {})
+    names = list(series)
     log_x = r.payload.get("log_x", False)
-    fig, axes = ps.new_figure("slide-16x9", title=r.title, params=params, nrows=1, ncols=2, squeeze=False)
-    flat = axes.ravel()
-    for ax, key in zip(flat, ("rollout", "reference")):
-        side = r.payload.get(key, {})
-        centers = np.asarray(side.get("centers", []))
-        means = np.asarray(side.get("means", []))
-        if len(centers) and means.size:
-            cum = np.zeros(len(centers))
-            for i in range(n_experts):
-                ax.fill_between(centers, cum, cum + means[:, i], alpha=0.7, label=f"expert {i}")
-                cum = cum + means[:, i]
-        if log_x:
-            ax.set_xscale("log")
-        ax.set_ylim(0, 1)
-        ax.set_title(_SERIES_LABELS[key], fontsize=8)
-        ax.set_xlabel(r.xlabel)
-    flat[0].set_ylabel("mean gate weight")
-    ps.style_legend(flat[0], title=f"{r.payload.get('router_type', '')} router")
+    fig, axes = ps.new_figure("slide-16x9", title=r.title, params=params, nrows=len(names), ncols=2, squeeze=False)
+    for row, name in enumerate(names):
+        entry = series[name]
+        n_experts = entry["n_experts"]
+        for col, key in enumerate(("rollout", "reference")):
+            ax = axes[row, col]
+            side = entry.get(key, {})
+            centers = np.asarray(side.get("centers", []))
+            means = np.asarray(side.get("means", []))
+            if len(centers) and means.size:
+                cum = np.zeros(len(centers))
+                for i in range(n_experts):
+                    ax.fill_between(centers, cum, cum + means[:, i], alpha=0.7, label=f"expert {i}")
+                    cum = cum + means[:, i]
+            if log_x:
+                ax.set_xscale("log")
+            ax.set_ylim(0, 1)
+            panel_label = _REFERENCE_LABEL if key == "reference" else "rollout"
+            ax.set_title(f"{name} — {panel_label}", fontsize=8)
+            if row == len(names) - 1:
+                ax.set_xlabel(r.xlabel)
+        axes[row, 0].set_ylabel("mean gate weight")
+    if names:
+        ps.style_legend(axes[0, 0], title=f"{series[names[0]]['router_type']} router")
     return fig
 
 
 def _render_router_share(r: Reduced, params: dict):
-    categories = r.payload["categories"]
-    n_experts = r.payload["n_experts"]
-    x = np.arange(len(categories))
-    present = [k for k in ("rollout", "reference") if k in r.payload]
+    series = r.payload.get("series", {})
+    names = list(series)
+    present: tuple[str, ...] = ("rollout", "reference")
+    if names:
+        present = tuple(k for k in ("rollout", "reference") if k in series[names[0]])
+    ncols = max(len(present), 1)
+    fig, axes = ps.new_figure("slide-16x9", title=r.title, params=params, nrows=len(names), ncols=ncols, squeeze=False)
+    for row, name in enumerate(names):
+        entry = series[name]
+        n_experts = entry["n_experts"]
+        cats = entry["categories"]
+        x = np.arange(len(cats))
+        for col, key in enumerate(present):
+            ax = axes[row, col]
+            side = entry.get(key)
+            if side is not None:
+                shares = np.array([side[c] for c in cats])  # (n_cat, n_experts)
+                bottom = np.zeros(len(cats))
+                for i in range(n_experts):
+                    ax.bar(x, shares[:, i], bottom=bottom, label=f"expert {i}")
+                    bottom += shares[:, i]
+            ax.set_xticks(x)
+            ax.set_xticklabels(cats, rotation=45, ha="right")
+            ax.set_ylim(0, 1)
+            panel_label = _REFERENCE_LABEL if key == "reference" else "rollout"
+            ax.set_title(f"{name} — {panel_label}", fontsize=8)
+        axes[row, 0].set_ylabel("share of rows dispatched to expert")
+    if names:
+        ps.style_legend(axes[0, 0], title=f"{series[names[0]]['router_type']} router")
+    return fig
+
+
+def _render_router_specialization(r: Reduced, params: dict):
+    fig, ax = ps.new_figure("thesis-single", title=r.title, params=params)
+    series = r.payload.get("series", {})
+    chance_levels: set[float] = set()
+    for i, (name, entry) in enumerate(series.items()):
+        color = ps.get_color(i)
+        if entry.get("chance_level") is not None:
+            chance_levels.add(entry["chance_level"])
+        for key, linestyle, label in (
+            ("rollout", "-", name),
+            ("reference", "--", f"{name} ({_REFERENCE_LABEL})"),
+        ):
+            side = entry.get(key)
+            if side and side["centers"]:
+                ax.plot(
+                    side["centers"],
+                    side["score"],
+                    label=label,
+                    color=color,
+                    linestyle=linestyle,
+                    marker="o",
+                    markersize=3,
+                )
+    for lvl in sorted(chance_levels):
+        ax.axhline(lvl, linestyle=":", color="gray")
+    if r.payload.get("log_x"):
+        ax.set_xscale("log")
+    ax.set_ylim(0, 1)
+    ax.set_xlabel(r.xlabel)
+    ax.set_ylabel("max gate weight")
+    ps.style_legend(ax, title="router")
+    return fig
+
+
+def _render_heatmap(r: Reduced, params: dict):
+    series = r.payload["series"]
+    row_labels = r.payload["row_labels"]
+    col_labels = r.payload["col_labels"]
+    names = list(series)
     fig, axes = ps.new_figure(
-        "slide-16x9",
+        "slide-16x9" if len(names) > 1 else "thesis-single",
         title=r.title,
         params=params,
         nrows=1,
-        ncols=len(present),
+        ncols=len(names),
         squeeze=False,
     )
     flat = axes.ravel()
-    for ax, key in zip(flat, present):
-        side = r.payload[key]
-        shares = np.array([side[c] for c in categories])  # (n_cat, n_experts)
-        bottom = np.zeros(len(categories))
-        for i in range(n_experts):
-            ax.bar(x, shares[:, i], bottom=bottom, label=f"expert {i}")
-            bottom += shares[:, i]
-        ax.set_xticks(x)
-        ax.set_xticklabels(categories, rotation=45, ha="right")
-        ax.set_ylim(0, 1)
-        ax.set_title(_SERIES_LABELS[key], fontsize=8)
-    flat[0].set_ylabel("share of rows dispatched to expert")
-    ps.style_legend(flat[0], title=f"{r.payload.get('router_type', '')} router")
+    im = None
+    for ax, name in zip(flat, names):
+        mat = np.asarray(series[name], dtype=float)
+        im = ax.imshow(
+            mat,
+            origin="upper",
+            aspect="auto",
+            cmap=r.payload.get("cmap", "viridis"),
+            vmin=r.payload.get("vmin"),
+            vmax=r.payload.get("vmax"),
+        )
+        ax.set_xticks(range(len(col_labels)))
+        ax.set_xticklabels(col_labels, rotation=45, ha="right")
+        ax.set_yticks(range(len(row_labels)))
+        ax.set_yticklabels(row_labels)
+        ax.set_xlabel(r.xlabel)
+        if len(names) > 1:
+            ax.set_title(name, fontsize=8)
+    flat[0].set_ylabel(r.payload.get("ylabel", ""))
+    fig.colorbar(im, ax=list(flat), label=r.payload.get("cbar_label", "value"))
     return fig
 
 
@@ -284,13 +436,21 @@ _RENDERERS = {
     "bar": _render_bar,
     "router_gating": _render_router_gating,
     "router_share": _render_router_share,
+    "router_specialization": _render_router_specialization,
+    "heatmap": _render_heatmap,
     "unavailable": _render_unavailable,
 }
 
 
 def render(r: Reduced, run_meta: dict | None = None):
-    """Build the matplotlib figure for one reduced artifact (dispatch on kind)."""
-    return _RENDERERS[r.kind](r, _figure_params(run_meta or {}))
+    """Build the matplotlib figure for one reduced artifact (dispatch on kind).
+
+    ``title``/``xlabel`` are LaTeX-escaped here, at the one point every kind's
+    renderer draws them from — ``_plot_metadata`` deliberately keeps using the
+    unescaped ``r`` for the gallery YAML, which isn't LaTeX.
+    """
+    escaped = dataclasses.replace(r, title=_tex_escape(r.title), xlabel=_tex_escape(r.xlabel))
+    return _RENDERERS[r.kind](escaped, _figure_params(run_meta or {}))
 
 
 def _plot_metadata(r: Reduced, run_meta: dict) -> dict:
@@ -349,7 +509,7 @@ def render_all(
         yaml.safe_dump(
             {
                 "title": run_meta.get("title", "GIANT rollout analysis"),
-                "description": "Autoregressive rollout compared against held-out Geant4 reference steps.",
+                "description": "Autoregressive rollout(s) compared against a held-out Geant4 reference steps file.",
                 "experiment": "GIANT",
                 "parameters": {k: v for k, v in run_meta.items() if k != "title"},
             },
@@ -375,15 +535,14 @@ def render_run(run_dir: str | Path, *, run_gallery: bool = False) -> list[Path]:
     (checkpoint, paths, cutoffs) from ``run_meta.json`` into every plot's
     gallery metadata and renders.
     """
-    from giant.analysis.condor import RunMeta, merge_all
+    from giant.analysis.run import RunMeta, merge_all
 
     run_dir = Path(run_dir)
     merge_all(run_dir)
     meta = RunMeta.load(run_dir / "run_meta.json")
     run_meta = {
         "title": meta.title,
-        "rollout": meta.rollout,
         "reference": meta.reference,
-        **meta.plot_meta,
+        "rollouts": {ro["name"]: ro["plot_meta"] for ro in meta.rollouts},
     }
     return render_all(run_dir / "reduced", run_dir / "plots", run_meta, run_gallery=run_gallery)

giant/analysis/router_gating.py

@@ -35,6 +35,7 @@ from giant.analysis.reduced import Reduced
 if TYPE_CHECKING:
     import torch
 
+    from giant.analysis.sources import RolloutSide
     from giant.data.transforms import Normalizer
 
 _SAMPLE_ROWS = 200_000
@@ -203,6 +204,7 @@ _TITLES = {
     "router_gating": "Router gating (mixture-of-experts decision boundaries)",
     "router_share_by_pdg": "Router expert share by particle species",
     "router_share_by_process": "Router expert share by physics process",
+    "router_specialization": "Router specialization score vs energy (max gate weight)",
 }
 
 
@@ -217,51 +219,103 @@ def _unavailable(spec_id: str) -> Reduced:
     )
 
 
-def compute_router_gating(
-    checkpoint: str | Path | None,
-    r_phys: pl.LazyFrame,
-    t_phys: pl.LazyFrame,
-    seed: int = 0,
-) -> Reduced:
-    """`Reduced` for the router-gating figure, or an explanatory note if n/a."""
+def _gating_entry(checkpoint: str | Path | None, r_phys: pl.LazyFrame, t_phys: pl.LazyFrame, seed: int) -> dict | None:
+    """One rollout's ``router_gating`` panel data, or ``None`` if not a MoE checkpoint."""
     handle = load_router(checkpoint) if checkpoint else None
     if handle is None:
-        return _unavailable("router_gating")
-
+        return None
     sides: dict[str, dict] = {}
     for name, lf in (("rollout", r_phys), ("reference", t_phys)):
         df = _subsample(lf, _SAMPLE_ROWS, seed)
         df, gate = _gate_for_df(handle, df)
         x = df["pre_E"].to_numpy()
         sides[name] = _quantile_bins(x, gate, _N_BINS) if len(x) else {"centers": [], "means": []}
+    return {"router_type": handle.router_type, "n_experts": handle.router.n_experts, **sides}
 
+
+def compute_router_gating(rollouts: dict[str, "RolloutSide"], t_phys: pl.LazyFrame, seed: int = 0) -> Reduced:
+    """`Reduced` for the router-gating figure: one panel-pair per rollout with
+    an enabled MoE router, or an explanatory note if none of them have one."""
+    series = {}
+    for name, rs in rollouts.items():
+        entry = _gating_entry(rs.checkpoint, rs.phys, t_phys, seed)
+        if entry is not None:
+            series[name] = entry
+    if not series:
+        return _unavailable("router_gating")
     return Reduced(
         id="router_gating",
         family="model",
         kind="router_gating",
         title=_TITLES["router_gating"],
         xlabel="pre-step energy [MeV]",
-        payload={
-            "router_type": handle.router_type,
-            "n_experts": handle.router.n_experts,
-            "log_x": True,
-            **sides,
-        },
+        payload={"series": series, "log_x": True},
     )
 
 
-def compute_router_share_by_pdg(
-    checkpoint: str | Path | None,
-    r_phys: pl.LazyFrame,
-    t_phys: pl.LazyFrame,
-    top_pdgs: list[int],
-    seed: int = 0,
-) -> Reduced:
-    """Stacked-bar share of each particle species dispatched to each expert."""
+def _specialization_entry(
+    checkpoint: str | Path | None, r_phys: pl.LazyFrame, t_phys: pl.LazyFrame, seed: int
+) -> dict | None:
+    """One rollout's ``router_specialization`` curve data, or ``None`` if not a MoE checkpoint.
+
+    Scalar specialization trend: max gate weight vs energy, per side.
+    Summarizes `router_gating`'s full per-expert stacked area into one curve —
+    the routing plan's own "how sharp is the boundary here" number (1/n_experts
+    = uniform/no specialization, 1.0 = one expert fully owns that energy). Same
+    quantile energy bins as `router_gating` (`_quantile_bins`), so this is
+    directly comparable to that plot's ceiling described in the roadmap's MoE
+    writeup.
+    """
     handle = load_router(checkpoint) if checkpoint else None
     if handle is None:
-        return _unavailable("router_share_by_pdg")
+        return None
+    sides: dict[str, dict] = {}
+    for name, lf in (("rollout", r_phys), ("reference", t_phys)):
+        df = _subsample(lf, _SAMPLE_ROWS, seed)
+        df, gate = _gate_for_df(handle, df)
+        x = df["pre_E"].to_numpy()
+        if len(x):
+            binned = _quantile_bins(x, gate, _N_BINS)
+            means = np.asarray(binned["means"])
+            score = means.max(axis=1).tolist() if means.size else []
+            sides[name] = {"centers": binned["centers"], "score": score}
+        else:
+            sides[name] = {"centers": [], "score": []}
+    return {
+        "router_type": handle.router_type,
+        "n_experts": handle.router.n_experts,
+        "chance_level": 1.0 / handle.router.n_experts,
+        **sides,
+    }
 
+
+def compute_router_specialization(rollouts: dict[str, "RolloutSide"], t_phys: pl.LazyFrame, seed: int = 0) -> Reduced:
+    """`Reduced` for the router-specialization figure, one curve per rollout with
+    an enabled MoE router (see `_specialization_entry`)."""
+    series = {}
+    for name, rs in rollouts.items():
+        entry = _specialization_entry(rs.checkpoint, rs.phys, t_phys, seed)
+        if entry is not None:
+            series[name] = entry
+    if not series:
+        return _unavailable("router_specialization")
+    return Reduced(
+        id="router_specialization",
+        family="model",
+        kind="router_specialization",
+        title=_TITLES["router_specialization"],
+        xlabel="pre-step energy [MeV]",
+        payload={"series": series, "log_x": True},
+    )
+
+
+def _share_by_pdg_entry(
+    checkpoint: str | Path | None, r_phys: pl.LazyFrame, t_phys: pl.LazyFrame, top_pdgs: list[int], seed: int
+) -> dict | None:
+    """One rollout's ``router_share_by_pdg`` panel-pair data, or ``None`` if not a MoE checkpoint."""
+    handle = load_router(checkpoint) if checkpoint else None
+    if handle is None:
+        return None
     labels = [pdg_label(p) for p in top_pdgs]
     sides: dict[str, dict] = {}
     for name, lf in (("rollout", r_phys), ("reference", t_phys)):
@@ -273,41 +327,36 @@ def compute_router_share_by_pdg(
         else:
             shares = {str(p): [0.0] * handle.router.n_experts for p in top_pdgs}
         sides[name] = {labels[i]: shares[str(p)] for i, p in enumerate(top_pdgs)}
+    return {"router_type": handle.router_type, "n_experts": handle.router.n_experts, "categories": labels, **sides}
 
+
+def compute_router_share_by_pdg(
+    rollouts: dict[str, "RolloutSide"], t_phys: pl.LazyFrame, top_pdgs: list[int], seed: int = 0
+) -> Reduced:
+    """`Reduced` for the router expert-share-by-species figure, one panel-pair
+    per rollout with an enabled MoE router."""
+    series = {}
+    for name, rs in rollouts.items():
+        entry = _share_by_pdg_entry(rs.checkpoint, rs.phys, t_phys, top_pdgs, seed)
+        if entry is not None:
+            series[name] = entry
+    if not series:
+        return _unavailable("router_share_by_pdg")
     return Reduced(
         id="router_share_by_pdg",
         family="model",
         kind="router_share",
         title=_TITLES["router_share_by_pdg"],
         xlabel="particle species",
-        payload={
-            "router_type": handle.router_type,
-            "n_experts": handle.router.n_experts,
-            "categories": labels,
-            **sides,
-        },
+        payload={"series": series},
     )
 
 
-def compute_router_share_by_process(
-    checkpoint: str | Path | None,
-    t_phys: pl.LazyFrame,
-    seed: int = 0,
-    top_k: int = _TOP_K_PROCESS,
-) -> Reduced:
-    """Stacked-bar share of each physics process dispatched to each expert.
-
-    Reference-only: ``process`` is the true post-step physics process — a
-    label the rollout side has no equivalent of (see
-    `giant.model.network.ProcessRouter`, which predicts it from pre-step
-    conditioning alone, never observes it at eval time). This plot instead
-    checks *after the fact*, on real data, how well the router's conditioning
-    -based dispatch lines up with the true process.
-    """
+def _share_by_process_entry(checkpoint: str | Path | None, t_phys: pl.LazyFrame, seed: int, top_k: int) -> dict | None:
+    """One rollout checkpoint's ``router_share_by_process`` panel data (reference-only), or ``None`` if not MoE."""
     handle = load_router(checkpoint) if checkpoint else None
     if handle is None:
-        return _unavailable("router_share_by_process")
-
+        return None
     df = _subsample(t_phys, _SAMPLE_ROWS, seed, extra_cols=("process",))
     df, gate = _gate_for_df(handle, df)
     if len(df):
@@ -317,17 +366,39 @@ def compute_router_share_by_process(
         shares = _top1_shares(df["process"].to_numpy(), idx, order, handle.router.n_experts)
     else:
         order, shares = [], {}
+    return {
+        "router_type": handle.router_type,
+        "n_experts": handle.router.n_experts,
+        "categories": order,
+        "reference": {p: shares[p] for p in order},
+    }
 
+
+def compute_router_share_by_process(
+    rollouts: dict[str, "RolloutSide"], t_phys: pl.LazyFrame, seed: int = 0, top_k: int = _TOP_K_PROCESS
+) -> Reduced:
+    """Stacked-bar share of each physics process dispatched to each expert, one
+    panel per rollout checkpoint with an enabled MoE router.
+
+    Reference-only: ``process`` is the true post-step physics process — a
+    label the rollout side has no equivalent of (see
+    `giant.model.network.ProcessRouter`, which predicts it from pre-step
+    conditioning alone, never observes it at eval time). This plot instead
+    checks *after the fact*, on real data, how well each checkpoint's router
+    -based dispatch lines up with the true process.
+    """
+    series = {}
+    for name, rs in rollouts.items():
+        entry = _share_by_process_entry(rs.checkpoint, t_phys, seed, top_k)
+        if entry is not None:
+            series[name] = entry
+    if not series:
+        return _unavailable("router_share_by_process")
     return Reduced(
         id="router_share_by_process",
         family="model",
         kind="router_share",
         title=_TITLES["router_share_by_process"],
         xlabel="physics process",
-        payload={
-            "router_type": handle.router_type,
-            "n_experts": handle.router.n_experts,
-            "categories": order,
-            "reference": {p: shares[p] for p in order},
-        },
+        payload={"series": series},
     )

giant/analysis/run.py

@@ -1,4 +1,6 @@
-"""HTCondor orchestration driven by a ``giant rollout`` YAML sidecar.
+"""Analysis run directories: prep, per-(plot, chunk) compute, and merge.
+
+Driven by one or more ``giant rollout`` YAML sidecars.
 
 A rollout writes a YAML sidecar (``giant/cli.py:_write_prediction_ref`` +
 rollout extras) that already names both files we need and carries the run's
@@ -10,8 +12,11 @@ provenance:
 * ``checkpoint``, ``geometry_oracle``, ``energy_cutoff``, ``steps``, ... —
   metadata that flows straight into every plot's gallery ``metadata.yaml``.
 
-So the analysis takes that one YAML as input, derives its own **run directory**
-next to the rollout parquet, and lays everything out under it:
+The analysis takes N such YAMLs — one series per rollout, all required to
+share the same ``dataset`` (the premise is "N candidates vs one ground
+truth") — resolves each one's series name (``load_rollout_yamls``), derives
+its own **run directory** next to the first rollout's parquet, and lays
+everything out under it:
 
     <run_dir>/shared.json     fixed bin edges / group sets (prep)
     <run_dir>/run_meta.json   resolved rollout/reference paths + plot metadata
@@ -19,13 +24,16 @@ next to the rollout parquet, and lays everything out under it:
     <run_dir>/reduced/<id>.json   merged, per plot
     <run_dir>/plots/<family>/<id>.pdf   rendered locally
 
-Job model (one condor job per (plot, chunk), compute/merge/render split):
+Job model (one job per (plot, chunk), compute/merge/render split). Job
+submission itself is b2luigi's (``giant/workflow/tasks.py`` — ``AnalysisPrepTask``
+/ ``AnalysisComputeTask`` / ``AnalysisRenderTask``); this module only provides
+the three steps they call:
 
-1. ``prep`` runs once on the submit node — reads the YAML, resolves the shared
+1. ``prep`` runs once locally — reads the YAML, resolves the shared
    context from a subsample, writes ``shared.json`` + ``run_meta.json``
    (including the run's configured ``n_chunks``).
 2. one job per catalog id x chunk index runs ``giant analyze compute-one
-   --run-dir`` on a worker — a single streaming pass over that
+   --run-dir`` (or ``compute_one`` in-process) on a worker — a single streaming pass over that
    ``event_id``-disjoint chunk, writing ``reduced_partial/<id>__<chunk>.json``
    (polars/numpy only, no LaTeX). Specs marked ``chunkable=False``
    (``PlotSpec``, ``catalog.py``) always run as a single chunk.
@@ -35,15 +43,16 @@ Job model (one condor job per (plot, chunk), compute/merge/render split):
    ``reduced/<id>.json``, then renders those into the styled PDF + gallery tree
    (that step imports plotstyle/LaTeX).
 
-Files on ``/ceph`` or ``/work`` are reached via ``ProvidesETPResources``; no
-HTCondor file transfer of the multi-GB inputs.
+Files on ``/ceph`` or ``/work`` are reached directly (see
+``giant/workflow/htcondor.py``); no HTCondor file transfer of the multi-GB
+inputs.
 """
 
 from __future__ import annotations
 
 import json
 import shutil
-import sys
+from collections.abc import Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -53,8 +62,7 @@ import yaml
 from giant.analysis.catalog import Bundle, catalog_ids, get_spec
 from giant.analysis.context import Context, build_context
 from giant.analysis.reduced import Partial
-from giant.analysis.runtime_estimate import estimate_runtime_s
-from giant.analysis.sources import Side, open_side
+from giant.analysis.sources import RolloutSpec, Side, open_side
 
 # Keys copied verbatim from a rollout YAML into each plot's gallery metadata.
 _PLOT_META_KEYS = (
@@ -111,8 +119,60 @@ def load_rollout_yaml(path: str | Path) -> dict:
     return d
 
 
+@dataclass
+class LoadedRollout:
+    """One rollout YAML plus its resolved series ``name`` (see ``load_rollout_yamls``)."""
+
+    name: str
+    yaml: dict
+
+
+def load_rollout_yamls(
+    paths: Sequence[str | Path], labels: Sequence[str] | None = None
+) -> tuple[list[LoadedRollout], str]:
+    """Load every rollout YAML, resolve each one's series name, and verify they
+    all share one reference (``dataset``) file — the premise is "N candidates
+    vs one ground truth", not N independent comparisons.
+
+    Names: an explicit ``labels[i]`` if given (``labels`` must be empty or
+    exactly ``len(paths)`` long); otherwise the YAML's stem for N>1, or
+    ``"rollout"`` for the single-YAML case — matching today's one-series
+    legend/payload key, so a single-rollout run renders identically to
+    before this feature existed. Raises ``ValueError`` if two rollouts
+    resolve to the same name, or if the YAMLs don't all name the same
+    ``dataset``.
+    """
+    if labels and len(labels) != len(paths):
+        raise ValueError(f"--label given {len(labels)} time(s) but {len(paths)} rollout YAML(s) were passed")
+    yamls = [load_rollout_yaml(p) for p in paths]
+    if labels:
+        names = list(labels)
+    elif len(paths) == 1:
+        names = ["rollout"]
+    else:
+        names = [Path(p).stem for p in paths]
+    if len(set(names)) != len(names):
+        dupes = sorted({n for n in names if names.count(n) > 1})
+        raise ValueError(f"rollout series names collide: {dupes} — pass --label to disambiguate")
+
+    references = {str(y["dataset"]) for y in yamls}
+    if len(references) > 1:
+        detail = "\n".join(f"  {p}: dataset={y['dataset']!r}" for p, y in zip(paths, yamls))
+        raise ValueError(
+            "all rollout YAMLs must be seeded from the same reference (dataset) "
+            f"file — got {len(references)} distinct ones:\n{detail}"
+        )
+
+    return [LoadedRollout(name=n, yaml=y) for n, y in zip(names, yamls)], yamls[0]["dataset"]
+
+
+def _run_tag(y: dict) -> str:
+    rollout = Path(y["output"])
+    return str(y.get("prediction_id") or rollout.stem)[:8]
+
+
 def derive_run_dir(
-    rollout_yaml: dict,
+    rollout_yamls: list[dict],
     run_dir: str | Path | None = None,
     default_base: str | Path | None = None,
 ) -> Path:
@@ -122,14 +182,24 @@ def derive_run_dir(
     ``default_base / analysis_<tag>`` if ``default_base`` is given (the CLI
     passes the repo's gitignored ``analysis_runs/``, so run directories don't
     pile up on ``/ceph`` next to the rollout parquet). Falls back to next to
-    the rollout parquet — the original convention — for callers that don't
-    care where the run directory lives.
+    the *first* rollout's parquet — the original convention — for callers
+    that don't care where the run directory lives.
+
+    ``tag`` is a single rollout's ``prediction_id``/output stem (matching
+    today's single-rollout convention exactly) when there's only one; for
+    N>1 it joins up to three tags with ``-``, then ``-plus<K>`` for any
+    beyond that, so a many-rollout run still gets a short, stable directory
+    name.
     """
     if run_dir is not None:
         return Path(run_dir)
-    rollout = Path(rollout_yaml["output"])
-    tag = str(rollout_yaml.get("prediction_id") or rollout.stem)[:8]
-    base = Path(default_base) if default_base is not None else rollout.parent
+    tags = [_run_tag(y) for y in rollout_yamls]
+    if len(tags) == 1:
+        tag = tags[0]
+    else:
+        shown, rest = tags[:3], tags[3:]
+        tag = "-".join(shown) + (f"-plus{len(rest)}" if rest else "")
+    base = Path(default_base) if default_base is not None else Path(rollout_yamls[0]["output"]).parent
     return base / f"analysis_{tag}"
 
 
@@ -139,17 +209,21 @@ def _plot_meta(rollout_yaml: dict) -> dict:
 
 @dataclass
 class RunMeta:
-    """Resolved paths + plot metadata for one analysis run (``run_meta.json``)."""
+    """Resolved paths + plot metadata for one analysis run (``run_meta.json``).
 
-    rollout: str
+    ``rollouts`` is ``[{"name", "path", "plot_meta"}, ...]``, insertion order
+    = the order rollouts were given on the CLI (and so the order every
+    ``Reduced.payload["series"]`` dict is built in — see ``catalog.py``).
+    """
+
+    rollouts: list[dict]
     reference: str
     run_dir: str
     title: str
-    plot_meta: dict
     n_chunks: int = 1
-    # rollout+reference row count of each event_id-disjoint chunk, and the
-    # dataset total — inputs to `runtime_estimate.estimate_runtime_s`. Empty/0
-    # on run directories written before this field existed.
+    # combined rollout+reference row count of each event_id-disjoint chunk,
+    # and the dataset total — inputs to `runtime_estimate.estimate_runtime_s`.
+    # Empty/0 on run directories written before this field existed.
     rows_per_chunk: list[int] = field(default_factory=list)
     total_rows: int = 0
 
@@ -161,8 +235,8 @@ class RunMeta:
         return cls(**json.loads(Path(path).read_text()))
 
 
-def _rows_per_chunk(rollout: str | Path, reference: str | Path, n_chunks: int) -> list[int]:
-    """Rollout+reference row count of each ``event_id % n_chunks`` chunk.
+def _rows_per_chunk(rollouts: list[str | Path], reference: str | Path, n_chunks: int) -> list[int]:
+    """Combined rollout+reference row count of each ``event_id % n_chunks`` chunk.
 
     One cheap streaming ``group_by`` per side (just the ``event_id`` column) —
     the sizing input every job's estimated walltime
@@ -178,7 +252,8 @@ def _rows_per_chunk(rollout: str | Path, reference: str | Path, n_chunks: int) -
         )
 
     out = [0] * n_chunks
-    for lf in (open_side(rollout, Side.rollout), open_side(reference, Side.reference)):
+    sides = [open_side(reference, Side.reference)] + [open_side(r, Side.rollout) for r in rollouts]
+    for lf in sides:
         df = counts(lf)
         for c, n in zip(df["_c"].to_list(), df["n"].to_list()):
             out[c] += n
@@ -186,20 +261,22 @@ def _rows_per_chunk(rollout: str | Path, reference: str | Path, n_chunks: int) -
 
 
 def prep(
-    rollout_yaml: str | Path,
+    rollout_yamls: Sequence[str | Path],
     run_dir: str | Path | None = None,
     n_chunks: int = 1,
     default_base: str | Path | None = None,
+    labels: Sequence[str] | None = None,
     **ctx_kwargs,
 ) -> Path:
-    """Read the rollout YAML, build the shared context, and lay out the run dir.
+    """Read the rollout YAML(s), build the shared context, and lay out the run dir.
 
     Writes ``shared.json`` + ``run_meta.json`` and returns the run directory.
     ``n_chunks`` is the run-level chunk count every ``compute-one``/``merge-one``
     job reads back out of ``run_meta.json`` (via ``RunMeta.n_chunks``), so it is
     resolved once here rather than re-passed (and risking disagreement) at every
     later step. See ``derive_run_dir`` for how ``run_dir``/``default_base``
-    resolve the actual directory.
+    resolve the actual directory, and ``load_rollout_yamls`` for how
+    ``labels``/YAML stems resolve each rollout's series name.
 
     Clears any existing ``reduced_partial/``/``reduced/`` from a prior prep of
     this same ``run_dir``: partial files carry no record of what context
@@ -208,8 +285,8 @@ def prep(
     rollout/reference files changed) would otherwise let ``merge_one`` silently
     merge stale partials against the new ``shared.json``.
     """
-    y = load_rollout_yaml(rollout_yaml)
-    run_path = derive_run_dir(y, run_dir, default_base=default_base)
+    loaded, reference = load_rollout_yamls(list(rollout_yamls), labels)
+    run_path = derive_run_dir([lr.yaml for lr in loaded], run_dir, default_base=default_base)
     run_path.mkdir(parents=True, exist_ok=True)
 
     for stale in ("reduced_partial", "reduced"):
@@ -217,19 +294,22 @@ def prep(
         if stale_dir.exists():
             shutil.rmtree(stale_dir)
 
-    rollout, reference = y["output"], y["dataset"]
-    ctx = build_context(rollout, reference, **ctx_kwargs)
+    rollout_specs = [RolloutSpec(name=lr.name, source=lr.yaml["output"]) for lr in loaded]
+    ctx = build_context(rollout_specs, reference, **ctx_kwargs)
     ctx.save(run_path / "shared.json")
 
-    rows_per_chunk = _rows_per_chunk(rollout, reference, n_chunks)
+    rows_per_chunk = _rows_per_chunk([lr.yaml["output"] for lr in loaded], reference, n_chunks)
+
+    rollouts_meta = [
+        {"name": lr.name, "path": str(lr.yaml["output"]), "plot_meta": _plot_meta(lr.yaml)} for lr in loaded
+    ]
+    ckpts = ", ".join(Path(lr.yaml.get("checkpoint", "")).name or "rollout" for lr in loaded)
 
-    ckpt = Path(y.get("checkpoint", "")).name or "rollout"
     RunMeta(
-        rollout=str(rollout),
+        rollouts=rollouts_meta,
         reference=str(reference),
         run_dir=str(run_path),
-        title=f"GIANT rollout analysis — {ckpt}",
-        plot_meta=_plot_meta(y),
+        title=f"GIANT rollout analysis — {ckpts}",
         n_chunks=n_chunks,
         rows_per_chunk=rows_per_chunk,
         total_rows=sum(rows_per_chunk),
@@ -244,17 +324,19 @@ def prep(
 
 def compute_reduced(
     spec_id: str,
-    rollout: str | Path,
+    rollouts: list[dict],
     reference: str | Path,
     shared: str | Path,
     out: str | Path,
-    checkpoint: str | None = None,
     chunk_index: int = 0,
     n_chunks: int = 1,
-    type_embedding_l1_dist: dict | None = None,
 ) -> Path:
     """Core: run one (plot, chunk)'s partial reduction against explicit paths.
 
+    ``rollouts``: ``[{"name", "path", "checkpoint"?, "type_embedding_l1_dist"?},
+    ...]``, one per rollout series (insertion order preserved through to every
+    plot's ``Reduced.payload["series"]``).
+
     Writes a ``Partial`` JSON — the raw, not-yet-merged output of
     ``PlotSpec.compute_partial`` — never a finished ``Reduced``; ``merge_one``
     is what combines every chunk's ``Partial`` for a plot into the final
@@ -268,14 +350,16 @@ def compute_reduced(
         raise ValueError(
             f"{spec_id}: chunk_index={chunk_index} out of range for n_chunks={effective_n} (chunkable={spec.chunkable})"
         )
-    bundle = Bundle.open(
-        rollout,
-        reference,
-        ctx,
-        checkpoint=checkpoint,
-        chunk=(chunk_index, effective_n),
-        type_embedding_l1_dist=type_embedding_l1_dist,
-    )
+    rollout_specs = [
+        RolloutSpec(
+            name=r["name"],
+            source=r["path"],
+            checkpoint=r.get("checkpoint"),
+            type_embedding_l1_dist=r.get("type_embedding_l1_dist"),
+        )
+        for r in rollouts
+    ]
+    bundle = Bundle.open(rollout_specs, reference, ctx, chunk=(chunk_index, effective_n))
     partial = Partial(
         id=spec_id,
         family=spec.family,
@@ -291,16 +375,23 @@ def compute_one(spec_id: str, run_dir: str | Path, chunk_index: int = 0) -> Path
     """Run one (plot, chunk)'s partial reduction from a prepped run directory."""
     run_path = Path(run_dir)
     meta = RunMeta.load(run_path / "run_meta.json")
+    rollouts = [
+        {
+            "name": ro["name"],
+            "path": ro["path"],
+            "checkpoint": ro["plot_meta"].get("checkpoint"),
+            "type_embedding_l1_dist": ro["plot_meta"].get("type_embedding_l1_dist"),
+        }
+        for ro in meta.rollouts
+    ]
     return compute_reduced(
         spec_id,
-        meta.rollout,
+        rollouts,
         meta.reference,
         run_path / "shared.json",
         run_path / "reduced_partial" / f"{spec_id}__{chunk_index}.json",
-        checkpoint=meta.plot_meta.get("checkpoint"),
         chunk_index=chunk_index,
         n_chunks=meta.n_chunks,
-        type_embedding_l1_dist=meta.plot_meta.get("type_embedding_l1_dist"),
     )
 
 
@@ -342,136 +433,3 @@ def merge_one(spec_id: str, run_dir: str | Path) -> Path:
 def merge_all(run_dir: str | Path) -> list[Path]:
     """Merge every catalog plot's chunk partials into ``reduced/<id>.json``."""
     return [merge_one(spec_id, run_dir) for spec_id in catalog_ids()]
-
-
-# ---------------------------------------------------------------------------
-# submit description
-# ---------------------------------------------------------------------------
-
-
-@dataclass
-class SubmitConfig:
-    run_dir: Path
-    accounting_group: str
-    repo_dir: Path
-    docker_image: str = "cverstege/alma9-gridjob"
-    request_memory_mb: int = 8192
-    request_cpus: int = 1
-    remote: bool = False  # +RemoteJob (grid I/O) vs ProvidesETPResources (local files)
-    n_chunks: int = 1  # per-plot data chunks; ignored for chunkable=False specs
-
-
-_WRAPPER = """#!/bin/bash
-set -euo pipefail
-cd {repo_dir}
-exec {giant_exe} analyze compute-one --id "$1" --chunk "$2" --run-dir {run_dir}
-"""
-
-
-def _submit_description(cfg: SubmitConfig, wrapper: Path, jobs_file: Path) -> str:
-    reqs_attrs = "+RemoteJob = True\n" if cfg.remote else "requirements = TARGET.ProvidesETPResources\n"
-    return (
-        "universe = docker\n"
-        f"docker_image = {cfg.docker_image}\n"
-        f"executable = {wrapper}\n"
-        "arguments = $(plotid) $(chunk)\n"
-        "should_transfer_files = YES\n"
-        "when_to_transfer_output = ON_EXIT\n"
-        f"request_memory = {cfg.request_memory_mb}\n"
-        f"request_cpus = {cfg.request_cpus}\n"
-        "+RequestWalltime = $(walltime)\n"
-        f"accounting_group = {cfg.accounting_group}\n"
-        f"{reqs_attrs}"
-        f"output = {cfg.run_dir}/logs/$(plotid)__$(chunk).out\n"
-        f"error = {cfg.run_dir}/logs/$(plotid)__$(chunk).err\n"
-        f"log = {cfg.run_dir}/logs/condor.log\n"
-        f"queue plotid,chunk,walltime from {jobs_file}\n"
-    )
-
-
-def _job_walltimes(run_dir: Path, ids: list[str], n_chunks: int) -> list[tuple[str, int, int]]:
-    """``(spec_id, chunk, walltime_s)`` for every job, sized from ``run_meta.json``.
-
-    Row counts come from ``prep``'s ``RunMeta.rows_per_chunk``/``total_rows``;
-    ``chunkable=False`` specs (router diagnostics) always use the dataset
-    total since they run as a single job regardless of ``n_chunks``.
-    """
-    meta = RunMeta.load(run_dir / "run_meta.json")
-    jobs: list[tuple[str, int, int]] = []
-    for spec_id in ids:
-        chunkable = get_spec(spec_id).chunkable
-        chunks = range(n_chunks) if chunkable else [0]
-        for chunk in chunks:
-            n_rows = meta.rows_per_chunk[chunk] if chunkable else meta.total_rows
-            jobs.append((spec_id, chunk, estimate_runtime_s(spec_id, n_rows)))
-    return jobs
-
-
-def _resolve_giant_executable(repo_dir: Path) -> Path:
-    """Path to the ``giant`` entry point to bake into the condor wrapper script.
-
-    Prefers the venv currently running this process (``sys.executable``'s
-    sibling ``giant``) so a submit from a non-default venv (e.g. ``--extra
-    cuda`` on a dev box) doesn't silently pick up a different one; falls back
-    to ``repo_dir/.venv/bin/giant`` for the case this is invoked from outside
-    any venv (e.g. a system Python).
-    """
-    active = Path(sys.executable).parent / "giant"
-    if active.exists():
-        return active
-    venv_giant = repo_dir / ".venv" / "bin" / "giant"
-    if not venv_giant.exists():
-        raise FileNotFoundError(
-            f"no `giant` executable found next to {sys.executable} or at "
-            f"{venv_giant} — condor jobs run it directly (no `uv` on the "
-            f"worker image), so run `uv sync --extra cpu` in {repo_dir} "
-            "before submitting."
-        )
-    return venv_giant
-
-
-def write_submit(cfg: SubmitConfig, ids: list[str] | None = None) -> Path:
-    """Write the wrapper script, (plot, chunk) job list, and HTCondor submit
-    description.
-
-    Each catalog id gets ``cfg.n_chunks`` jobs, except ``chunkable=False``
-    specs (the router diagnostics), which always get exactly one regardless of
-    ``cfg.n_chunks``. Every job's ``+RequestWalltime`` is estimated from its
-    chunk's row count (``runtime_estimate.estimate_runtime_s``, requires
-    ``run_meta.json`` from ``prep`` to already carry ``rows_per_chunk``).
-    Returns the submit description path (``<run_dir>/analyze.sub``). Does not
-    submit — call ``condor_submit`` on the returned file.
-
-    ``cfg.n_chunks`` and the run directory's own ``RunMeta.n_chunks`` (fixed by
-    ``prep``, and what ``RunMeta.rows_per_chunk`` was sized against) are two
-    independent values — checked equal up front so a mismatch is a clear error
-    here rather than an ``IndexError`` out of ``_job_walltimes``.
-    """
-    giant_exe = _resolve_giant_executable(cfg.repo_dir)
-
-    ids = ids or catalog_ids()
-    run_dir = cfg.run_dir
-    meta = RunMeta.load(run_dir / "run_meta.json")
-    if cfg.n_chunks != meta.n_chunks:
-        raise ValueError(
-            f"SubmitConfig.n_chunks={cfg.n_chunks} does not match the "
-            f"n_chunks this run directory was prepped with "
-            f"(RunMeta.n_chunks={meta.n_chunks} in {run_dir}/run_meta.json) — "
-            "re-run `prep` with the desired n_chunks, or fix cfg.n_chunks to "
-            "match it."
-        )
-    (run_dir / "logs").mkdir(parents=True, exist_ok=True)
-    (run_dir / "reduced").mkdir(parents=True, exist_ok=True)
-    (run_dir / "reduced_partial").mkdir(parents=True, exist_ok=True)
-
-    wrapper = run_dir / "run_compute.sh"
-    wrapper.write_text(_WRAPPER.format(repo_dir=cfg.repo_dir, giant_exe=giant_exe, run_dir=run_dir))
-    wrapper.chmod(0o755)
-
-    jobs = _job_walltimes(run_dir, ids, cfg.n_chunks)
-    jobs_file = run_dir / "jobs.txt"
-    jobs_file.write_text("\n".join(f"{i},{k},{w}" for i, k, w in jobs) + "\n")
-
-    sub = run_dir / "analyze.sub"
-    sub.write_text(_submit_description(cfg, wrapper, jobs_file))
-    return sub

giant/analysis/runtime_estimate.py

@@ -1,4 +1,4 @@
-"""Per-(plot, chunk) HTCondor walltime estimates for `giant analyze submit`.
+"""Per-(plot, chunk) HTCondor walltime estimates for the analysis compute jobs.
 
 Each catalog spec's compute cost is close to linear in the number of input
 rows a `compute-one` job streams over — every spec is one (or a couple of)
@@ -6,8 +6,8 @@ streaming `group_by` pass(es) over the chunk (see `catalog.py`/`reduce.py`).
 `_COST_MODEL` below is ``spec_id -> (intercept_s, seconds_per_row)``.
 ``n_rows`` is the combined rollout+reference row count of the job's input:
 the chunk's row count for `chunkable=True` specs, the whole dataset's for the
-three `chunkable=False` router specs (they always run as a single job
-regardless of chunk count).
+`chunkable=False` router specs in `_ROUTER_IDS` (they always run as a single
+job regardless of chunk count).
 
 Calibrated 2026-07-27 from real HTCondor timings (`condor_history`
 ``RemoteWallClockTime``) of a production run: prediction ``563f5ee3``
@@ -54,7 +54,7 @@ _FIXED_OVERHEAD_S = 60.0
 # scan. Calibrated from the 3 real router jobs' observed wall times (119, 66,
 # 124s) — max minus _FIXED_OVERHEAD_S, on top of it.
 _ROUTER_FIXED_S = 64.0
-_ROUTER_IDS = frozenset({"router_gating", "router_share_by_pdg", "router_share_by_process"})
+_ROUTER_IDS = frozenset({"router_gating", "router_share_by_pdg", "router_share_by_process", "router_specialization"})
 
 # Conservative fallback for any catalog id not in _COST_MODEL (e.g. a plot
 # added after the last calibration run) — the most expensive fitted per-row

giant/analysis/sources.py

@@ -1,7 +1,12 @@
-"""Canonical world-frame LazyFrame builders for the two sides of a comparison.
+"""Canonical world-frame LazyFrame builders for the two kinds of comparison input.
 
-The analysis compares one autoregressive ``giant rollout`` (the *generated* side)
-against a raw miniCaloSim steps file (the *reference* / real side). Both carry a
+The analysis compares one or more autoregressive ``giant rollout`` runs (the
+*generated* side — one named series each, see ``RolloutSpec``) against a single
+raw miniCaloSim steps file shared by all of them (the *reference* / real side).
+Every rollout is the same *kind* of file regardless of how many there are, so
+``Side`` stays binary: it describes a file's schema (rollout column layout +
+synthetic-termination rows + per-track secondary view, vs. reference
+``sec_*_list`` columns), not series identity. Both kinds carry a
 **shared world-frame physical column subset** under identical names, so no
 renaming or coordinate decode is needed — everything is already in world-frame
 mm / MeV:
@@ -26,6 +31,7 @@ HTCondor workers that have no LaTeX toolchain.
 
 from __future__ import annotations
 
+from dataclasses import dataclass
 from enum import Enum
 from pathlib import Path
 
@@ -82,12 +88,44 @@ SYNTHETIC_TERMINATION_REASONS: frozenset[str] = frozenset(
 
 
 class Side(str, Enum):
-    """Which of the two comparison inputs a file is."""
+    """Which of the two comparison-input *kinds* a file is."""
 
     rollout = "rollout"
     reference = "reference"
 
 
+@dataclass
+class RolloutSpec:
+    """One named rollout input, as fed to ``build_context``/``Bundle.open``.
+
+    ``name`` is the series' identity throughout the rest of the pipeline (a
+    plot's ``payload["series"]`` key, a figure's legend label, its color) —
+    resolved once in ``condor.load_rollout_yamls`` from ``--label`` or the
+    YAML stem, then threaded through unchanged. ``checkpoint`` /
+    ``type_embedding_l1_dist`` are only used by the router/type-embedding
+    diagnostics (``catalog.py``'s ``chunkable=False`` specs).
+    """
+
+    name: str
+    source: str | Path | pl.LazyFrame
+    checkpoint: str | None = None
+    type_embedding_l1_dist: dict | None = None
+
+
+@dataclass
+class RolloutSide:
+    """One rollout's opened frames + per-checkpoint diagnostic inputs (``catalog.Bundle.rollouts`` value)."""
+
+    all: pl.LazyFrame  # rollout, all rows (incl. synthetic termination rows)
+    phys: pl.LazyFrame  # rollout, physical steps only
+    checkpoint: str | None = None  # from the rollout YAML; router_gating only
+    # Diagnostic pre-aggregated at rollout time (giant.rollout.
+    # L1DistCollector.summary()) — from the rollout YAML, type_embedding_l1_distance
+    # only. Unlike checkpoint, this needs no live model: it's already a
+    # finished histogram, just passed through.
+    type_embedding_l1_dist: dict | None = None
+
+
 def _check_rollout_metadata(path: Path) -> None:
     """Raise if ``path`` carries coord metadata that isn't the rollout tag.
 

giant/analysis/type_embedding_distance.py

@@ -20,26 +20,37 @@ redesign exists to fix.
 
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from giant.analysis.reduced import Reduced
 
+if TYPE_CHECKING:
+    from giant.analysis.sources import RolloutSide
+
 _NOTE_NOT_APPLICABLE = (
-    "not applicable: this rollout's checkpoint doesn't use "
+    "not applicable: none of these rollouts' checkpoints use "
     "stage2_model.particle_type.target='embedding' (or generated no "
     "secondaries), so giant rollout recorded no type_embedding_l1_dist "
-    "diagnostic in its YAML sidecar"
+    "diagnostic in their YAML sidecar"
 )
 
 
-def compute_type_embedding_l1_distance(l1_dist: dict | None) -> Reduced:
-    """`Reduced` for the type-embedding-distance figure, or an explanatory
-    note if this checkpoint never populated the diagnostic.
+def compute_type_embedding_l1_distance(rollouts: dict[str, "RolloutSide"]) -> Reduced:
+    """`Reduced` for the type-embedding-distance figure: one series per rollout
+    whose checkpoint populated the diagnostic, or an explanatory note if none did.
 
-    `l1_dist`: `giant.rollout.L1DistCollector.summary()`'s dict, as recorded
-    in the rollout YAML's `type_embedding_l1_dist` key (`Bundle.
-    type_embedding_l1_dist`) — `{"n", "mean", "std", "min", "max",
-    "hist_edges", "hist_counts"}`.
+    Each rollout's `RolloutSide.type_embedding_l1_dist` is
+    `giant.rollout.L1DistCollector.summary()`'s dict, as recorded in that
+    rollout's YAML `type_embedding_l1_dist` key — `{"n", "mean", "std",
+    "min", "max", "hist_edges", "hist_counts"}`. Every collector uses the
+    same fixed log-spaced edges (`L1DistCollector.__init__`'s defaults, never
+    overridden — see `giant/cli.py`'s rollout command), so it's safe to plot
+    every rollout's counts against the first one's edges.
     """
-    if l1_dist is None:
+    entries = {
+        name: rs.type_embedding_l1_dist for name, rs in rollouts.items() if rs.type_embedding_l1_dist is not None
+    }
+    if not entries:
         return Reduced(
             id="type_embedding_l1_distance",
             family="model",
@@ -49,6 +60,11 @@ def compute_type_embedding_l1_distance(l1_dist: dict | None) -> Reduced:
             payload={"note": _NOTE_NOT_APPLICABLE},
         )
 
+    edges = next(iter(entries.values()))["hist_edges"]
+    notes = [
+        f"{name}: n={d['n']:,} mean={d['mean']:.4g} std={d['std']:.4g} min={d['min']:.4g} max={d['max']:.4g}"
+        for name, d in entries.items()
+    ]
     return Reduced(
         id="type_embedding_l1_distance",
         family="model",
@@ -56,15 +72,10 @@ def compute_type_embedding_l1_distance(l1_dist: dict | None) -> Reduced:
         title="Secondary-type embedding L1 distance (predicted vector -> nearest PDG row)",
         xlabel="L1 distance",
         payload={
-            "edges": l1_dist["hist_edges"],
-            "rollout": l1_dist["hist_counts"],
+            "edges": edges,
+            "series": {name: d["hist_counts"] for name, d in entries.items()},
             "log_y": True,
             "log_x": True,
-            "note": (
-                f"n={l1_dist['n']:,}  mean={l1_dist['mean']:.4g}  "
-                f"std={l1_dist['std']:.4g}  min={l1_dist['min']:.4g}  "
-                f"max={l1_dist['max']:.4g}; rollout only, no reference "
-                "concept for a raw pre-decode vector"
-            ),
+            "note": "; ".join(notes) + "; rollout only, no reference concept for a raw pre-decode vector",
         },
     )

giant/cli.py

@@ -4,6 +4,7 @@ from enum import Enum
 import math
 from pathlib import Path
 import re
+import sys
 from typing import Optional
 import uuid as uuid_mod
 
@@ -191,8 +192,17 @@ def _write_prediction_ref(
     out: Path,
     dataset_path: Path,
     comment: str | None = None,
+    explicit_out: bool = False,
 ) -> Path:
-    """Write a YAML sidecar in the checkpoint directory and return its path."""
+    """Write the YAML sidecar and return its path.
+
+    With an explicit ``--out`` the sidecar sits next to the output file as
+    ``out.with_suffix(".yaml")`` — a *deterministic* path, which is what lets
+    a workflow task (``giant/workflow/tasks.py``) declare it as a target.
+    Without one, the historic uuid-named file under the checkpoint directory
+    is kept, so ad-hoc runs and the ``/ceph`` predictions convention are
+    unaffected.
+    """
     ref = {
         "prediction_id": pred_uuid,
         "output": str(out),
@@ -202,7 +212,7 @@ def _write_prediction_ref(
     }
     if comment is not None:
         ref["comment"] = comment
-    ref_path = checkpoint.parent / f"{pred_uuid}.yaml"
+    ref_path = out.with_suffix(".yaml") if explicit_out else checkpoint.parent / f"{pred_uuid}.yaml"
     ref_path.write_text(yaml.dump(ref, default_flow_style=False, sort_keys=False))
     return ref_path
 
@@ -1076,6 +1086,7 @@ def predict(
     bs = batch_size_value
 
     # --- Output path ---
+    explicit_out = out is not None
     out, dataset_path, pred_uuid = _resolve_prediction_output(data, out)
     out.parent.mkdir(parents=True, exist_ok=True)
     typer.echo(f"output: {out}")
@@ -1295,7 +1306,7 @@ def predict(
     if writer is not None:
         writer.close()
 
-    ref_path = _write_prediction_ref(checkpoint, pred_uuid, out, dataset_path, comment)
+    ref_path = _write_prediction_ref(checkpoint, pred_uuid, out, dataset_path, comment, explicit_out=explicit_out)
     typer.echo(f"reference: {ref_path}")
 
     if skipped:
@@ -1443,6 +1454,7 @@ def rollout(
     seeds = _seed_from_data(files, n_events)
     typer.echo(f"seeded {len(seeds['event_id']):,} shower(s)")
 
+    explicit_out = out is not None
     out, dataset_path, pred_uuid = _resolve_prediction_output(data, out)
     out.parent.mkdir(parents=True, exist_ok=True)
 
@@ -1503,7 +1515,7 @@ def rollout(
 
     l1_summary = l1_dist_collector.summary()
 
-    ref_path = _write_prediction_ref(checkpoint, pred_uuid, out, dataset_path)
+    ref_path = _write_prediction_ref(checkpoint, pred_uuid, out, dataset_path, explicit_out=explicit_out)
     ref = yaml.safe_load(ref_path.read_text())
     ref.update(
         {
@@ -1547,6 +1559,49 @@ def rollout(
     typer.echo(f"reference: {ref_path}")
 
 
+workflow_app = typer.Typer(
+    no_args_is_help=True,
+    help="b2luigi pipeline orchestration: one spec file -> cache-warm, train, rollout, analysis.",
+)
+app.add_typer(workflow_app, name="workflow")
+
+
+@workflow_app.command("run")
+def workflow_run(
+    spec: Annotated[Path, typer.Argument(help="Workflow TOML (see configs/workflow_example.toml)")],
+    batch: Annotated[
+        bool,
+        typer.Option("--batch/--local", help="Submit batch-system tasks to HTCondor, or run everything locally"),
+    ] = False,
+    workers: Annotated[int, typer.Option("--workers", help="Concurrent luigi workers")] = 1,
+    mode: Annotated[
+        str,
+        typer.Option(
+            "--mode",
+            help="run | dry-run (print pending tasks) | show-output (print every target) | remove (delete outputs)",
+        ),
+    ] = "run",
+    scheduler_host: Annotated[Optional[str], typer.Option("--scheduler-host", help="luigid host")] = None,
+    scheduler_port: Annotated[Optional[int], typer.Option("--scheduler-port", help="luigid port")] = None,
+) -> None:
+    """Run a workflow spec end to end (the only sanctioned multi-step entry point).
+
+    A thin exec of `giant/workflow/run.py`, which b2luigi also re-executes on
+    every worker — so there is one documented entry point and one code path.
+    """
+    import subprocess
+
+    script = Path(__file__).resolve().parent / "workflow" / "run.py"
+    cmd = [sys.executable, str(script), "--spec", str(spec), "--workers", str(workers), "--mode", mode]
+    if batch:
+        cmd.append("--batch")
+    if scheduler_host:
+        cmd += ["--scheduler-host", scheduler_host]
+    if scheduler_port:
+        cmd += ["--scheduler-port", str(scheduler_port)]
+    raise typer.Exit(subprocess.run(cmd).returncode)
+
+
 analyze_app = typer.Typer(
     no_args_is_help=True,
     help="Rollout-vs-reference analysis: parallel compute on HTCondor + local render.",
@@ -1556,10 +1611,23 @@ app.add_typer(analyze_app, name="analyze")
 
 @analyze_app.command("prep")
 def analyze_prep(
-    rollout_yaml: Annotated[
-        Path,
-        typer.Argument(help="giant rollout YAML sidecar (names the rollout + reference files)"),
+    rollout_yamls: Annotated[
+        list[Path],
+        typer.Argument(
+            help="giant rollout YAML sidecar(s) (names the rollout + reference files). "
+            "Multiple compare N rollouts against one shared reference — every YAML must "
+            "name the same `dataset`."
+        ),
     ],
+    label: Annotated[
+        Optional[list[str]],
+        typer.Option(
+            "--label",
+            help="Series name for a rollout YAML, positionally matched to it — give none, "
+            'or exactly one per YAML. Defaults to the YAML stem (or "rollout" for a '
+            "single YAML).",
+        ),
+    ] = None,
     run_dir: Annotated[
         Optional[Path],
         typer.Option(
@@ -1576,14 +1644,15 @@ def analyze_prep(
         typer.Option("--chunks", help="Split each plot's data into this many event_id chunks"),
     ] = 1,
 ) -> None:
-    """Read the rollout YAML → shared.json + run_meta.json in the run directory."""
+    """Read the rollout YAML(s) → shared.json + run_meta.json in the run directory."""
     from giant.analysis import prep
 
     path = prep(
-        rollout_yaml,
+        rollout_yamls,
         run_dir,
         n_chunks=chunks,
         default_base=Path.cwd() / "analysis_runs",
+        labels=label,
         n_energy_bins=n_energy_bins,
         n_marginal_bins=n_marginal_bins,
         top_k_pdg=top_k_pdg,
@@ -1644,66 +1713,23 @@ def analyze_render(
     typer.echo(f"rendered {len(pdfs)} plots → {Path(run_dir) / 'plots'}")
 
 
-@analyze_app.command("submit")
-def analyze_submit(
-    rollout_yaml: Annotated[Path, typer.Argument(help="giant rollout YAML sidecar")],
-    accounting_group: Annotated[str, typer.Option("--accounting-group")],
-    run_dir: Annotated[
+@analyze_app.command("metrics")
+def analyze_metrics(
+    run_dir: Annotated[Path, typer.Argument(help="Run directory containing metrics.csv (from `giant train`)")],
+    out_dir: Annotated[
         Optional[Path],
         typer.Option(
-            "--run-dir",
+            "--out",
             "-o",
-            help="Override the run directory (default: <cwd>/analysis_runs/analysis_<id>)",
+            help="Override the output directory (default: <cwd>/analysis_runs/metrics_<run_dir name>)",
         ),
     ] = None,
-    docker_image: Annotated[str, typer.Option("--docker-image")] = "cverstege/alma9-gridjob",
-    request_memory: Annotated[int, typer.Option("--request-memory", help="MB")] = 8192,
-    remote: Annotated[
-        bool,
-        typer.Option("--remote/--local", help="+RemoteJob vs ProvidesETPResources"),
-    ] = False,
-    chunks: Annotated[
-        int,
-        typer.Option(
-            "--chunks",
-            help="Split each plot's data into this many event_id chunks/jobs",
-        ),
-    ] = 1,
-    n_energy_bins: Annotated[int, typer.Option("--energy-bins")] = 4,
-    n_marginal_bins: Annotated[int, typer.Option("--bins")] = 50,
-    top_k_pdg: Annotated[int, typer.Option("--top-pdg")] = 6,
-    dry_run: Annotated[bool, typer.Option("--dry-run", help="Write files but don't condor_submit")] = False,
 ) -> None:
-    """prep + write the HTCondor submit description (one job per plot x chunk), then submit."""
-    import subprocess
+    """Render training-progress plots (loss/lr/accuracy/grad-norm/router/wgan/throughput) from <run_dir>/metrics.csv."""
+    from giant.training.plots import render_metrics
 
-    from giant.analysis import SubmitConfig, prep, write_submit
-
-    path = prep(
-        rollout_yaml,
-        run_dir,
-        n_chunks=chunks,
-        default_base=Path.cwd() / "analysis_runs",
-        n_energy_bins=n_energy_bins,
-        n_marginal_bins=n_marginal_bins,
-        top_k_pdg=top_k_pdg,
-    )
-    cfg = SubmitConfig(
-        run_dir=path,
-        accounting_group=accounting_group,
-        repo_dir=Path.cwd(),
-        docker_image=docker_image,
-        request_memory_mb=request_memory,
-        remote=remote,
-        n_chunks=chunks,
-    )
-    sub = write_submit(cfg)
-    typer.echo(f"run directory: {path}")
-    typer.echo(f"wrote submit description: {sub}")
-    if dry_run:
-        typer.echo("dry-run: not submitting")
-        return
-    subprocess.run(["condor_submit", str(sub)], check=True)
+    paths = render_metrics(run_dir, out_dir, default_base=Path.cwd() / "analysis_runs")
+    typer.echo(f"rendered {len(paths)} plots -> {paths[0].parent if paths else '(nothing to render)'}")
 
 
 if __name__ == "__main__":

giant/config.py

@@ -1769,6 +1769,19 @@ def resolve_default_out_dir(cfg: dict, base: Path = Path("checkpoints")) -> Path
     return out_dir
 
 
+def epoch_seed(seed: int, epoch: int) -> int:
+    """Per-epoch derivative of the run seed.
+
+    Reseeding the global RNGs from this at the top of every epoch makes epoch
+    *k* draw the same noise whether it runs inside one long `giant train` or
+    as its own resumed job in a per-epoch workflow chain
+    (`giant/workflow/tasks.py:TrainEpochTask`) — without it, a fresh process
+    would restart the stream at epoch 1's state. Mirrors what
+    `StreamingStepsDataset.set_epoch` does for the batch order.
+    """
+    return (int(seed) * 1_000_003 + int(epoch)) % (2**32)
+
+
 def seed_everything(seed: int) -> None:
     random.seed(seed)
     np.random.seed(seed)

giant/data/dataset.py

@@ -97,6 +97,7 @@ class StreamingStepsDataset(IterableDataset):
         mat_topn_map: dict[str, int] | None = None,
         sec_type_class_map: dict | None = None,
         k_max: int = K_MAX,
+        seed: int = 0,
     ) -> None:
         self.files = list(files)
         self._offsets = {path: event_id_offset(i) for i, path in enumerate(self.files)}
@@ -117,16 +118,35 @@ class StreamingStepsDataset(IterableDataset):
         self.mat_topn_map = mat_topn_map
         self.sec_type_class_map = sec_type_class_map
         self.k_max = k_max
+        self.seed = seed
+        self.epoch = 0
+        self._rng = np.random.default_rng()
+
+    def set_epoch(self, epoch: int) -> None:
+        """Select the shuffle stream for `epoch` (the DistributedSampler convention).
+
+        The training loop calls this at the top of every epoch. Shuffling is
+        seeded from `(seed, epoch, worker_id)` rather than the global numpy
+        state so epoch *k*'s batch order is the same whether it runs as epoch
+        *k* of one long `giant train`, or as its own resumed job in a
+        per-epoch workflow chain (`giant/workflow/tasks.py:TrainEpochTask`).
+        Workers are re-forked from this object each epoch (no
+        `persistent_workers`), so setting it here reaches them.
+        """
+        self.epoch = int(epoch)
 
     def __iter__(self):
         worker_info = torch.utils.data.get_worker_info()
         files = self.files
+        worker_id = worker_info.id if worker_info is not None else 0
         if worker_info is not None:
-            files = files[worker_info.id :: worker_info.num_workers]
+            files = files[worker_id :: worker_info.num_workers]
+
+        self._rng = np.random.default_rng([self.seed, self.epoch, worker_id])
 
         if self.shuffle:
             files = list(files)
-            np.random.default_rng().shuffle(files)
+            self._rng.shuffle(files)
 
         buf_cont: list[np.ndarray] = []
         buf_cat: list[np.ndarray] = []
@@ -222,7 +242,7 @@ class StreamingStepsDataset(IterableDataset):
         styp = np.concatenate(buf_type)
 
         if self.shuffle:
-            idx = np.random.permutation(len(cont))
+            idx = self._rng.permutation(len(cont))
             cont, cat, tgt = cont[idx], cat[idx], tgt[idx]
             nsec, sec, proc, styp = nsec[idx], sec[idx], proc[idx], styp[idx]
 

giant/pipeline.py

@@ -435,6 +435,7 @@ def run_train_job(
         mat_topn_map=cond_mat_topn,
         sec_type_class_map=sec_type_class_map,
         k_max=k_max,
+        seed=t["seed"],
     )
     val_ds = StreamingStepsDataset(
         files=files,

giant/tools/profile_analysis_costs.py

@@ -26,8 +26,9 @@ import numpy as np
 import polars as pl
 
 from giant.analysis.catalog import catalog_ids, get_spec
-from giant.analysis.condor import compute_reduced
+from giant.analysis.run import compute_reduced
 from giant.analysis.context import build_context
+from giant.analysis.sources import RolloutSpec
 
 # Row counts (per side) to benchmark at. Kept in local memory/CPU range so the
 # whole sweep finishes in about a minute; the fit is linear so it extrapolates
@@ -165,11 +166,10 @@ def _time(spec_id: str, rollout: Path, reference: Path, shared: Path, out: Path)
     t0 = time.perf_counter()
     compute_reduced(
         spec_id,
-        rollout,
+        [{"name": "rollout", "path": str(rollout)}],
         reference,
         shared,
         out,
-        checkpoint=None,
         chunk_index=0,
         n_chunks=1,
     )
@@ -191,7 +191,7 @@ def main() -> None:
 
             shared = tmp_path / f"shared_{n_side}.json"
             ctx = build_context(
-                rollout,
+                [RolloutSpec(name="rollout", source=rollout)],
                 reference,
                 n_energy_bins=4,
                 n_marginal_bins=50,

giant/training/loop.py

@@ -18,6 +18,7 @@ import torch
 from torch.utils.data import DataLoader
 from tqdm import tqdm
 
+from giant import config
 from giant.data.loader import TopNMap
 from giant.data.setup_cache import topnmap_to_json
 from giant.training.checkpoint import build_checkpoint, init_stages_from_checkpoints, load_checkpoint
@@ -184,6 +185,17 @@ def train(
             if device.type == "cuda":
                 torch.cuda.reset_peak_memory_stats(device)
             collector.start_epoch(epoch)
+            # Epoch-aware RNG: same noise (and, below, same batch order) for
+            # epoch k whether the run is one process or a chain of per-epoch
+            # jobs. See giant.config.epoch_seed.
+            config.seed_everything(config.epoch_seed(t["seed"], epoch))
+            # Epoch-aware shuffle stream (see StreamingStepsDataset.set_epoch):
+            # keeps epoch k's batch order identical whether it runs here or as
+            # its own resumed per-epoch job in a b2luigi workflow.
+            # (tests hand `train` a plain list of batches, which has neither)
+            set_epoch = getattr(getattr(train_loader, "dataset", None), "set_epoch", None)
+            if callable(set_epoch):
+                set_epoch(epoch)
             for trainer in trainers.values():
                 trainer.train_mode()
 

giant/training/plots.py

@@ -0,0 +1,356 @@
+"""Training-progress plots from `<run_dir>/metrics.csv` (gitea #75).
+
+`MetricsCollector` (`giant.training.metrics`) writes one row per epoch with a
+column set that varies by run — flow/ddpm vs wgan, routed vs not (see the
+`MetricSpec` declarations in `giant.training.trainers`). This module reads
+that header dynamically rather than hardcoding a column list, buckets columns
+by the fixed naming convention `MetricsCollector` itself documents
+(`<stage>/train/<key>`, `<stage>/val/<key>`, `<stage>/router/<key>`,
+`<stage>/<key>` for point-in-time values, and an unprefixed run-level tail —
+see `giant.training.metrics`'s module docstring), and renders one PDF per
+applicable figure with the same `plotstyle` conventions
+`giant.analysis.render` uses, for visual consistency with the
+rollout-vs-reference plots.
+
+Unlike `giant.analysis`, there is no reduce/chunk/condor split here — the CSV
+is tiny and this always runs as one local pass — but the CLI entry point
+still lives under `giant analyze` (`analyze metrics`) as the shared home for
+plotstyle-rendered diagnostics, and shares its `analysis_runs/` output
+convention (see `derive_metrics_dir`) so training-progress plots don't get
+written into the training run directory itself.
+"""
+
+from __future__ import annotations
+
+import csv
+import math
+from dataclasses import dataclass
+from pathlib import Path
+
+# Stage names are always exactly these two — hardcoded in
+# `giant.training.trainers.build_stage_trainers` — so a column belongs to a
+# stage iff it's prefixed by one of these, and everything else (bar `epoch`)
+# is run-level. This is what makes dynamic header parsing tractable without
+# needing to know the per-run metric keys themselves.
+_STAGE_NAMES = ("stage1", "stage2")
+
+_ACC_KEYS = {"nsec_acc", "stop_acc", "type_acc"}
+_WGAN_BALANCE_KEYS = {"d_loss", "g_loss", "wasserstein", "gp_loss"}
+_ROUTER_KEYS = ("entropy", "util_min", "util_max", "util_std")
+
+
+@dataclass
+class MetricsTable:
+    """`<run_dir>/metrics.csv`, parsed with no hardcoded column list."""
+
+    epochs: list[int]
+    columns: dict[str, list[float]]
+
+    @classmethod
+    def load(cls, path: str | Path) -> "MetricsTable":
+        with open(path, newline="") as f:
+            rows = list(csv.DictReader(f))
+        epochs = [int(float(r["epoch"])) for r in rows]
+        fieldnames = rows[0].keys() if rows else []
+        columns = {name: [float(r[name]) for r in rows] for name in fieldnames if name != "epoch"}
+        return cls(epochs=epochs, columns=columns)
+
+    def best_epochs(self) -> list[int]:
+        is_best = self.columns.get("is_best")
+        if not is_best:
+            return []
+        return [epoch for epoch, flag in zip(self.epochs, is_best) if flag]
+
+
+# --- column classification --------------------------------------------------
+
+
+def _stages(columns: dict) -> list[str]:
+    return [s for s in _STAGE_NAMES if any(name.startswith(f"{s}/") for name in columns)]
+
+
+def _split(columns: dict, stage: str, split: str) -> dict[str, str]:
+    prefix = f"{stage}/{split}/"
+    return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)}
+
+
+def _point_in_time(columns: dict, stage: str) -> dict[str, str]:
+    prefix = f"{stage}/"
+    out = {}
+    for name in columns:
+        if not name.startswith(prefix):
+            continue
+        rest = name[len(prefix) :]
+        head = rest.split("/", 1)[0]
+        if head not in ("train", "val", "router"):
+            out[rest] = name
+    return out
+
+
+def _router(columns: dict, stage: str) -> dict[str, str]:
+    prefix = f"{stage}/router/"
+    return {name[len(prefix) :]: name for name in columns if name.startswith(prefix)}
+
+
+def _run_level(columns: dict) -> dict[str, str]:
+    known_prefixes = tuple(f"{s}/" for s in _STAGE_NAMES)
+    return {name: name for name in columns if not name.startswith(known_prefixes)}
+
+
+def _loss_keys(train: dict[str, str], val: dict[str, str]) -> list[str]:
+    keys = {k for k in train if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"}
+    keys |= {k for k in val if k not in _ACC_KEYS and k not in _WGAN_BALANCE_KEYS and k != "grad_norm"}
+    return sorted(keys)
+
+
+# --- output location ---------------------------------------------------------
+
+
+def derive_metrics_dir(
+    run_dir: str | Path,
+    out_dir: str | Path | None = None,
+    default_base: str | Path | None = None,
+) -> Path:
+    """Plots output directory.
+
+    Precedence: an explicit `out_dir` always wins. Otherwise
+    `default_base / f"metrics_{run_dir.name}"` (the CLI passes the repo's
+    gitignored `analysis_runs/`, matching `giant.analysis.run.derive_run_dir`'s
+    convention) — training-progress plots live alongside rollout-vs-reference
+    analysis runs, not inside the training run directory itself.
+    """
+    if out_dir is not None:
+        return Path(out_dir)
+    base = Path(default_base) if default_base is not None else Path.cwd() / "analysis_runs"
+    return base / f"metrics_{Path(run_dir).name}"
+
+
+# --- figures ------------------------------------------------------------------
+
+
+def _mark_best(ax, table: MetricsTable) -> None:
+    for epoch in table.best_epochs():
+        ax.axvline(epoch, color="grey", linestyle="--", linewidth=0.8, alpha=0.7)
+
+
+def _overview_figure(table: MetricsTable):
+    import plotstyle as ps
+
+    run_level = _run_level(table.columns)
+    if "val/loss" not in run_level:
+        return None
+    fig, ax = ps.new_figure("thesis-single", title="training overview")
+    ax.plot(table.epochs, table.columns["val/loss"], label="val/loss")
+    if "val/marginal_kl" in run_level:
+        kl = table.columns["val/marginal_kl"]
+        if any(math.isfinite(v) for v in kl):
+            ax.plot(table.epochs, kl, label="val/marginal_kl")
+    _mark_best(ax, table)
+    best = table.best_epochs()
+    if best:
+        idx = table.epochs.index(best[-1])
+        ax.annotate(
+            f"best: epoch {best[-1]}\nval/loss={table.columns['val/loss'][idx]:.4g}",
+            xy=(best[-1], table.columns["val/loss"][idx]),
+            xytext=(0.98, 0.95),
+            textcoords="axes fraction",
+            ha="right",
+            va="top",
+            fontsize=8,
+        )
+    ax.set_xlabel("epoch")
+    ax.set_ylabel("loss")
+    ps.style_legend(ax, title="series")
+    return fig
+
+
+def _loss_figure(table: MetricsTable, stage: str):
+    import plotstyle as ps
+
+    train = _split(table.columns, stage, "train")
+    val = _split(table.columns, stage, "val")
+    keys = _loss_keys(train, val)
+    if not keys:
+        return None
+    n = len(keys)
+    ncols = min(3, n)
+    nrows = (n + ncols - 1) // ncols
+    fig, axes = ps.new_figure(
+        "slide-16x9",
+        title=f"{stage} loss",
+        nrows=nrows,
+        ncols=ncols,
+        squeeze=False,
+    )
+    flat = axes.ravel()
+    for ax, key in zip(flat, keys):
+        if key in train:
+            ax.plot(table.epochs, table.columns[train[key]], label="train")
+        if key in val:
+            ax.plot(table.epochs, table.columns[val[key]], label="val")
+        ax.set_yscale("log")
+        ax.set_title(key, fontsize=8)
+        ax.set_xlabel("epoch")
+    for j in range(n, len(flat)):
+        flat[j].set_visible(False)
+    ps.style_legend(flat[0], title="series")
+    return fig
+
+
+def _lr_figure(table: MetricsTable):
+    import plotstyle as ps
+
+    series: dict[str, str] = {}
+    for stage in _stages(table.columns):
+        for key, col in _point_in_time(table.columns, stage).items():
+            series[f"{stage}/{key}"] = col
+    if not series:
+        return None
+    fig, ax = ps.new_figure("thesis-single", title="learning rate schedule")
+    for label, col in series.items():
+        ax.plot(table.epochs, table.columns[col], label=label)
+    ax.set_xlabel("epoch")
+    ax.set_ylabel("learning rate")
+    ps.style_legend(ax, title="series")
+    return fig
+
+
+def _accuracy_figure(table: MetricsTable, stage: str):
+    import plotstyle as ps
+
+    train = _split(table.columns, stage, "train")
+    val = _split(table.columns, stage, "val")
+    keys = sorted((set(train) | set(val)) & _ACC_KEYS)
+    if not keys:
+        return None
+    n = len(keys)
+    fig, axes = ps.new_figure("slide-16x9", title=f"{stage} accuracy", nrows=1, ncols=n, squeeze=False)
+    flat = axes.ravel()
+    for ax, key in zip(flat, keys):
+        if key in train:
+            ax.plot(table.epochs, table.columns[train[key]], label="train")
+        if key in val:
+            ax.plot(table.epochs, table.columns[val[key]], label="val")
+        ax.set_title(key, fontsize=8)
+        ax.set_xlabel("epoch")
+        ax.set_ylim(0, 1)
+    ps.style_legend(flat[0], title="series")
+    return fig
+
+
+def _grad_norm_figure(table: MetricsTable):
+    import plotstyle as ps
+
+    run_level = _run_level(table.columns)
+    if "grad_norm" not in run_level:
+        return None
+    fig, ax = ps.new_figure("thesis-single", title="gradient norm")
+    ax.plot(table.epochs, table.columns["grad_norm"], label="grad_norm")
+    for stage in _stages(table.columns):
+        train = _split(table.columns, stage, "train")
+        for key in ("grad_norm_d", "grad_norm_g", "grad_norm_type_slice", "grad_norm_cont_slice"):
+            if key in train:
+                ax.plot(table.epochs, table.columns[train[key]], label=f"{stage}/{key}")
+    ax.set_yscale("log")
+    ax.set_xlabel("epoch")
+    ax.set_ylabel("grad norm")
+    ps.style_legend(ax, title="series")
+    return fig
+
+
+def _router_figure(table: MetricsTable, stage: str):
+    import plotstyle as ps
+
+    router = _router(table.columns, stage)
+    if "entropy" not in router:
+        return None
+    fig, ax = ps.new_figure("thesis-single", title=f"{stage} router health")
+    ax.plot(table.epochs, table.columns[router["entropy"]], label="entropy", color="black")
+    ax.set_xlabel("epoch")
+    ax.set_ylabel("entropy [bits]")
+    ax2 = ax.twinx()
+    for key in ("util_min", "util_max", "util_std"):
+        if key in router:
+            ax2.plot(table.epochs, table.columns[router[key]], label=key, linestyle="--")
+    ax2.set_ylabel("expert utilization")
+    ax2.set_ylim(0, 1)
+    lines1, labels1 = ax.get_legend_handles_labels()
+    lines2, labels2 = ax2.get_legend_handles_labels()
+    ax.legend(lines1 + lines2, labels1 + labels2, loc="upper right", frameon=False, fontsize=7)
+    return fig
+
+
+def _wgan_balance_figure(table: MetricsTable, stage: str):
+    import plotstyle as ps
+
+    train = _split(table.columns, stage, "train")
+    keys = [k for k in _WGAN_BALANCE_KEYS if k in train]
+    if not keys:
+        return None
+    fig, ax = ps.new_figure("thesis-single", title=f"{stage} WGAN critic/generator balance")
+    for key in sorted(keys):
+        ax.plot(table.epochs, table.columns[train[key]], label=key)
+    ax.set_xlabel("epoch")
+    ax.set_ylabel("value")
+    ps.style_legend(ax, title="series")
+    return fig
+
+
+def _throughput_figure(table: MetricsTable):
+    import plotstyle as ps
+
+    run_level = _run_level(table.columns)
+    keys = [k for k in ("samples_per_sec", "gpu_mem_mb", "epoch_time_s") if k in run_level]
+    if not keys:
+        return None
+    fig, axes = ps.new_figure("slide-16x9", title="throughput / resources", nrows=1, ncols=len(keys), squeeze=False)
+    flat = axes.ravel()
+    for ax, key in zip(flat, keys):
+        ax.plot(table.epochs, table.columns[key])
+        _mark_best(ax, table)
+        ax.set_title(key, fontsize=8)
+        ax.set_xlabel("epoch")
+    return fig
+
+
+# --- entry point ---------------------------------------------------------
+
+
+def render_metrics(
+    run_dir: str | Path,
+    out_dir: str | Path | None = None,
+    default_base: str | Path | None = None,
+) -> list[Path]:
+    """`<run_dir>/metrics.csv` -> `<plots dir>/<name>.pdf`.
+
+    See `derive_metrics_dir` for how the plots directory is resolved.
+    """
+    import matplotlib.pyplot as plt
+    import plotstyle as ps
+
+    ps.use()
+    table = MetricsTable.load(Path(run_dir) / "metrics.csv")
+    plots_dir = derive_metrics_dir(run_dir, out_dir, default_base)
+    plots_dir.mkdir(parents=True, exist_ok=True)
+
+    figures = [("overview", _overview_figure(table))]
+    for stage in _stages(table.columns):
+        figures.append((f"{stage}_loss", _loss_figure(table, stage)))
+    figures.append(("lr", _lr_figure(table)))
+    for stage in _stages(table.columns):
+        figures.append((f"{stage}_accuracy", _accuracy_figure(table, stage)))
+    figures.append(("grad_norm", _grad_norm_figure(table)))
+    for stage in _stages(table.columns):
+        figures.append((f"{stage}_router", _router_figure(table, stage)))
+        figures.append((f"{stage}_wgan_balance", _wgan_balance_figure(table, stage)))
+    figures.append(("throughput", _throughput_figure(table)))
+
+    paths: list[Path] = []
+    for name, fig in figures:
+        if fig is None:
+            continue
+        path = plots_dir / name
+        ps.savefig(fig, str(path), formats=("pdf",))
+        plt.close(fig)
+        paths.append(path.with_suffix(".pdf"))
+    return paths

giant/workflow/__init__.py

@@ -0,0 +1,40 @@
+"""b2luigi orchestration of the full GIANT pipeline.
+
+One workflow TOML (``spec.py``) parameterises an entire experiment — dataset,
+geometry oracle, N trainings, N rollouts, N analyses — and ``giant workflow
+run <spec.toml>`` turns it into a b2luigi task graph (``tasks.py``) whose
+targets are files on ``/ceph``: nothing is recomputed that already exists,
+every step waits for its inputs, and HTCondor submission/polling is b2luigi's
+job rather than a hand-rolled submit-file generator.
+
+This is the only sanctioned way to run a multi-step pipeline; ``giant`` and
+``dwarf`` stay single-step primitives that these tasks invoke.
+
+``tasks``/``run`` import b2luigi, so they are *not* imported here — a plain
+``import giant.workflow`` (or ``giant.workflow.spec``) works without the
+``workflow`` extra installed.
+"""
+
+from giant.workflow.spec import (
+    AnalysisSpec,
+    CondorSpec,
+    DatasetSpec,
+    GeometrySpec,
+    RolloutSpec,
+    TrainSpec,
+    WorkflowSpec,
+    load_spec,
+    spec_hash,
+)
+
+__all__ = [
+    "AnalysisSpec",
+    "CondorSpec",
+    "DatasetSpec",
+    "GeometrySpec",
+    "RolloutSpec",
+    "TrainSpec",
+    "WorkflowSpec",
+    "load_spec",
+    "spec_hash",
+]

giant/workflow/htcondor.py

@@ -0,0 +1,82 @@
+"""HTCondor job descriptions for the workflow tasks.
+
+b2luigi writes every key of a task's ``htcondor_settings`` dict straight into
+that job's submit description, so these helpers are just the ETP-specific
+resource/requirement conventions in one place:
+
+* **CPU jobs** (setup cache, geometry oracle, analysis compute) keep what
+  the deleted ``giant analyze submit`` used: ``+RemoteJob`` for grid I/O, or
+  ``TARGET.ProvidesETPResources`` when the files are local to the cluster.
+* **GPU jobs** (training epochs, rollout) are remote-only, so they always
+  carry ``+RemoteJob`` and reach ``/ceph`` through
+  ``TARGET.ProvidesEtpCeph`` — the requirement strings are ported from the
+  ``condor-gpu-train-rollout`` branch's ``giant/condor.py`` rather than
+  rewritten, since they encode what the ETP HTCondor wiki documents for
+  TOpAS/NEMO2 GPU workers.
+"""
+
+from __future__ import annotations
+
+from giant.workflow.spec import CondorSpec
+
+__all__ = ["cpu_settings", "gpu_settings", "gpu_requirements"]
+
+
+def cpu_settings(
+    condor: CondorSpec,
+    *,
+    request_memory_mb: int | None = None,
+    request_cpus: int | None = None,
+    walltime_s: int | None = None,
+) -> dict:
+    settings: dict = {
+        "universe": "docker",
+        "docker_image": condor.docker_image_cpu,
+        "request_memory": request_memory_mb if request_memory_mb is not None else condor.request_memory_mb,
+        "request_cpus": request_cpus if request_cpus is not None else condor.request_cpus,
+        "accounting_group": condor.accounting_group,
+        "should_transfer_files": "YES",
+        "when_to_transfer_output": "ON_EXIT",
+    }
+    if condor.remote:
+        settings["+RemoteJob"] = "True"
+    else:
+        settings["requirements"] = "TARGET.ProvidesETPResources"
+    if walltime_s is not None:
+        settings["+RequestWalltime"] = int(walltime_s)
+    return settings
+
+
+def gpu_requirements(gpu_type: str | None = None, gpu_memory_mb: int | None = None) -> str:
+    """``TARGET.ProvidesEtpCeph`` (remote /ceph access) ANDed with any GPU pin."""
+    clauses = ["TARGET.ProvidesEtpCeph =?= True"]
+    if gpu_type is not None:
+        clauses.append(f'TARGET.GPUs_DeviceName =?= "{gpu_type}"')
+    if gpu_memory_mb is not None:
+        clauses.append(f"TARGET.GPUs_GlobalMemoryMb >= {gpu_memory_mb}")
+    return " && ".join(clauses)
+
+
+def gpu_settings(
+    condor: CondorSpec,
+    *,
+    request_gpus: int = 1,
+    gpu_type: str | None = None,
+    gpu_memory_mb: int | None = None,
+    request_memory_mb: int = 16384,
+    request_cpus: int = 4,
+    walltime_s: int = 86400,
+) -> dict:
+    return {
+        "universe": "docker",
+        "docker_image": condor.docker_image_gpu,
+        "request_memory": request_memory_mb,
+        "request_cpus": request_cpus,
+        "RequestGPUs": request_gpus,
+        "+RequestWalltime": int(walltime_s),
+        "accounting_group": condor.accounting_group,
+        "should_transfer_files": "YES",
+        "when_to_transfer_output": "ON_EXIT",
+        "+RemoteJob": "True",
+        "requirements": f"({gpu_requirements(gpu_type, gpu_memory_mb)})",
+    }

giant/workflow/run.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+"""Entry point b2luigi re-executes on every worker.
+
+Locally this is what ``giant workflow run <spec.toml>`` execs; on a batch
+worker it is what the generated wrapper script runs (after ``cd repo_dir`` and
+sourcing ``env_script``), with ``--spec`` forwarded via the
+``task_cmd_additional_args`` setting so the worker resolves exactly the same
+spec — and therefore the same task graph and output paths — as the submitter.
+
+b2luigi needs a real script path for that re-execution, which is why this is a
+script rather than a ``python -m`` module.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+# Allow `python giant/workflow/run.py` from a checkout that isn't installed.
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+import b2luigi  # noqa: E402
+
+from giant.workflow.spec import WorkflowSpec, load_spec  # noqa: E402
+from giant.workflow.tasks import WorkflowTask, set_spec  # noqa: E402
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Run a GIANT workflow spec with b2luigi.")
+    parser.add_argument("--spec", required=True, help="Workflow TOML (see configs/workflow_example.toml)")
+    parser.add_argument("--workers", type=int, default=1, help="Concurrent luigi workers")
+    parser.add_argument(
+        "--batch",
+        action="store_true",
+        help="Submit batch-system tasks to HTCondor (otherwise everything runs locally)",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=("run", "dry-run", "show-output", "remove"),
+        default="run",
+        help="run (default), dry-run (print pending tasks), show-output (print every target), remove (delete outputs)",
+    )
+    parser.add_argument("--scheduler-host", default=None, help="luigid host (default: local scheduler)")
+    parser.add_argument("--scheduler-port", type=int, default=None, help="luigid port")
+    return parser
+
+
+def configure(spec: WorkflowSpec, spec_path: Path, batch: bool) -> None:
+    """Wire b2luigi's settings from the spec.
+
+    ``/ceph`` is shared between submit host and workers, so there is
+    deliberately no ``transfer_files``: ``result_dir``/``log_dir`` must live
+    somewhere both sides can see.
+    """
+    set_spec(spec)
+
+    b2luigi.set_setting("result_dir", spec.result_dir)
+    b2luigi.set_setting("log_dir", spec.log_dir)
+    b2luigi.set_setting("task_file_dir", str(Path(spec.result_dir) / "task_files"))
+    b2luigi.set_setting("use_parameter_name_in_output", True)
+    b2luigi.set_setting("batch_system", "htcondor" if batch else "local")
+    b2luigi.set_setting("working_dir", spec.condor.repo_dir)
+    b2luigi.set_setting("job_name", spec.name)
+    if spec.condor.env_script:
+        b2luigi.set_setting("env_script", spec.condor.env_script)
+    # The worker command is `<executable> [<basename of this file>] --batch-runner
+    # --task-id ...`, run after `cd working_dir`. Only the *basename* would be
+    # used, so the filename is dropped and the repo-relative script path is
+    # made part of the executable instead.
+    b2luigi.set_setting("add_filename_to_cmd", False)
+    b2luigi.set_setting("executable", [".venv/bin/python", "giant/workflow/run.py"])
+    b2luigi.set_setting("task_cmd_additional_args", ["--spec", str(spec_path)])
+
+
+def main(argv: list[str] | None = None) -> None:
+    args, _ = build_parser().parse_known_args(argv)
+    spec_path = Path(args.spec).resolve()
+    spec = load_spec(spec_path)
+    configure(spec, spec_path, batch=args.batch)
+
+    kwargs: dict = {}
+    if args.scheduler_host:
+        kwargs["scheduler_host"] = args.scheduler_host
+    if args.scheduler_port:
+        kwargs["scheduler_port"] = args.scheduler_port
+
+    b2luigi.process(
+        WorkflowTask(workflow_name=spec.name),
+        workers=args.workers,
+        batch=args.batch,
+        dry_run=args.mode == "dry-run",
+        show_output=args.mode == "show-output",
+        remove=args.mode == "remove",
+        auto_confirm=args.mode == "remove",
+        # run.py owns --spec/--mode/...; b2luigi must not choke on them.
+        ignore_additional_command_line_args=True,
+        **kwargs,
+    )
+
+
+if __name__ == "__main__":
+    main()

giant/workflow/spec.py

@@ -0,0 +1,383 @@
+"""Workflow TOML -> frozen dataclasses, validation, and per-task spec hashes.
+
+One spec file is the only place a pipeline is parameterised (see
+``configs/workflow_example.toml``):
+
+    [workflow]  name / result_dir / log_dir
+    [condor]    accounting group, repo dir, env script, docker images
+    [dataset]   steps (training) + reference (rollout seeds & analysis truth)
+    [geometry]  geometry-oracle build options
+    [[train]]   one per training run      (name, config, epochs, overrides, ...)
+    [[rollout]] one per rollout           (name, train = <a [[train]].name>, ...)
+    [[analysis]] one per comparison       (name, rollouts = [<[[rollout]].name>, ...])
+
+Every task carries its ``name`` plus a short ``spec_hash`` — 8 hex of the
+canonical JSON of its own resolved sub-spec **including its transitive
+parents**. That is what makes an edited spec produce a fresh result directory
+instead of silently reusing outputs computed under different settings: change
+the dataset and every hash downstream of it changes too.
+
+Unknown keys are rejected (with the valid ones listed), in the same spirit as
+``giant.config.validate_config_keys`` — a typo in a workflow spec would
+otherwise be a silently ignored setting on a multi-day pipeline.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import tomllib
+from dataclasses import MISSING, dataclass, field, fields, is_dataclass
+from pathlib import Path
+from typing import Any
+
+__all__ = [
+    "AnalysisSpec",
+    "CondorSpec",
+    "DatasetSpec",
+    "GeometrySpec",
+    "RolloutSpec",
+    "TrainSpec",
+    "WorkflowSpec",
+    "load_spec",
+    "spec_hash",
+]
+
+
+class WorkflowSpecError(ValueError):
+    """Raised for any malformed workflow spec (unknown key, bad reference, ...)."""
+
+
+# ---------------------------------------------------------------------------
+# sub-specs
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class CondorSpec:
+    """Where and how jobs run — the batch-system half of the spec.
+
+    ``repo_dir`` doubles as b2luigi's ``working_dir`` (jobs ``cd`` there before
+    running ``giant/workflow/run.py``), and ``env_script`` is sourced first,
+    since submit and worker machines don't share an environment.
+    """
+
+    accounting_group: str
+    repo_dir: str
+    env_script: str = ""
+    docker_image_cpu: str = "cverstege/alma9-gridjob"
+    docker_image_gpu: str = "mschnepf/slc7-condocker"
+    remote: bool = True
+    request_cpus: int = 1
+    request_memory_mb: int = 8192
+
+
+@dataclass(frozen=True)
+class DatasetSpec:
+    """The two datasets every pipeline needs.
+
+    ``steps`` is what training reads; ``reference`` is the held-out file
+    rollouts are seeded from and the analysis compares against (the "one
+    ground truth" premise of ``giant.analysis``).
+    """
+
+    steps: str
+    reference: str
+
+
+@dataclass(frozen=True)
+class GeometrySpec:
+    """``dwarf build-geometry-oracle`` options (see giant/tools/geometry_oracle.py)."""
+
+    method: str = "slab"
+    k: int = 1
+    subsample: int = 500_000
+    escape_factor: float = 5.0
+    seed: int = 0
+    depth_axis: int = 2
+    n_bins: int = 2000
+
+
+@dataclass(frozen=True)
+class TrainSpec:
+    """One training run, fanned out into ``ceil(epochs / epochs_per_job)`` jobs.
+
+    ``overrides`` are ``[train]``/model config keys merged on top of ``config``
+    exactly as ``giant train``'s flags are (``giant.config.merge_cli_overrides``),
+    so anything expressible on the CLI is expressible here.
+    """
+
+    name: str
+    config: str | None = None
+    epochs: int = 1
+    epochs_per_job: int = 1
+    overrides: dict[str, Any] = field(default_factory=dict)
+    request_gpus: int = 1
+    gpu_type: str | None = None
+    gpu_memory_mb: int | None = None
+    request_memory_mb: int = 16384
+    request_cpus: int = 4
+    walltime_s: int = 86400
+    num_workers: int = 4
+    shuffle_buffer: int = 65536
+    device: str | None = None
+
+
+@dataclass(frozen=True)
+class RolloutSpec:
+    """One ``giant rollout`` run against the checkpoint of ``train``."""
+
+    name: str
+    train: str
+    n_events: int | None = None
+    energy_cutoff: float = 0.1
+    max_steps: int = 1000
+    steps: int = 10
+    batch_size: int = 4096
+    max_tracks_per_event: int | None = None
+    escape_threshold: float | None = None
+    weights: str = "raw"
+    seed: int | None = None
+    request_gpus: int = 1
+    gpu_type: str | None = None
+    gpu_memory_mb: int | None = None
+    request_memory_mb: int = 16384
+    request_cpus: int = 2
+    walltime_s: int = 86400
+    device: str | None = None
+
+
+@dataclass(frozen=True)
+class AnalysisSpec:
+    """One rollout-vs-reference comparison (N rollout series, one reference)."""
+
+    name: str
+    rollouts: tuple[str, ...]
+    chunks: int = 1
+    energy_bins: int = 4
+    bins: int = 50
+    top_pdg: int = 6
+    gallery: bool = False
+    request_memory_mb: int = 8192
+    request_cpus: int = 1
+
+
+@dataclass(frozen=True)
+class WorkflowSpec:
+    """A whole pipeline: the parsed spec file plus name-keyed lookups."""
+
+    name: str
+    result_dir: str
+    log_dir: str
+    condor: CondorSpec
+    dataset: DatasetSpec
+    geometry: GeometrySpec
+    trains: tuple[TrainSpec, ...]
+    rollouts: tuple[RolloutSpec, ...]
+    analyses: tuple[AnalysisSpec, ...]
+    path: str = ""
+
+    # -- lookups ----------------------------------------------------------
+    def train(self, name: str) -> TrainSpec:
+        return _lookup(self.trains, name, "train")
+
+    def rollout(self, name: str) -> RolloutSpec:
+        return _lookup(self.rollouts, name, "rollout")
+
+    def analysis(self, name: str) -> AnalysisSpec:
+        return _lookup(self.analyses, name, "analysis")
+
+    # -- hashes -----------------------------------------------------------
+    # Each one folds in everything upstream of it, so a change anywhere in a
+    # task's ancestry moves its result directory (and only the affected
+    # subtree's).
+    def dataset_hash(self) -> str:
+        return spec_hash(self.dataset)
+
+    def warm_cache_hash(self, train_name: str) -> str:
+        # The setup cache depends on the dataset and on what this training's
+        # config asks of it (val split, conditioning, router) — not on how
+        # many epochs it runs for, so epochs/resources are deliberately left
+        # out and two trainings sharing a config share one warm-cache job.
+        t = self.train(train_name)
+        return spec_hash(self.dataset, t.config, t.overrides)
+
+    def geometry_hash(self) -> str:
+        return spec_hash(self.dataset, self.geometry)
+
+    def train_hash(self, name: str) -> str:
+        return spec_hash(self.dataset, self.train(name))
+
+    def rollout_hash(self, name: str) -> str:
+        ro = self.rollout(name)
+        return spec_hash(self.dataset, self.geometry, self.train(ro.train), ro)
+
+    def analysis_hash(self, name: str) -> str:
+        an = self.analysis(name)
+        parents = [self.rollout(r) for r in an.rollouts]
+        train_parents = [self.train(r.train) for r in parents]
+        return spec_hash(self.dataset, self.geometry, train_parents, parents, an)
+
+
+def _lookup(items, name: str, kind: str):
+    for item in items:
+        if item.name == name:
+            return item
+    known = ", ".join(sorted(i.name for i in items)) or "(none defined)"
+    raise WorkflowSpecError(f"no [[{kind}]] named {name!r} in this workflow — defined: {known}")
+
+
+# ---------------------------------------------------------------------------
+# hashing
+# ---------------------------------------------------------------------------
+
+
+def spec_hash(*parts: Any, length: int = 8) -> str:
+    """Short stable hash of one or more (sub-)specs.
+
+    Canonical JSON (sorted keys, dataclasses expanded) so the value depends
+    only on the resolved settings — not on key order in the TOML, nor on
+    which defaults were written out explicitly.
+    """
+    payload = json.dumps([_canonical(p) for p in parts], sort_keys=True, separators=(",", ":"))
+    return hashlib.sha256(payload.encode()).hexdigest()[:length]
+
+
+def _canonical(value: Any) -> Any:
+    if is_dataclass(value) and not isinstance(value, type):
+        return {f.name: _canonical(getattr(value, f.name)) for f in fields(value)}
+    if isinstance(value, dict):
+        return {str(k): _canonical(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_canonical(v) for v in value]
+    if isinstance(value, Path):
+        return str(value)
+    return value
+
+
+# ---------------------------------------------------------------------------
+# parsing
+# ---------------------------------------------------------------------------
+
+
+def _build(cls, data: dict, where: str):
+    """Instantiate a frozen sub-spec, rejecting unknown/missing keys loudly."""
+    valid = {f.name for f in fields(cls)}
+    unknown = sorted(set(data) - valid)
+    if unknown:
+        raise WorkflowSpecError(f"{where}: unknown key(s) {unknown} — valid keys: {sorted(valid)}")
+    required = {f.name for f in fields(cls) if f.default is MISSING and f.default_factory is MISSING}
+    missing = sorted(required - set(data))
+    if missing:
+        raise WorkflowSpecError(f"{where}: missing required key(s) {missing}")
+    return cls(**data)
+
+
+def load_spec(path: str | Path) -> WorkflowSpec:
+    """Parse and validate a workflow TOML file."""
+    path = Path(path)
+    try:
+        raw = tomllib.loads(path.read_text())
+    except tomllib.TOMLDecodeError as exc:
+        raise WorkflowSpecError(f"{path}: not valid TOML — {exc}") from exc
+    return parse_spec(raw, path=path)
+
+
+def parse_spec(raw: dict, path: str | Path = "") -> WorkflowSpec:
+    """Validate an already-parsed workflow spec mapping."""
+    top_valid = {"workflow", "condor", "dataset", "geometry", "train", "rollout", "analysis"}
+    unknown = sorted(set(raw) - top_valid)
+    if unknown:
+        raise WorkflowSpecError(
+            f"{path or '<spec>'}: unknown top-level table(s) {unknown} — valid: {sorted(top_valid)}"
+        )
+
+    for required in ("workflow", "condor", "dataset"):
+        if required not in raw:
+            raise WorkflowSpecError(f"{path or '<spec>'}: missing required [{required}] table")
+
+    wf = dict(raw["workflow"])
+    wf_valid = {"name", "result_dir", "log_dir"}
+    wf_unknown = sorted(set(wf) - wf_valid)
+    if wf_unknown:
+        raise WorkflowSpecError(f"[workflow]: unknown key(s) {wf_unknown} — valid keys: {sorted(wf_valid)}")
+    if "name" not in wf or "result_dir" not in wf:
+        raise WorkflowSpecError("[workflow]: 'name' and 'result_dir' are required")
+    result_dir = str(Path(wf["result_dir"]).expanduser())
+    log_dir = str(Path(wf.get("log_dir", Path(result_dir) / "logs")).expanduser())
+
+    condor = _build(CondorSpec, dict(raw["condor"]), "[condor]")
+    dataset = _build(DatasetSpec, dict(raw["dataset"]), "[dataset]")
+    geometry = _build(GeometrySpec, dict(raw.get("geometry", {})), "[geometry]")
+
+    trains = tuple(_build(TrainSpec, dict(t), f"[[train]] #{i}") for i, t in enumerate(raw.get("train", [])))
+    rollouts = tuple(_build(RolloutSpec, dict(r), f"[[rollout]] #{i}") for i, r in enumerate(raw.get("rollout", [])))
+    analyses = tuple(
+        _build(AnalysisSpec, {**a, "rollouts": tuple(a.get("rollouts", ()))}, f"[[analysis]] #{i}")
+        for i, a in enumerate(raw.get("analysis", []))
+    )
+
+    _check_unique(trains, "train")
+    _check_unique(rollouts, "rollout")
+    _check_unique(analyses, "analysis")
+
+    train_names = {t.name for t in trains}
+    for ro in rollouts:
+        if ro.train not in train_names:
+            raise WorkflowSpecError(
+                f"[[rollout]] {ro.name!r}: train={ro.train!r} names no [[train]] — defined: {sorted(train_names)}"
+            )
+    rollout_names = {r.name for r in rollouts}
+    for an in analyses:
+        if not an.rollouts:
+            raise WorkflowSpecError(f"[[analysis]] {an.name!r}: 'rollouts' must name at least one [[rollout]]")
+        for r in an.rollouts:
+            if r not in rollout_names:
+                raise WorkflowSpecError(
+                    f"[[analysis]] {an.name!r}: rollout {r!r} is not defined — "
+                    f"defined: {sorted(rollout_names) or '(none)'}"
+                )
+        if len(set(an.rollouts)) != len(an.rollouts):
+            raise WorkflowSpecError(f"[[analysis]] {an.name!r}: repeated rollout name(s) in 'rollouts'")
+        if an.chunks < 1:
+            raise WorkflowSpecError(f"[[analysis]] {an.name!r}: chunks must be >= 1, got {an.chunks}")
+
+    for t in trains:
+        if t.epochs < 1:
+            raise WorkflowSpecError(f"[[train]] {t.name!r}: epochs must be >= 1, got {t.epochs}")
+        if t.epochs_per_job < 1:
+            raise WorkflowSpecError(f"[[train]] {t.name!r}: epochs_per_job must be >= 1, got {t.epochs_per_job}")
+
+    return WorkflowSpec(
+        name=wf["name"],
+        result_dir=result_dir,
+        log_dir=log_dir,
+        condor=condor,
+        dataset=dataset,
+        geometry=geometry,
+        trains=trains,
+        rollouts=rollouts,
+        analyses=analyses,
+        path=str(path),
+    )
+
+
+def _check_unique(items, kind: str) -> None:
+    names = [i.name for i in items]
+    dupes = sorted({n for n in names if names.count(n) > 1})
+    if dupes:
+        raise WorkflowSpecError(f"[[{kind}]] names must be unique — repeated: {dupes}")
+
+
+def epoch_milestones(train: TrainSpec) -> list[int]:
+    """Cumulative epoch counts, one per chained ``TrainEpochTask``.
+
+    ``epochs_per_job`` trades queue waits against job length: with
+    ``epochs=10, epochs_per_job=3`` this is ``[3, 6, 9, 10]``, i.e. job *k*
+    resumes job *k-1*'s ``last.pt`` and trains up to its own milestone.
+    """
+    step = train.epochs_per_job
+    milestones = list(range(step, train.epochs + 1, step))
+    if not milestones or milestones[-1] != train.epochs:
+        milestones.append(train.epochs)
+    return milestones

giant/workflow/tasks.py

@@ -0,0 +1,647 @@
+"""The b2luigi task graph: cache-warm -> train -> rollout -> analysis.
+
+    DatasetTask (external)  ─┬─> WarmCacheTask(train) ─> TrainEpochTask(train, 1..N) ─> TrainTask(train) ─┐
+                             └─> GeometryOracleTask ──┐                                                   │
+                                                      └──> RolloutTask(rollout) <────────────────────────┘
+                                                                       │
+    AnalysisPrepTask(analysis) ─> AnalysisComputeTask(analysis, plot, chunk) ─> AnalysisRenderTask(analysis)
+                                                                                        ^
+    WorkflowTask (wrapper) ─────────────────────────────────────────────────────────────┘
+
+Every task's output directory is ``<result_dir>/<kind>/name=<name>/spec_hash=
+<hash>/…`` — the hash covers the task's resolved sub-spec *and its transitive
+parents* (``giant/workflow/spec.py``), so editing the spec produces a fresh
+directory for exactly the affected subtree instead of silently reusing stale
+outputs.
+
+Task bodies never reimplement anything: they call the same entry points the
+CLIs do (``run_warm_setup_cache``, ``run_build_geometry_oracle``,
+``run_train_job``, ``giant.analysis.prep``/``compute_one``/``merge_all``,
+``render_run``), or shell out to ``giant rollout``, which has no library-level
+entry point of its own.
+
+Training is fanned out into **one short GPU job per epoch** (or per
+``epochs_per_job`` epochs): job *k* runs ``run_train_job`` with ``epochs = k``
+and ``resume = <job k-1>/last.pt``, which the training loop already handles
+(``giant/training/loop.py`` sets ``start_epoch = ckpt["epoch"] + 1`` and
+returns early when the checkpoint already covers ``epochs``). A 200-epoch run
+then becomes 200 schedulable jobs that survive preemption and give luigi a
+real progress signal, at the cost of one (cache-warmed) setup scan and one
+queue wait per job.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import b2luigi
+
+from giant.workflow.htcondor import cpu_settings, gpu_settings
+from giant.workflow.spec import WorkflowSpec, epoch_milestones
+
+__all__ = [
+    "AnalysisComputeTask",
+    "AnalysisPrepTask",
+    "AnalysisRenderTask",
+    "DatasetTask",
+    "GeometryOracleTask",
+    "RolloutTask",
+    "TrainEpochTask",
+    "TrainTask",
+    "WarmCacheTask",
+    "WorkflowTask",
+    "analysis_dir",
+    "analysis_jobs",
+    "get_spec",
+    "set_spec",
+]
+
+
+# ---------------------------------------------------------------------------
+# the active spec
+# ---------------------------------------------------------------------------
+# luigi parameters must be simple scalars, so tasks carry only `name` +
+# `spec_hash` and read the rest out of the one spec this process was started
+# with. Batch workers re-execute `run.py --spec <same file>` (see
+# `task_cmd_additional_args` there), so they resolve the identical spec.
+
+_SPEC: WorkflowSpec | None = None
+
+
+def set_spec(spec: WorkflowSpec) -> None:
+    global _SPEC
+    _SPEC = spec
+
+
+def get_spec() -> WorkflowSpec:
+    if _SPEC is None:
+        raise RuntimeError("no workflow spec loaded — call giant.workflow.tasks.set_spec() first")
+    return _SPEC
+
+
+def _result_dir(*parts: str) -> Path:
+    return Path(get_spec().result_dir).joinpath(*parts)
+
+
+def _task_dir(kind: str, name: str, spec_hash: str) -> Path:
+    """``<result_dir>/<kind>/name=<name>/spec_hash=<hash>``."""
+    return _result_dir(kind, f"name={name}", f"spec_hash={spec_hash}")
+
+
+def analysis_dir(spec: WorkflowSpec, name: str) -> Path:
+    """The analysis run directory — what ``prep`` lays out and every later step reads."""
+    return Path(spec.result_dir) / "analysis" / f"name={name}" / f"spec_hash={spec.analysis_hash(name)}"
+
+
+def analysis_jobs(spec: WorkflowSpec, name: str) -> list[tuple[str, int]]:
+    """Every ``(plot_id, chunk)`` compute job of one analysis.
+
+    ``chunkable=False`` specs (the checkpoint-bound diagnostics, already
+    bounded/subsampled) always run as a single chunk — the same rule the
+    deleted ``_job_walltimes`` applied.
+    """
+    from giant.analysis.catalog import catalog_ids, get_spec as get_plot_spec
+
+    chunks = spec.analysis(name).chunks
+    jobs: list[tuple[str, int]] = []
+    for plot_id in catalog_ids():
+        n = chunks if get_plot_spec(plot_id).chunkable else 1
+        jobs.extend((plot_id, chunk) for chunk in range(n))
+    return jobs
+
+
+def _giant_cmd() -> list[str]:
+    """How to invoke the ``giant`` CLI from inside a task (worker or locally)."""
+    return [sys.executable, "-m", "giant.cli"]
+
+
+# ---------------------------------------------------------------------------
+# inputs
+# ---------------------------------------------------------------------------
+
+
+class DatasetTask(b2luigi.ExternalTask):
+    """A steps parquet file or directory that must already exist.
+
+    Nothing produces it, so a missing path is a hard, immediate error rather
+    than a job that fails hours later — the usual cause being ``/ceph`` not
+    mounted on the machine the workflow was started from.
+    """
+
+    path = b2luigi.Parameter()
+
+    def output(self):
+        return b2luigi.LocalTarget(str(self.path))
+
+    def complete(self):
+        if not Path(str(self.path)).exists():
+            raise FileNotFoundError(
+                f"dataset {self.path!r} does not exist — is /ceph mounted on this machine? "
+                "(see CLAUDE.md's Compute environment section)"
+            )
+        return True
+
+
+# ---------------------------------------------------------------------------
+# setup stage
+# ---------------------------------------------------------------------------
+
+
+class WarmCacheTask(b2luigi.Task):
+    """Precompute one training's setup-stage sidecar (vocab maps, event split,
+    normalizer stats) so every per-epoch job is a cache hit instead of a
+    full rescan.
+
+    The real product (``<data>.giant_train_cache.json``) lives next to the
+    dataset, not under ``result_dir``, so the target here is a small stamp
+    recording that sidecar's path/mtime/size.
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+
+    @property
+    def htcondor_settings(self):
+        spec = get_spec()
+        return cpu_settings(spec.condor, request_memory_mb=32768, request_cpus=4, walltime_s=21600)
+
+    def requires(self):
+        yield DatasetTask(path=get_spec().dataset.steps)
+
+    def output(self):
+        return b2luigi.LocalTarget(str(_task_dir("warm_cache", str(self.name), str(self.spec_hash)) / "stamp.json"))
+
+    def run(self):
+        from giant.data.setup_cache import sidecar_path
+        from giant.tools.warm_setup_cache import run_warm_setup_cache
+
+        spec = get_spec()
+        train = spec.train(str(self.name))
+        run_warm_setup_cache(
+            data=spec.dataset.steps,
+            config_path=Path(train.config) if train.config else None,
+        )
+        sidecar = Path(sidecar_path(spec.dataset.steps))
+        stamp = {
+            "sidecar": str(sidecar),
+            "mtime": sidecar.stat().st_mtime if sidecar.exists() else None,
+            "size": sidecar.stat().st_size if sidecar.exists() else None,
+        }
+        out = Path(self.output().path)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(json.dumps(stamp, indent=2))
+
+
+class GeometryOracleTask(b2luigi.Task):
+    """Build the position -> (material, layer_id) oracle every rollout needs."""
+
+    spec_hash = b2luigi.Parameter()
+
+    @property
+    def htcondor_settings(self):
+        spec = get_spec()
+        return cpu_settings(spec.condor, request_memory_mb=32768, request_cpus=4, walltime_s=21600)
+
+    def requires(self):
+        yield DatasetTask(path=get_spec().dataset.steps)
+
+    def output(self):
+        return b2luigi.LocalTarget(
+            str(_result_dir("geometry", f"spec_hash={self.spec_hash}") / "oracle.pkl"),
+        )
+
+    def run(self):
+        from giant.tools.geometry_oracle import run_build_geometry_oracle
+
+        spec = get_spec()
+        g = spec.geometry
+        out = Path(self.output().path)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        run_build_geometry_oracle(
+            data=Path(spec.dataset.steps),
+            out=out,
+            method=g.method,
+            k=g.k,
+            subsample=g.subsample,
+            escape_factor=g.escape_factor,
+            seed=g.seed,
+            depth_axis=g.depth_axis,
+            n_bins=g.n_bins,
+        )
+
+
+# ---------------------------------------------------------------------------
+# training
+# ---------------------------------------------------------------------------
+
+
+def _train_cfg(spec: WorkflowSpec, name: str, epochs: int) -> dict:
+    """The merged config one training job runs, resolved exactly as `giant train` does."""
+    from giant import config as gconfig
+
+    train = spec.train(name)
+    flags = {**train.overrides, "epochs": epochs}
+    overrides = gconfig.overrides_from_flags(flags)
+    cfg = gconfig.merge_cli_overrides(
+        gconfig.DEFAULT_CONFIG,
+        Path(train.config) if train.config else None,
+        overrides,
+    )
+    gconfig.validate_config(cfg, resume=True)
+    return cfg
+
+
+class TrainEpochTask(b2luigi.Task):
+    """Epochs up to ``milestone`` of one training, resuming the previous job.
+
+    Target is ``last.pt``. ``best.pt`` is written by the loop *only when that
+    epoch improved*, and ``best_val_loss`` travels inside the checkpoint, so
+    the global best comparison stays correct across jobs: "``best.pt`` exists
+    in milestone dir *k*" means exactly "one of that job's epochs was the best
+    so far".
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+    milestone = b2luigi.IntParameter()
+
+    @property
+    def htcondor_settings(self):
+        spec = get_spec()
+        train = spec.train(str(self.name))
+        return gpu_settings(
+            spec.condor,
+            request_gpus=train.request_gpus,
+            gpu_type=train.gpu_type,
+            gpu_memory_mb=train.gpu_memory_mb,
+            request_memory_mb=train.request_memory_mb,
+            request_cpus=train.request_cpus,
+            walltime_s=train.walltime_s,
+        )
+
+    @property
+    def _dir(self) -> Path:
+        return _task_dir("train_epoch", str(self.name), str(self.spec_hash)) / f"epochs={int(self.milestone)}"
+
+    def _previous_milestone(self) -> int | None:
+        spec = get_spec()
+        milestones = epoch_milestones(spec.train(str(self.name)))
+        index = milestones.index(int(self.milestone))
+        return milestones[index - 1] if index > 0 else None
+
+    def requires(self):
+        previous = self._previous_milestone()
+        if previous is None:
+            yield WarmCacheTask(name=self.name, spec_hash=get_spec().warm_cache_hash(str(self.name)))
+        else:
+            yield TrainEpochTask(name=self.name, spec_hash=self.spec_hash, milestone=previous)
+
+    def output(self):
+        return b2luigi.LocalTarget(str(self._dir / "last.pt"))
+
+    def run(self):
+        import torch
+
+        from giant import config as gconfig
+        from giant.pipeline import run_train_job
+
+        spec = get_spec()
+        train = spec.train(str(self.name))
+        cfg = _train_cfg(spec, str(self.name), int(self.milestone))
+
+        previous = self._previous_milestone()
+        resume = None
+        if previous is not None:
+            resume = _task_dir("train_epoch", str(self.name), str(self.spec_hash)) / f"epochs={previous}" / "last.pt"
+
+        device = torch.device(train.device) if train.device else gconfig.auto_device()
+        out_dir = self._dir
+        out_dir.mkdir(parents=True, exist_ok=True)
+        run_train_job(
+            data=Path(spec.dataset.steps),
+            cfg=cfg,
+            out_dir=out_dir,
+            device=device,
+            shuffle_buffer=train.shuffle_buffer,
+            num_workers=train.num_workers,
+            resume=resume,
+            cache_setup=True,
+        )
+
+
+class TrainTask(b2luigi.Task):
+    """Publish one training's canonical outputs, hiding the epoch fan-out.
+
+    Everything downstream (``RolloutTask``, humans, ``giant analyze metrics``)
+    points here and never has to know which milestone directory happened to
+    hold the best checkpoint.
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+    batch_system = "local"
+
+    @property
+    def _milestones(self) -> list[int]:
+        return epoch_milestones(get_spec().train(str(self.name)))
+
+    def requires(self):
+        yield TrainEpochTask(name=self.name, spec_hash=self.spec_hash, milestone=self._milestones[-1])
+
+    @property
+    def _dir(self) -> Path:
+        return _task_dir("train", str(self.name), str(self.spec_hash))
+
+    def output(self):
+        d = self._dir
+        return {
+            "best.pt": b2luigi.LocalTarget(str(d / "best.pt")),
+            "last.pt": b2luigi.LocalTarget(str(d / "last.pt")),
+            "metrics.csv": b2luigi.LocalTarget(str(d / "metrics.csv")),
+        }
+
+    def run(self):
+        epoch_base = _task_dir("train_epoch", str(self.name), str(self.spec_hash))
+        milestone_dirs = [epoch_base / f"epochs={m}" for m in self._milestones]
+
+        best_dirs = [d for d in milestone_dirs if (d / "best.pt").exists()]
+        if not best_dirs:
+            raise FileNotFoundError(
+                f"no best.pt in any milestone directory under {epoch_base} — "
+                "did every epoch job run with a validation split?"
+            )
+        out = self._dir
+        out.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(best_dirs[-1] / "best.pt", out / "best.pt")
+        shutil.copy2(milestone_dirs[-1] / "last.pt", out / "last.pt")
+        for extra in ("config.toml", "run_meta.json"):
+            src = milestone_dirs[-1] / extra
+            if src.exists():
+                shutil.copy2(src, out / extra)
+
+        # One metrics.csv for the whole run: the first job's header, then
+        # every job's rows in epoch order, so `giant analyze metrics` sees a
+        # single continuous training curve.
+        lines: list[str] = []
+        header: str | None = None
+        for d in milestone_dirs:
+            csv = d / "metrics.csv"
+            if not csv.exists():
+                continue
+            rows = csv.read_text().splitlines()
+            if not rows:
+                continue
+            if header is None:
+                header = rows[0]
+            lines.extend(rows[1:])
+        (out / "metrics.csv").write_text("\n".join([header or ""] + lines) + "\n")
+
+
+# ---------------------------------------------------------------------------
+# rollout
+# ---------------------------------------------------------------------------
+
+
+class RolloutTask(b2luigi.Task):
+    """Roll one trained checkpoint forward into full showers.
+
+    ``giant rollout`` has no library-level entry point, so this shells out to
+    the CLI — with an explicit ``--out``, which puts the YAML sidecar at the
+    deterministic ``rollout.yaml`` next to the parquet (see
+    ``giant/cli.py:_write_prediction_ref``).
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+
+    @property
+    def htcondor_settings(self):
+        spec = get_spec()
+        ro = spec.rollout(str(self.name))
+        return gpu_settings(
+            spec.condor,
+            request_gpus=ro.request_gpus,
+            gpu_type=ro.gpu_type,
+            gpu_memory_mb=ro.gpu_memory_mb,
+            request_memory_mb=ro.request_memory_mb,
+            request_cpus=ro.request_cpus,
+            walltime_s=ro.walltime_s,
+        )
+
+    @property
+    def _dir(self) -> Path:
+        return _task_dir("rollout", str(self.name), str(self.spec_hash))
+
+    def requires(self):
+        spec = get_spec()
+        ro = spec.rollout(str(self.name))
+        yield TrainTask(name=ro.train, spec_hash=spec.train_hash(ro.train))
+        yield GeometryOracleTask(spec_hash=spec.geometry_hash())
+        yield DatasetTask(path=spec.dataset.reference)
+
+    def output(self):
+        d = self._dir
+        return {
+            "rollout.parquet": b2luigi.LocalTarget(str(d / "rollout.parquet")),
+            "rollout.yaml": b2luigi.LocalTarget(str(d / "rollout.yaml")),
+        }
+
+    def run(self):
+        spec = get_spec()
+        ro = spec.rollout(str(self.name))
+        out = self._dir / "rollout.parquet"
+        out.parent.mkdir(parents=True, exist_ok=True)
+
+        checkpoint = _task_dir("train", ro.train, spec.train_hash(ro.train)) / "best.pt"
+        oracle = _result_dir("geometry", f"spec_hash={spec.geometry_hash()}") / "oracle.pkl"
+
+        cmd = [
+            *_giant_cmd(),
+            "rollout",
+            spec.dataset.reference,
+            "--checkpoint",
+            str(checkpoint),
+            "--geometry",
+            str(oracle),
+            "--out",
+            str(out),
+            "--energy-cutoff",
+            str(ro.energy_cutoff),
+            "--max-steps",
+            str(ro.max_steps),
+            "--steps",
+            str(ro.steps),
+            "--batch-size",
+            str(ro.batch_size),
+            "--weights",
+            ro.weights,
+        ]
+        for flag, value in (
+            ("--n-events", ro.n_events),
+            ("--max-tracks-per-event", ro.max_tracks_per_event),
+            ("--escape-threshold", ro.escape_threshold),
+            ("--seed", ro.seed),
+            ("--device", ro.device),
+        ):
+            if value is not None:
+                cmd += [flag, str(value)]
+        subprocess.run(cmd, check=True)
+
+
+# ---------------------------------------------------------------------------
+# analysis
+# ---------------------------------------------------------------------------
+
+
+class AnalysisPrepTask(b2luigi.Task):
+    """Resolve the shared bin edges/group sets once, for every compute job.
+
+    Cheap and streaming, so it runs locally: everything after it needs
+    ``shared.json``/``run_meta.json`` to already exist.
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+    batch_system = "local"
+
+    def requires(self):
+        spec = get_spec()
+        for rollout_name in spec.analysis(str(self.name)).rollouts:
+            yield RolloutTask(name=rollout_name, spec_hash=spec.rollout_hash(rollout_name))
+
+    @property
+    def _dir(self) -> Path:
+        return analysis_dir(get_spec(), str(self.name))
+
+    def output(self):
+        d = self._dir
+        return {
+            "shared.json": b2luigi.LocalTarget(str(d / "shared.json")),
+            "run_meta.json": b2luigi.LocalTarget(str(d / "run_meta.json")),
+        }
+
+    def run(self):
+        from giant.analysis import prep
+
+        spec = get_spec()
+        an = spec.analysis(str(self.name))
+        yamls = [_task_dir("rollout", r, spec.rollout_hash(r)) / "rollout.yaml" for r in an.rollouts]
+        prep(
+            yamls,
+            run_dir=self._dir,
+            n_chunks=an.chunks,
+            labels=list(an.rollouts),
+            n_energy_bins=an.energy_bins,
+            n_marginal_bins=an.bins,
+            top_k_pdg=an.top_pdg,
+        )
+
+
+class AnalysisComputeTask(b2luigi.Task):
+    """One (plot, chunk) streaming reduction — the replaced ``jobs.txt`` row.
+
+    The output path is the on-disk contract ``compute-one``/``merge_one``
+    already share (``reduced_partial/<id>__<chunk>.json``), declared
+    explicitly rather than through b2luigi's own output naming so that
+    contract is untouched.
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+    plot_id = b2luigi.Parameter()
+    chunk = b2luigi.IntParameter()
+
+    @property
+    def htcondor_settings(self):
+        # A property, so it is evaluated at submit time — i.e. after prep has
+        # written run_meta.json, whose row counts size the walltime request.
+        from giant.analysis import RunMeta
+        from giant.analysis.runtime_estimate import estimate_runtime_s
+
+        spec = get_spec()
+        an = spec.analysis(str(self.name))
+        walltime = None
+        meta_path = analysis_dir(spec, str(self.name)) / "run_meta.json"
+        if meta_path.exists():
+            from giant.analysis.catalog import get_spec as get_plot_spec
+
+            meta = RunMeta.load(meta_path)
+            chunkable = get_plot_spec(str(self.plot_id)).chunkable
+            n_rows = meta.rows_per_chunk[int(self.chunk)] if chunkable and meta.rows_per_chunk else meta.total_rows
+            walltime = estimate_runtime_s(str(self.plot_id), n_rows)
+        return cpu_settings(
+            spec.condor,
+            request_memory_mb=an.request_memory_mb,
+            request_cpus=an.request_cpus,
+            walltime_s=walltime,
+        )
+
+    def requires(self):
+        yield AnalysisPrepTask(name=self.name, spec_hash=self.spec_hash)
+
+    def output(self):
+        run_dir = analysis_dir(get_spec(), str(self.name))
+        return b2luigi.LocalTarget(str(run_dir / "reduced_partial" / f"{self.plot_id}__{int(self.chunk)}.json"))
+
+    def run(self):
+        from giant.analysis import compute_one
+
+        compute_one(str(self.plot_id), analysis_dir(get_spec(), str(self.name)), chunk_index=int(self.chunk))
+
+
+class AnalysisRenderTask(b2luigi.Task):
+    """Merge every plot's chunk partials, then render the PDFs + gallery.
+
+    Always local — this is the only step that imports plotstyle/LaTeX, which
+    the compute worker images don't have.
+    """
+
+    name = b2luigi.Parameter()
+    spec_hash = b2luigi.Parameter()
+    batch_system = "local"
+
+    def requires(self):
+        spec = get_spec()
+        for plot_id, chunk in analysis_jobs(spec, str(self.name)):
+            yield AnalysisComputeTask(
+                name=self.name,
+                spec_hash=self.spec_hash,
+                plot_id=plot_id,
+                chunk=chunk,
+            )
+
+    def output(self):
+        run_dir = analysis_dir(get_spec(), str(self.name))
+        return b2luigi.LocalTarget(str(run_dir / "plots" / "metadata.yaml"))
+
+    def run(self):
+        # render_run joins every plot's chunk partials (merge_all) before
+        # rendering, so this one call is the whole merge+render step.
+        from giant.analysis.render import render_run
+
+        spec = get_spec()
+        render_run(analysis_dir(spec, str(self.name)), run_gallery=spec.analysis(str(self.name)).gallery)
+
+
+class WorkflowTask(b2luigi.WrapperTask):
+    """The whole pipeline: every analysis in the spec, rendered."""
+
+    workflow_name = b2luigi.Parameter()
+
+    def requires(self):
+        spec = get_spec()
+        if not spec.analyses:
+            # A spec with no [[analysis]] still has work to do — fall back to
+            # the deepest tasks it does define.
+            for ro in spec.rollouts:
+                yield RolloutTask(name=ro.name, spec_hash=spec.rollout_hash(ro.name))
+            if not spec.rollouts:
+                for tr in spec.trains:
+                    yield TrainTask(name=tr.name, spec_hash=spec.train_hash(tr.name))
+            return
+        for an in spec.analyses:
+            yield AnalysisRenderTask(name=an.name, spec_hash=spec.analysis_hash(an.name))

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "giant"
-version = "0.3.6"
+version = "0.3.10"
 description = "Geant4 step-function surrogate via conditional flow matching"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dev = [
     "ty>=0.0.50,<0.1",
     "bump-my-version>=1.2,<2",
     "git-cliff>=2,<3",
-    "giant[convert,analysis,geometry,wandb]",
+    "giant[convert,analysis,geometry,wandb,workflow]",
 ]
 geometry = [
     "scikit-learn>=1.4,<2",
@@ -49,6 +49,11 @@ analysis = [
     # `giant analyze render` step imports it; compute workers never do.
     "plotstyle>=1.0.0",
 ]
+# b2luigi pulls luigi + tenacity; the only sanctioned way to chain a
+# multi-step pipeline (see giant/workflow/).
+workflow = [
+    "b2luigi>=1.0,<2",
+]
 
 [project.scripts]
 giant = "giant.cli:app"
@@ -103,3 +108,12 @@ explicit = true
 name = "larsbogner"
 url = "https://git.larsbogner.de/api/packages/lars/pypi/simple/"
 explicit = true
+
+# luigi builds task constructors from class-level Parameter descriptors, so a
+# static checker sees no keyword parameters at all on `Task(name=..., ...)`.
+# The workflow code is written against that API; nothing else in the repo is.
+[[tool.ty.overrides]]
+include = ["giant/workflow/**", "tests/test_workflow_tasks.py"]
+
+[tool.ty.overrides.rules]
+unknown-argument = "ignore"

tests/test_analysis_reduce.py

@@ -159,6 +159,20 @@ def test_secondaries_rollout_vs_reference_align():
     assert t["pdg"].to_list() == [22, 22]
 
 
+def test_sec_count_by_event_zero_fills_events_with_no_secondaries():
+    r_phys = physical_steps(_rollout_frame(), Side.rollout)
+    r_sec = secondaries(_rollout_frame(), Side.rollout)
+    ev, n = R.sec_count_by_event(r_phys, r_sec)
+    # event 1 has one secondary track; event 2 has none and must still appear (as 0),
+    # not silently drop out of a plain group_by on the secondaries frame alone.
+    assert dict(zip(ev.tolist(), n.tolist())) == {1: 1, 2: 0}
+
+    t_all = _reference_frame()
+    t_sec = secondaries(t_all, Side.reference)
+    ev, n = R.sec_count_by_event(t_all, t_sec)
+    assert dict(zip(ev.tolist(), n.tolist())) == {1: 1, 2: 1}
+
+
 def test_leakage_fraction():
     frac = R.leakage_fraction(_rollout_frame())
     # event 1: escaped pre_E=30, deposited=90 -> 30/120 = 0.25; event 2: 0

tests/test_analysis_run.py

@@ -1,8 +1,7 @@
-"""Tests for the rollout-YAML → run-directory flow, compute, and submit."""
+"""Tests for the rollout-YAML(s) → run-directory flow, compute, and merge."""
 
 from __future__ import annotations
 
-import sys
 from pathlib import Path
 
 import pyarrow.parquet as pq
@@ -11,30 +10,31 @@ import yaml
 
 from giant.analysis import (
     RunMeta,
-    SubmitConfig,
-    catalog_ids,
     compute_one,
     compute_reduced,
     derive_run_dir,
     load_rollout_yaml,
+    load_rollout_yamls,
     merge_one,
     prep,
-    write_submit,
 )
-from giant.analysis.catalog import get_spec
-from giant.analysis.condor import Context
+from giant.analysis.run import Context
 from giant.analysis.reduced import Partial, Reduced
 from giant.constants import PREDICT_COORD_METADATA_KEY, ROLLOUT_COORD_VALUE
 from tests.test_analysis_reduce import _reference_frame, _rollout_frame
 
 
+def _write_rollout(path: Path) -> None:
+    tbl = _rollout_frame().collect().to_arrow()
+    tbl = tbl.replace_schema_metadata({PREDICT_COORD_METADATA_KEY: ROLLOUT_COORD_VALUE})
+    pq.write_table(tbl, path)
+
+
 def _write_inputs(tmp_path: Path) -> Path:
     """Materialize rollout+reference parquet and a rollout YAML; return the YAML path."""
     rollout = tmp_path / "rollout.parquet"
     reference = tmp_path / "reference.parquet"
-    tbl = _rollout_frame().collect().to_arrow()
-    tbl = tbl.replace_schema_metadata({PREDICT_COORD_METADATA_KEY: ROLLOUT_COORD_VALUE})
-    pq.write_table(tbl, rollout)
+    _write_rollout(rollout)
     _reference_frame().collect().write_parquet(reference)
 
     yaml_path = tmp_path / "run.yaml"
@@ -54,20 +54,40 @@ def _write_inputs(tmp_path: Path) -> Path:
     return yaml_path
 
 
-def _fake_venv(repo_dir: Path) -> None:
-    """Stand in for a `uv sync`'d venv: write_submit checks `.venv/bin/giant` exists."""
-    giant = repo_dir / ".venv" / "bin" / "giant"
-    giant.parent.mkdir(parents=True, exist_ok=True)
-    giant.write_text("#!/bin/bash\n")
-    giant.chmod(0o755)
+def _write_two_inputs(tmp_path: Path) -> tuple[Path, Path]:
+    """Two rollout YAMLs (distinct output files) sharing one reference file."""
+    reference = tmp_path / "reference.parquet"
+    _reference_frame().collect().write_parquet(reference)
+
+    paths = []
+    for tag, pred_id in (("a", "aaaa1111ef"), ("b", "bbbb2222ef")):
+        rollout = tmp_path / f"rollout_{tag}.parquet"
+        _write_rollout(rollout)
+        yaml_path = tmp_path / f"run_{tag}.yaml"
+        yaml_path.write_text(
+            yaml.safe_dump(
+                {
+                    "prediction_id": pred_id,
+                    "output": str(rollout),
+                    "dataset": str(reference),
+                    "checkpoint": f"/ckpt/{tag}.pt",
+                    "kind": "rollout",
+                    "energy_cutoff": 0.1,
+                    "steps": 10,
+                }
+            )
+        )
+        paths.append(yaml_path)
+    return paths[0], paths[1]
 
 
-def _prep(rollout_yaml: Path, run_dir: str | Path | None = None, chunks: int = 1) -> Path:
+def _prep(rollout_yamls, run_dir: str | Path | None = None, chunks: int = 1, labels=None) -> Path:
     """``prep`` with small test-sized context bins/sampling."""
     return prep(
-        rollout_yaml,
+        rollout_yamls,
         run_dir,
         n_chunks=chunks,
+        labels=labels,
         n_energy_bins=2,
         n_marginal_bins=8,
         top_k_pdg=3,
@@ -82,39 +102,108 @@ def test_load_rollout_yaml_requires_paths(tmp_path: Path):
         load_rollout_yaml(bad)
 
 
+def test_load_rollout_yamls_single_defaults_to_rollout_name(tmp_path: Path):
+    yaml_path = _write_inputs(tmp_path)
+    loaded, reference = load_rollout_yamls([yaml_path])
+    assert [lr.name for lr in loaded] == ["rollout"]
+    assert reference.endswith("reference.parquet")
+
+
+def test_load_rollout_yamls_multi_defaults_to_stem(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    loaded, _ = load_rollout_yamls([a, b])
+    assert [lr.name for lr in loaded] == ["run_a", "run_b"]
+
+
+def test_load_rollout_yamls_explicit_labels(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    loaded, _ = load_rollout_yamls([a, b], labels=["flow", "wgan"])
+    assert [lr.name for lr in loaded] == ["flow", "wgan"]
+
+
+def test_load_rollout_yamls_label_count_mismatch(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    with pytest.raises(ValueError, match="--label"):
+        load_rollout_yamls([a, b], labels=["only-one"])
+
+
+def test_load_rollout_yamls_rejects_duplicate_names(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    with pytest.raises(ValueError, match="collide"):
+        load_rollout_yamls([a, b], labels=["same", "same"])
+
+
+def test_load_rollout_yamls_rejects_mismatched_reference(tmp_path: Path):
+    a, _ = _write_two_inputs(tmp_path)
+    other_ref = tmp_path / "other_reference.parquet"
+    _reference_frame().collect().write_parquet(other_ref)
+    c = tmp_path / "run_c.yaml"
+    c.write_text(
+        yaml.safe_dump(
+            {"prediction_id": "cccc3333ef", "output": str(tmp_path / "rollout_c.parquet"), "dataset": str(other_ref)}
+        )
+    )
+    _write_rollout(tmp_path / "rollout_c.parquet")
+    with pytest.raises(ValueError, match="same reference"):
+        load_rollout_yamls([a, c])
+
+
 def test_derive_run_dir_next_to_rollout():
     y = {"output": "/data/roll.parquet", "prediction_id": "abcd1234ef", "dataset": "d"}
-    assert derive_run_dir(y) == Path("/data/analysis_abcd1234")
-    assert derive_run_dir(y, "/somewhere") == Path("/somewhere")
+    assert derive_run_dir([y]) == Path("/data/analysis_abcd1234")
+    assert derive_run_dir([y], "/somewhere") == Path("/somewhere")
 
 
 def test_derive_run_dir_default_base():
     y = {"output": "/data/roll.parquet", "prediction_id": "abcd1234ef", "dataset": "d"}
-    assert derive_run_dir(y, default_base="/work/lbogner/giant2/analysis_runs") == Path(
+    assert derive_run_dir([y], default_base="/work/lbogner/giant2/analysis_runs") == Path(
         "/work/lbogner/giant2/analysis_runs/analysis_abcd1234"
     )
     # an explicit run_dir still wins over default_base
-    assert derive_run_dir(y, "/somewhere", default_base="/other") == Path("/somewhere")
+    assert derive_run_dir([y], "/somewhere", default_base="/other") == Path("/somewhere")
+
+
+def test_derive_run_dir_multi_rollout_joins_tags():
+    ys = [{"output": f"/data/roll_{i}.parquet", "prediction_id": f"tag{i}xxxx", "dataset": "d"} for i in range(2)]
+    assert derive_run_dir(ys, default_base="/base") == Path("/base/analysis_tag0xxxx-tag1xxxx")
+
+
+def test_derive_run_dir_many_rollouts_truncates_with_plus_count():
+    ys = [{"output": f"/data/roll_{i}.parquet", "prediction_id": f"tag{i}xxxx", "dataset": "d"} for i in range(5)]
+    run_dir = derive_run_dir(ys, default_base="/base")
+    assert run_dir == Path("/base/analysis_tag0xxxx-tag1xxxx-tag2xxxx-plus2")
 
 
 def test_prep_lays_out_run_dir(tmp_path: Path):
     yaml_path = _write_inputs(tmp_path)
-    run_dir = _prep(yaml_path)
+    run_dir = _prep([yaml_path])
     assert run_dir == tmp_path / "analysis_abcd1234"
     assert (run_dir / "shared.json").exists()
     ctx = Context.load(run_dir / "shared.json")
     assert set(ctx.var_ranges) == {"step_length", "edep", "delta_e", "post_E"}
     meta = RunMeta.load(run_dir / "run_meta.json")
     assert meta.reference.endswith("reference.parquet")
-    assert meta.plot_meta["checkpoint"] == "/ckpt/best.pt"
+    assert [ro["name"] for ro in meta.rollouts] == ["rollout"]
+    assert meta.rollouts[0]["plot_meta"]["checkpoint"] == "/ckpt/best.pt"
     assert "best.pt" in meta.title
     assert meta.n_chunks == 1
     assert meta.rows_per_chunk == [meta.total_rows]  # single chunk holds everything
     assert meta.total_rows == 8  # 5 rollout rows + 3 reference rows
 
 
+def test_prep_multi_rollout_lays_out_run_dir(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    run_dir = _prep([a, b], labels=["flow", "wgan"])
+    meta = RunMeta.load(run_dir / "run_meta.json")
+    assert [ro["name"] for ro in meta.rollouts] == ["flow", "wgan"]
+    assert meta.rollouts[0]["plot_meta"]["checkpoint"] == "/ckpt/a.pt"
+    assert meta.rollouts[1]["plot_meta"]["checkpoint"] == "/ckpt/b.pt"
+    # 5 rows from each rollout + 3 from the shared reference
+    assert meta.total_rows == 13
+
+
 def test_prep_splits_rows_per_chunk(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path), chunks=2)
+    run_dir = _prep([_write_inputs(tmp_path)], chunks=2)
     meta = RunMeta.load(run_dir / "run_meta.json")
     assert len(meta.rows_per_chunk) == 2
     assert sum(meta.rows_per_chunk) == meta.total_rows == 8
@@ -125,7 +214,7 @@ def test_reprep_clears_stale_partials_from_a_different_chunk_count(tmp_path: Pat
     partials on disk for merge_one to silently merge against the new
     context (they'd be keyed/sized for the old n_chunks)."""
     yaml_path = _write_inputs(tmp_path)
-    run_dir = _prep(yaml_path, chunks=2)
+    run_dir = _prep([yaml_path], chunks=2)
     compute_one("marginal_edep", run_dir, chunk_index=0)
     compute_one("marginal_edep", run_dir, chunk_index=1)
     stale = run_dir / "reduced_partial" / "marginal_edep__0.json"
@@ -133,7 +222,7 @@ def test_reprep_clears_stale_partials_from_a_different_chunk_count(tmp_path: Pat
     (run_dir / "reduced").mkdir(exist_ok=True)
     (run_dir / "reduced" / "marginal_edep.json").write_text("{}")
 
-    _prep(yaml_path, run_dir, chunks=1)
+    _prep([yaml_path], run_dir, chunks=1)
 
     assert not stale.exists()
     assert not (run_dir / "reduced" / "marginal_edep.json").exists()
@@ -141,20 +230,22 @@ def test_reprep_clears_stale_partials_from_a_different_chunk_count(tmp_path: Pat
 
 
 def test_compute_one_from_run_dir(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))
+    run_dir = _prep([_write_inputs(tmp_path)])
     out = compute_one("marginal_edep", run_dir)
     assert out == run_dir / "reduced_partial" / "marginal_edep__0.json"
     partial = Partial.load(out)
     assert partial.id == "marginal_edep" and partial.chunk == 0
     assert "r" in partial.data and "t" in partial.data
+    assert list(partial.data["r"]) == ["rollout"]
 
 
 def test_compute_reduced_explicit_paths(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))
+    run_dir = _prep([_write_inputs(tmp_path)])
     meta = RunMeta.load(run_dir / "run_meta.json")
+    rollouts = [{"name": ro["name"], "path": ro["path"]} for ro in meta.rollouts]
     out = compute_reduced(
         "marginal_step_length",
-        meta.rollout,
+        rollouts,
         meta.reference,
         run_dir / "shared.json",
         tmp_path / "r.json",
@@ -163,17 +254,17 @@ def test_compute_reduced_explicit_paths(tmp_path: Path):
 
 
 def test_merge_one_produces_reduced(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))
+    run_dir = _prep([_write_inputs(tmp_path)])
     compute_one("marginal_edep", run_dir)
     out = merge_one("marginal_edep", run_dir)
     assert out == run_dir / "reduced" / "marginal_edep.json"
     reduced = Reduced.load(out)
     assert reduced.id == "marginal_edep"
-    assert len(reduced.payload["rollout"]) == len(reduced.payload["edges"]) - 1
+    assert len(reduced.payload["series"]["rollout"]) == len(reduced.payload["edges"]) - 1
 
 
 def test_merge_one_fails_loudly_on_missing_chunk(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path), chunks=2)
+    run_dir = _prep([_write_inputs(tmp_path)], chunks=2)
     compute_one("marginal_edep", run_dir, chunk_index=0)  # chunk 1 never computed
     with pytest.raises(FileNotFoundError, match="missing chunk"):
         merge_one("marginal_edep", run_dir)
@@ -182,11 +273,11 @@ def test_merge_one_fails_loudly_on_missing_chunk(tmp_path: Path):
 def test_chunked_compute_and_merge_matches_unchunked(tmp_path: Path):
     (tmp_path / "a").mkdir()
     (tmp_path / "b").mkdir()
-    unchunked_dir = _prep(_write_inputs(tmp_path / "a"))
+    unchunked_dir = _prep([_write_inputs(tmp_path / "a")])
     compute_one("marginal_step_length", unchunked_dir)
     unchunked = Reduced.load(merge_one("marginal_step_length", unchunked_dir))
 
-    chunked_dir = _prep(_write_inputs(tmp_path / "b"), chunks=2)
+    chunked_dir = _prep([_write_inputs(tmp_path / "b")], chunks=2)
     for k in range(2):
         compute_one("marginal_step_length", chunked_dir, chunk_index=k)
     chunked = Reduced.load(merge_one("marginal_step_length", chunked_dir))
@@ -194,79 +285,21 @@ def test_chunked_compute_and_merge_matches_unchunked(tmp_path: Path):
     assert chunked.payload == unchunked.payload
 
 
+def test_two_rollout_compute_and_merge_produces_both_series(tmp_path: Path):
+    a, b = _write_two_inputs(tmp_path)
+    run_dir = _prep([a, b], labels=["flow", "wgan"])
+    compute_one("marginal_edep", run_dir)
+    reduced = Reduced.load(merge_one("marginal_edep", run_dir))
+    assert list(reduced.payload["series"]) == ["flow", "wgan"]
+    assert "reference" in reduced.payload
+
+
 def test_compute_reduced_rejects_out_of_range_chunk(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))  # n_chunks=1 (default)
+    run_dir = _prep([_write_inputs(tmp_path)])  # n_chunks=1 (default)
     with pytest.raises(ValueError, match="out of range"):
         compute_one("marginal_edep", run_dir, chunk_index=1)
 
 
-def test_write_submit_description(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))
-    _fake_venv(tmp_path)
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path)
-    txt = write_submit(cfg).read_text()
-    assert "universe = docker" in txt
-    assert "docker_image = cverstege/alma9-gridjob" in txt
-    assert "requirements = TARGET.ProvidesETPResources" in txt
-    assert "accounting_group = cms" in txt
-    assert "+RequestWalltime = $(walltime)" in txt
-    assert "queue plotid,chunk,walltime from" in txt
-    jobs = [line.split(",") for line in (run_dir / "jobs.txt").read_text().split()]
-    assert [i for i, _, _ in jobs] == catalog_ids()
-    assert all(k == "0" for _, k, _ in jobs)  # n_chunks=1 default
-    assert all(int(w) > 0 for _, _, w in jobs)
-    wrapper = run_dir / "run_compute.sh"
-    assert wrapper.exists() and (wrapper.stat().st_mode & 0o111)
-    body = wrapper.read_text()
-    assert "giant analyze compute-one --id" in body
-    assert "--chunk" in body and "--run-dir" in body
-
-
-def test_write_submit_requires_synced_venv(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
-    run_dir = _prep(_write_inputs(tmp_path))
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path)
-    # No `giant` next to the (fake) active interpreter, so this falls through
-    # to repo_dir/.venv/bin/giant, which _write_inputs/_prep also didn't create.
-    monkeypatch.setattr(sys, "executable", str(tmp_path / "not-a-venv" / "bin" / "python"))
-    with pytest.raises(FileNotFoundError, match="uv sync"):
-        write_submit(cfg)
-
-
-def test_write_submit_remote_flag(tmp_path: Path):
-    run_dir = _prep(_write_inputs(tmp_path))
-    _fake_venv(tmp_path)
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path, remote=True)
-    txt = write_submit(cfg).read_text()
-    assert "+RemoteJob = True" in txt
-    assert "ProvidesETPResources" not in txt
-
-
-def test_write_submit_chunks_respect_chunkable(tmp_path: Path):
-    assert get_spec("router_gating").chunkable is False
-    run_dir = _prep(_write_inputs(tmp_path), chunks=4)
-    _fake_venv(tmp_path)
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path, n_chunks=4)
-    write_submit(cfg)
-    jobs = [line.split(",") for line in (run_dir / "jobs.txt").read_text().split()]
-    counts: dict[str, int] = {}
-    for spec_id, _, _ in jobs:
-        counts[spec_id] = counts.get(spec_id, 0) + 1
-    assert counts["marginal_edep"] == 4
-    assert counts["router_gating"] == 1  # chunkable=False, ignores n_chunks
-
-
-def test_write_submit_rejects_n_chunks_mismatch_with_run_meta(tmp_path: Path):
-    """cfg.n_chunks must match the n_chunks the run_dir was actually prepped
-    with — RunMeta.rows_per_chunk is sized to the prepped value, so a
-    mismatch would otherwise surface as a confusing IndexError deep inside
-    _job_walltimes instead of a clear error here."""
-    run_dir = _prep(_write_inputs(tmp_path), chunks=2)
-    _fake_venv(tmp_path)
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path, n_chunks=4)
-    with pytest.raises(ValueError, match="n_chunks"):
-        write_submit(cfg)
-
-
 def test_estimate_runtime_s_scales_with_rows_and_margin():
     from giant.analysis import RUNTIME_SAFETY_MARGIN, estimate_runtime_s
     from giant.analysis.runtime_estimate import _FIXED_OVERHEAD_S
@@ -276,18 +309,3 @@ def test_estimate_runtime_s_scales_with_rows_and_margin():
     large = estimate_runtime_s("marginal_edep", 100_000_000)
     assert small >= (1 + RUNTIME_SAFETY_MARGIN) * _FIXED_OVERHEAD_S
     assert large > small  # bigger chunk -> longer estimate
-
-
-def test_write_submit_walltime_grows_with_chunk_rows(tmp_path: Path):
-    """A chunked run's later job walltimes track that chunk's row count."""
-    from giant.analysis.runtime_estimate import estimate_runtime_s
-
-    run_dir = _prep(_write_inputs(tmp_path), chunks=2)
-    meta = RunMeta.load(run_dir / "run_meta.json")
-    _fake_venv(tmp_path)
-    cfg = SubmitConfig(run_dir=run_dir, accounting_group="cms", repo_dir=tmp_path, n_chunks=2)
-    write_submit(cfg)
-    jobs = {(i, int(k)): int(w) for i, k, w in (line.split(",") for line in (run_dir / "jobs.txt").read_text().split())}
-    for chunk in range(2):
-        expected = estimate_runtime_s("marginal_edep", meta.rows_per_chunk[chunk])
-        assert jobs[("marginal_edep", chunk)] == expected

tests/test_catalog.py

@@ -6,14 +6,29 @@ import numpy as np
 import pytest
 
 from giant.analysis import build_catalog, catalog_ids, get_spec
-from giant.analysis.catalog import Bundle, PlotSpec
+from giant.analysis.catalog import (
+    Bundle,
+    PlotSpec,
+    _containment_depths,
+    _integer_confusion,
+    _ks_statistic,
+)
 from giant.analysis.context import Context, build_context
+from giant.analysis.sources import RolloutSpec
 from tests.test_analysis_reduce import _reference_frame, _rollout_frame
 
 
 def _build_ctx() -> Context:
     r, t = _rollout_frame(), _reference_frame()
-    return build_context(r, t, n_energy_bins=2, n_marginal_bins=10, top_k_pdg=3, sample_rows=1000)
+    return build_context(
+        [RolloutSpec("rollout", r)], t, n_energy_bins=2, n_marginal_bins=10, top_k_pdg=3, sample_rows=1000
+    )
+
+
+def _two_rollout_specs() -> list[RolloutSpec]:
+    # Two distinct rollout sources so multi-series merging/finalize code is
+    # exercised even though the underlying frame is the same fixture.
+    return [RolloutSpec("flow", _rollout_frame()), RolloutSpec("wgan", _rollout_frame())]
 
 
 @pytest.fixture(scope="module")
@@ -21,9 +36,20 @@ def ctx() -> Context:
     return _build_ctx()
 
 
+@pytest.fixture(scope="module")
+def two_ctx() -> Context:
+    t = _reference_frame()
+    return build_context(_two_rollout_specs(), t, n_energy_bins=2, n_marginal_bins=10, top_k_pdg=3, sample_rows=1000)
+
+
 @pytest.fixture(scope="module")
 def bundle(ctx: Context) -> Bundle:
-    return Bundle.open(_rollout_frame(), _reference_frame(), ctx)
+    return Bundle.open([RolloutSpec("rollout", _rollout_frame())], _reference_frame(), ctx)
+
+
+@pytest.fixture(scope="module")
+def two_bundle(two_ctx: Context) -> Bundle:
+    return Bundle.open(_two_rollout_specs(), _reference_frame(), two_ctx)
 
 
 def test_catalog_ids_unique_and_nonempty():
@@ -53,41 +79,76 @@ def test_every_spec_computes_valid_reduced(bundle: Bundle):
             "single_hist",
             "router_gating",
             "router_share",
+            "router_specialization",
+            "heatmap",
             "unavailable",
         }
         assert r.title and r.xlabel
-        _validate_payload(r)
+        _validate_payload(r, ["rollout"])
 
 
-def _validate_payload(r) -> None:
+def test_every_spec_computes_valid_reduced_with_two_rollouts(two_bundle: Bundle):
+    for spec in build_catalog():
+        r = spec.finalize([spec.compute_partial(two_bundle)], two_bundle.ctx)
+        assert r.id == spec.id
+        _validate_payload(r, ["flow", "wgan"])
+
+
+def _validate_payload(r, names: list[str]) -> None:
     p = r.payload
     if r.kind == "overlay_hist":
         n = len(p["edges"]) - 1
-        assert len(p["rollout"]) == n and len(p["reference"]) == n
+        assert list(p["series"]) == names
+        for v in p["series"].values():
+            assert len(v) == n
+        assert len(p["reference"]) == n
     elif r.kind == "single_hist":
-        assert len(p["rollout"]) == len(p["edges"]) - 1
+        assert list(p["series"]) == names
+        for v in p["series"].values():
+            assert len(v) == len(p["edges"]) - 1
     elif r.kind == "grouped_hist":
         n = len(p["edges"]) - 1
         assert p["groups"], "grouped hist must have at least one group"
         for g in p["groups"].values():
-            assert len(g["rollout"]) == n and len(g["reference"]) == n
+            assert list(g["series"]) == names
+            for v in g["series"].values():
+                assert len(v) == n
+            assert len(g["reference"]) == n
     elif r.kind == "profile":
         n = len(p["edges"]) - 1
-        for k in ("rollout_mean", "rollout_std", "reference_mean", "reference_std"):
-            assert len(p[k]) == n
+        assert list(p["series"]) == names
+        for side in p["series"].values():
+            assert len(side["mean"]) == n and len(side["std"]) == n
+        assert len(p["reference"]["mean"]) == n and len(p["reference"]["std"]) == n
     elif r.kind == "bar":
-        assert len(p["labels"]) == len(p["rollout"]) == len(p["reference"])
+        assert list(p["series"]) == names
+        for v in p["series"].values():
+            assert len(p["labels"]) == len(v)
+        assert len(p["labels"]) == len(p["reference"])
     elif r.kind == "unavailable":
         assert p["note"]
     elif r.kind == "router_gating":
-        for side in ("rollout", "reference"):
-            if side in p:
-                assert len(p[side]["centers"]) == len(p[side]["means"])
-    elif r.kind == "router_share":
-        for cat in p["categories"]:
+        for entry in p["series"].values():
             for side in ("rollout", "reference"):
-                if side in p:
-                    assert cat in p[side]
+                if side in entry:
+                    assert len(entry[side]["centers"]) == len(entry[side]["means"])
+    elif r.kind == "router_share":
+        for entry in p["series"].values():
+            for cat in entry["categories"]:
+                for side in ("rollout", "reference"):
+                    if side in entry:
+                        assert cat in entry[side]
+    elif r.kind == "router_specialization":
+        for entry in p["series"].values():
+            for side in ("rollout", "reference"):
+                if side in entry:
+                    assert len(entry[side]["centers"]) == len(entry[side]["score"])
+    elif r.kind == "heatmap":
+        assert list(p["series"]) == names
+        for mat in p["series"].values():
+            assert len(mat) == len(p["row_labels"])
+            for row in mat:
+                assert len(row) == len(p["col_labels"])
 
 
 # ---------------------------------------------------------------------------
@@ -98,7 +159,10 @@ def _validate_payload(r) -> None:
 # sec_count_per_species via pdg-keyed sums), concat-then-finalize with
 # data-dependent edges (event_total_edep), concat-then-mean/std (shower_
 # longitudinal), concat-then-max-edge (leakage_fraction), pdg-keyed sum with a
-# ratio (species_edep_share), and a chunkable=False passthrough (router_gating).
+# ratio (species_edep_share), a chunkable=False passthrough (router_gating),
+# nested sum-merge into a scorecard (marginal_distance_summary), concat-then-
+# event-id-join (n_sec_confusion), and concat-then-per-event-derived-quantity
+# (shower_containment_depth_90, reusing the profile matrix's own merge shape).
 _CHUNK_EQUIVALENCE_IDS = [
     "marginal_edep",
     "species_edep_share",
@@ -107,6 +171,9 @@ _CHUNK_EQUIVALENCE_IDS = [
     "leakage_fraction",
     "sec_count_per_species",
     "router_gating",
+    "marginal_distance_summary",
+    "n_sec_confusion",
+    "shower_containment_depth_90",
 ]
 
 
@@ -128,21 +195,81 @@ def _assert_payload_close(a, b, path: str = "payload") -> None:
 
 
 @pytest.mark.parametrize("spec_id", _CHUNK_EQUIVALENCE_IDS)
-def test_chunked_matches_unchunked(ctx: Context, spec_id: str):
+def test_chunked_matches_unchunked(two_ctx: Context, spec_id: str):
     """A plot computed over N event-disjoint chunks then merged must equal the
     same plot computed in one unchunked pass — the core chunking correctness
-    guarantee (see the analysis-rollout-plots chunking plan)."""
+    guarantee (see the analysis-rollout-plots chunking plan). Exercised with
+    two rollout series so the per-rollout merge path is covered too."""
     spec: PlotSpec = get_spec(spec_id)
-    r, t = _rollout_frame(), _reference_frame()
+    rollouts, t = _two_rollout_specs(), _reference_frame()
 
-    unchunked_bundle = Bundle.open(r, t, ctx)
-    unchunked = spec.finalize([spec.compute_partial(unchunked_bundle)], ctx)
+    unchunked_bundle = Bundle.open(rollouts, t, two_ctx)
+    unchunked = spec.finalize([spec.compute_partial(unchunked_bundle)], two_ctx)
 
     # 4 chunks over only 2 distinct event_ids also exercises empty chunks.
     n_chunks = 4 if spec.chunkable else 1
-    parts = [spec.compute_partial(Bundle.open(r, t, ctx, chunk=(k, n_chunks))) for k in range(n_chunks)]
-    chunked = spec.finalize(parts, ctx)
+    parts = [spec.compute_partial(Bundle.open(rollouts, t, two_ctx, chunk=(k, n_chunks))) for k in range(n_chunks)]
+    chunked = spec.finalize(parts, two_ctx)
 
     assert chunked.id == unchunked.id
     assert chunked.kind == unchunked.kind
     _assert_payload_close(unchunked.payload, chunked.payload)
+
+
+# ---------------------------------------------------------------------------
+# new (gitea #76) reductions: KS distance, confusion matrix, containment depth
+# ---------------------------------------------------------------------------
+
+
+def test_ks_statistic():
+    assert _ks_statistic([10, 10], [10, 10]) == 0.0  # identical shape -> 0
+    assert _ks_statistic([10, 0], [0, 10]) == 1.0  # fully disjoint -> 1
+    assert _ks_statistic([0, 0], [0, 0]) != _ks_statistic([0, 0], [0, 0])  # nan (no data either side)
+    assert _ks_statistic([10, 0], [0, 0]) == 1.0  # one side empty, other isn't -> maximal mismatch
+
+
+def test_integer_confusion_matches_event_pairing():
+    # true (reference) n_sec = [1, 1]; predicted (rollout) n_sec = [1, 0]
+    labels, mat = _integer_confusion(np.array([1, 1]), np.array([1, 0]))
+    assert labels == ["0", "1+"]
+    assert mat.tolist() == [[0, 0], [1, 1]]  # row=true, col=pred
+
+
+def test_integer_confusion_caps_pathological_outliers():
+    labels, mat = _integer_confusion(np.array([0, 500]), np.array([0, 0]), max_bins=5)
+    assert labels[-1] == "4+"
+    assert mat.shape == (5, 5)
+    assert mat.sum() == 2
+
+
+def test_integer_confusion_explicit_cap_overrides_local_range():
+    # Even though this pair's own max is 1, an explicit shared cap forces a
+    # wider (and so cross-rollout-consistent) label set.
+    labels, mat = _integer_confusion(np.array([1, 1]), np.array([0, 1]), cap=3)
+    assert labels == ["0", "1", "2", "3+"]
+    assert mat.shape == (4, 4)
+
+
+def test_containment_depths_simple_ramp():
+    # one event, edep concentrated in the first bin -> 90%/95% containment
+    # depth is the first bin's right edge; a zero-energy event is dropped.
+    mat = np.array([[9.0, 1.0, 0.0], [0.0, 0.0, 0.0]])
+    edges = np.array([0.0, 1.0, 2.0, 3.0])
+    depths = _containment_depths(mat, edges, 0.90)
+    assert depths.tolist() == [1.0]
+
+
+def test_n_sec_confusion_spec(bundle):
+    spec = get_spec("n_sec_confusion")
+    r = spec.finalize([spec.compute_partial(bundle)], bundle.ctx)
+    assert r.payload["row_labels"] == r.payload["col_labels"] == ["0", "1+"]
+    assert r.payload["series"]["rollout"] == [[0, 0], [1, 1]]
+
+
+def test_n_sec_confusion_shares_one_cap_across_rollouts(two_bundle):
+    spec = get_spec("n_sec_confusion")
+    r = spec.finalize([spec.compute_partial(two_bundle)], two_bundle.ctx)
+    assert list(r.payload["series"]) == ["flow", "wgan"]
+    # both rollouts share the same fixture data here, so their matrices (and
+    # the shared label set) must be identical.
+    assert r.payload["series"]["flow"] == r.payload["series"]["wgan"]

tests/test_render.py

@@ -25,41 +25,96 @@ def test_render_router_diagnostics_and_edge_cases(tmp_path: Path):
     reduced = [
         Reduced(
             "rg",
-            "router",
+            "model",
             "router_gating",
             "Router gating",
             "pre-step energy [MeV]",
             {
-                "n_experts": 2,
                 "log_x": True,
-                "router_type": "energy",
-                "rollout": {
-                    "centers": [1.0, 10.0, 100.0],
-                    "means": [[0.6, 0.4], [0.5, 0.5], [0.4, 0.6]],
-                },
-                "reference": {
-                    "centers": [1.0, 10.0, 100.0],
-                    "means": [[0.55, 0.45], [0.5, 0.5], [0.45, 0.55]],
+                "series": {
+                    "flow": {
+                        "n_experts": 2,
+                        "router_type": "energy",
+                        "rollout": {
+                            "centers": [1.0, 10.0, 100.0],
+                            "means": [[0.6, 0.4], [0.5, 0.5], [0.4, 0.6]],
+                        },
+                        "reference": {
+                            "centers": [1.0, 10.0, 100.0],
+                            "means": [[0.55, 0.45], [0.5, 0.5], [0.45, 0.55]],
+                        },
+                    },
+                    "wgan": {
+                        "n_experts": 2,
+                        "router_type": "energy",
+                        "rollout": {"centers": [1.0], "means": [[0.5, 0.5]]},
+                        "reference": {"centers": [1.0], "means": [[0.5, 0.5]]},
+                    },
                 },
             },
         ),
         Reduced(
             "rs",
-            "router",
+            "model",
             "router_share",
             "Router share",
             "species",
             {
-                "categories": ["e-", "gamma"],
-                "n_experts": 2,
-                "router_type": "energy",
-                "rollout": {"e-": [0.7, 0.3], "gamma": [0.2, 0.8]},
-                "reference": {"e-": [0.6, 0.4], "gamma": [0.3, 0.7]},
+                "series": {
+                    "flow": {
+                        "categories": ["e-", "gamma"],
+                        "n_experts": 2,
+                        "router_type": "energy",
+                        "rollout": {"e-": [0.7, 0.3], "gamma": [0.2, 0.8]},
+                        "reference": {"e-": [0.6, 0.4], "gamma": [0.3, 0.7]},
+                    },
+                },
+            },
+        ),
+        Reduced(
+            "rp",
+            "model",
+            "router_share",
+            "Router share by process (reference-only)",
+            "process",
+            {
+                "series": {
+                    "flow": {
+                        "categories": ["compt", "phot"],
+                        "n_experts": 2,
+                        "router_type": "energy",
+                        "reference": {"compt": [0.4, 0.6], "phot": [0.9, 0.1]},
+                    },
+                },
+            },
+        ),
+        Reduced(
+            "rz",
+            "model",
+            "router_specialization",
+            "Router specialization",
+            "pre-step energy [MeV]",
+            {
+                "log_x": True,
+                "series": {
+                    "flow": {
+                        "n_experts": 2,
+                        "chance_level": 0.5,
+                        "rollout": {"centers": [1.0, 10.0], "score": [0.6, 0.7]},
+                        "reference": {"centers": [1.0, 10.0], "score": [0.55, 0.65]},
+                    },
+                    "wgan": {
+                        "n_experts": 4,
+                        "chance_level": 0.25,
+                        "rollout": {"centers": [1.0, 10.0], "score": [0.3, 0.4]},
+                        "reference": {"centers": [], "score": []},
+                    },
+                },
             },
         ),
         Reduced(
             "ru",
-            "router",
+            "model",
             "unavailable",
             "Router unavailable",
             "x",
@@ -73,7 +128,10 @@ def test_render_router_diagnostics_and_edge_cases(tmp_path: Path):
             "x",
             {
                 "edges": [0, 1, 2],
-                "groups": {lbl: {"rollout": [1, 2], "reference": [2, 1]} for lbl in ("a", "b", "c", "d")},
+                "groups": {
+                    lbl: {"series": {"flow": [1, 2], "wgan": [2, 1]}, "reference": [2, 1]}
+                    for lbl in ("a", "b", "c", "d")
+                },
                 "log_y": True,
             },
         ),
@@ -83,7 +141,22 @@ def test_render_router_diagnostics_and_edge_cases(tmp_path: Path):
             "single_hist",
             "Single (log-x)",
             "x",
-            {"edges": [1, 10, 100], "rollout": [5, 1], "log_x": True, "log_y": True},
+            {"edges": [1, 10, 100], "series": {"flow": [5, 1], "wgan": [3, 2]}, "log_x": True, "log_y": True},
+        ),
+        Reduced(
+            "hm",
+            "quality",
+            "heatmap",
+            "Distance summary (2 rollouts)",
+            "grouping axis",
+            {
+                "series": {"flow": [[0.1, 0.2], [0.3, 0.4]], "wgan": [[0.5, 0.6], [0.7, 0.8]]},
+                "row_labels": ["step_length", "edep"],
+                "col_labels": ["overall", "energy"],
+                "cbar_label": "KS statistic",
+                "vmin": 0.0,
+                "vmax": 1.0,
+            },
         ),
     ]
     try:
@@ -117,7 +190,7 @@ def test_render_all_run_gallery_invokes_subprocess(tmp_path: Path, monkeypatch):
             "single_hist",
             "Single",
             "x",
-            {"edges": [0, 1, 2], "rollout": [5, 1]},
+            {"edges": [0, 1, 2], "series": {"rollout": [5, 1]}},
         )
     ]
     for r in reduced:
@@ -133,24 +206,23 @@ def test_render_all_run_gallery_invokes_subprocess(tmp_path: Path, monkeypatch):
     assert kwargs == {"check": True}
 
 
-def test_render_run_glues_condor_run_meta_into_render_all(tmp_path: Path, monkeypatch):
-    from giant.analysis import condor as condor_mod
+def test_render_run_glues_run_meta_into_render_all(tmp_path: Path, monkeypatch):
+    from giant.analysis import run as run_mod
 
     run_dir = tmp_path / "run"
     (run_dir / "reduced").mkdir(parents=True)
 
     merge_calls = []
-    monkeypatch.setattr(condor_mod, "merge_all", lambda rd: merge_calls.append(Path(rd)))
-    meta = condor_mod.RunMeta(
-        rollout="rollout.parquet",
+    monkeypatch.setattr(run_mod, "merge_all", lambda rd: merge_calls.append(Path(rd)))
+    meta = run_mod.RunMeta(
+        rollouts=[{"name": "rollout", "path": "rollout.parquet", "plot_meta": {"checkpoint": "ckpt/best.pt"}}],
         reference="reference.parquet",
         run_dir=str(run_dir),
         title="my-run",
-        plot_meta={"checkpoint": "ckpt/best.pt"},
     )
-    monkeypatch.setattr(condor_mod.RunMeta, "load", classmethod(lambda cls, p: meta))
+    monkeypatch.setattr(run_mod.RunMeta, "load", classmethod(lambda cls, p: meta))
 
-    Reduced("s", "species", "single_hist", "Single", "x", {"edges": [0, 1], "rollout": [1]}).save(
+    Reduced("s", "species", "single_hist", "Single", "x", {"edges": [0, 1], "series": {"rollout": [1]}}).save(
         run_dir / "reduced" / "s.json"
     )
 
@@ -177,7 +249,7 @@ def test_render_one_of_each_kind(tmp_path: Path):
             "x",
             {
                 "edges": [0, 1, 2, 3],
-                "rollout": [1, 2, 3],
+                "series": {"flow": [1, 2, 3], "wgan": [2, 2, 2]},
                 "reference": [3, 2, 1],
                 "log_y": False,
             },
@@ -190,7 +262,7 @@ def test_render_one_of_each_kind(tmp_path: Path):
             "x",
             {
                 "edges": [0, 1, 2],
-                "groups": {"a": {"rollout": [1, 2], "reference": [2, 1]}},
+                "groups": {"a": {"series": {"flow": [1, 2]}, "reference": [2, 1]}},
                 "log_y": False,
             },
         ),
@@ -202,10 +274,8 @@ def test_render_one_of_each_kind(tmp_path: Path):
             "depth",
             {
                 "edges": [0, 1, 2],
-                "rollout_mean": [1, 2],
-                "rollout_std": [0.1, 0.2],
-                "reference_mean": [1.1, 1.9],
-                "reference_std": [0.1, 0.1],
+                "series": {"flow": {"mean": [1, 2], "std": [0.1, 0.2]}},
+                "reference": {"mean": [1.1, 1.9], "std": [0.1, 0.1]},
                 "ylabel": "e",
             },
         ),
@@ -217,7 +287,7 @@ def test_render_one_of_each_kind(tmp_path: Path):
             "species",
             {
                 "labels": ["e-", "gamma"],
-                "rollout": [0.6, 0.4],
+                "series": {"flow": [0.6, 0.4], "wgan": [0.55, 0.45]},
                 "reference": [0.5, 0.5],
                 "ylabel": "frac",
             },
@@ -228,7 +298,20 @@ def test_render_one_of_each_kind(tmp_path: Path):
             "single_hist",
             "Single",
             "x",
-            {"edges": [0, 1, 2], "rollout": [5, 1], "log_y": True},
+            {"edges": [0, 1, 2], "series": {"flow": [5, 1]}, "log_y": True},
+        ),
+        Reduced(
+            "hm1",
+            "secondaries",
+            "heatmap",
+            "Confusion (single rollout)",
+            "predicted",
+            {
+                "series": {"flow": [[1, 0], [0, 1]]},
+                "row_labels": ["0", "1+"],
+                "col_labels": ["0", "1+"],
+                "cbar_label": "count",
+            },
         ),
     ]
     try:
@@ -276,8 +359,8 @@ def test_figure_params_v2_basics_and_router_and_epoch():
         },
         "conditioning": {"particle": {"type": "physical"}},
     }
-    run_meta = {"training_epoch": 12, "best_val_loss": 0.123456, "steps": 10}
-    params = render_mod._figure_params(run_meta | {"model_config": mc})
+    meta = {"training_epoch": 12, "best_val_loss": 0.123456, "steps": 10, "model_config": mc}
+    params = render_mod._figure_params({"rollouts": {"rollout": meta}})
     assert params == {
         "hidden_dim": 256,
         "n_res_blocks": 4,
@@ -297,8 +380,8 @@ def test_figure_params_v2_wgan_reports_noise_dim_not_steps():
             "wgan": {"noise_dim": 32},
         },
     }
-    run_meta = {"model_config": mc, "steps": 10}
-    params = render_mod._figure_params(run_meta)
+    meta = {"model_config": mc, "steps": 10}
+    params = render_mod._figure_params({"rollouts": {"rollout": meta}})
     assert params["mode"] == "wgan"
     assert params["noise_dim"] == 32
     assert "steps" not in params
@@ -309,18 +392,18 @@ def test_figure_params_v2_reports_mode_s2_only_when_it_differs():
         "stage1_model": {"generator": "flow"},
         "stage2_model": {"generator": "flow"},
     }
-    assert "mode_s2" not in render_mod._figure_params({"model_config": same})
+    assert "mode_s2" not in render_mod._figure_params({"rollouts": {"rollout": {"model_config": same}}})
 
     mixed = {
         "stage1_model": {"generator": "flow"},
         "stage2_model": {"generator": "wgan"},
     }
-    params = render_mod._figure_params({"model_config": mixed})
+    params = render_mod._figure_params({"rollouts": {"rollout": {"model_config": mixed}}})
     assert params["mode_s2"] == "wgan"
 
 
 def test_figure_params_old_shape_basics():
-    run_meta = {
+    meta = {
         "model_config": {
             "hidden_dim": 128,
             "n_blocks": 3,
@@ -332,7 +415,7 @@ def test_figure_params_old_shape_basics():
         "best_val_loss": 0.5,
         "steps": 20,
     }
-    params = render_mod._figure_params(run_meta)
+    params = render_mod._figure_params({"rollouts": {"rollout": meta}})
     assert params == {
         "hidden_dim": 128,
         "n_blocks": 3,
@@ -346,20 +429,30 @@ def test_figure_params_old_shape_basics():
 
 
 def test_figure_params_old_shape_wgan_reports_noise_dim_not_steps():
-    run_meta = {
+    meta = {
         "model_config": {"mode": "wgan", "noise_dim": 16},
         "steps": 20,
     }
-    params = render_mod._figure_params(run_meta)
+    params = render_mod._figure_params({"rollouts": {"rollout": meta}})
     assert params["noise_dim"] == 16
     assert "steps" not in params
 
 
+def test_figure_params_multi_rollout_names_the_series():
+    run_meta = {"rollouts": {"flow": {"model_config": {"mode": "flow"}}, "wgan": {"model_config": {"mode": "wgan"}}}}
+    assert render_mod._figure_params(run_meta) == {"rollouts": "flow, wgan"}
+
+
+def test_figure_params_empty_rollouts_is_empty():
+    assert render_mod._figure_params({}) == {}
+    assert render_mod._figure_params({"rollouts": {}}) == {}
+
+
 def test_plot_metadata_includes_note_and_run_meta_parameters():
     r = Reduced("u", "router", "unavailable", "Unavailable", "x", {"note": "no router data"})
-    meta = render_mod._plot_metadata(r, {"title": "run-1", "checkpoint": "ckpt.pt"})
+    meta = render_mod._plot_metadata(r, {"title": "run-1", "reference": "ref.parquet", "rollouts": {"rollout": {}}})
     assert meta["note"] == "no router data"
-    assert meta["parameters"] == {"checkpoint": "ckpt.pt"}
+    assert meta["parameters"] == {"reference": "ref.parquet", "rollouts": {"rollout": {}}}
     assert "title" not in meta["parameters"]
 
 
@@ -368,3 +461,30 @@ def test_plot_metadata_omits_parameters_when_run_meta_empty():
     meta = render_mod._plot_metadata(r, {})
     assert "parameters" not in meta
     assert "note" not in meta
+
+
+def test_tex_escape_handles_percent_and_other_special_chars():
+    assert render_mod._tex_escape("90% of deposited energy") == r"90\% of deposited energy"
+    assert render_mod._tex_escape(r"a_b & c#d $e {f} \bar") == r"a\_b \& c\#d \$e \{f\} \textbackslash{}bar"
+
+
+def test_render_survives_title_and_xlabel_with_literal_percent(tmp_path: Path):
+    # Regression test for gitea #81: a literal "%" in a catalog title (e.g.
+    # "Shower containment depth (90% of deposited energy)") crashed the whole
+    # LaTeX render, since usetex treats an unescaped "%" as a comment marker.
+    reduced = [
+        Reduced(
+            "shower_containment_depth_90",
+            "shower",
+            "single_hist",
+            "Shower containment depth (90% of deposited energy)",
+            "depth containing 90% of deposited energy [mm]",
+            {"edges": [0, 1, 2], "series": {"flow": [5, 1]}},
+        ),
+    ]
+    try:
+        pdfs = _try_render(reduced, tmp_path)
+    except RuntimeError as e:  # LaTeX missing at render time
+        pytest.skip(f"LaTeX rendering unavailable: {e}")
+    assert len(pdfs) == 1
+    assert pdfs[0].exists()

tests/test_router_gating.py

@@ -10,7 +10,9 @@ from giant.analysis.router_gating import (
     compute_router_gating,
     compute_router_share_by_pdg,
     compute_router_share_by_process,
+    compute_router_specialization,
 )
+from giant.analysis.sources import RolloutSide
 from giant.data.transforms import Normalizer
 from giant.model.network import build_models
 
@@ -34,7 +36,7 @@ def _model_cfg() -> dict:
     }
 
 
-def _write_checkpoint(tmp_path) -> str:
+def _write_checkpoint(tmp_path, name: str = "ckpt.pt") -> str:
     cfg = _model_cfg()
     stage1 = build_models(cfg)["stage1"]
     assert stage1 is not None
@@ -48,7 +50,7 @@ def _write_checkpoint(tmp_path) -> str:
         "mat_map": _MAT_MAP,
         "normalizer": {"cond": norm.to_dict()},
     }
-    path = tmp_path / "ckpt.pt"
+    path = tmp_path / name
     torch.save(ckpt, path)
     return str(path)
 
@@ -86,42 +88,80 @@ def _steps_frame(process: bool = False) -> pl.LazyFrame:
     return pl.DataFrame(data).lazy()
 
 
+def _side(checkpoint: str | None, lf: pl.LazyFrame) -> RolloutSide:
+    return RolloutSide(all=lf, phys=lf, checkpoint=checkpoint)
+
+
 def test_compute_router_gating_shapes(tmp_path):
     checkpoint = _write_checkpoint(tmp_path)
     lf = _steps_frame()
-    r = compute_router_gating(checkpoint, lf, lf)
+    r = compute_router_gating({"rollout": _side(checkpoint, lf)}, lf)
     assert r.kind == "router_gating"
-    assert r.payload["n_experts"] == 2
+    assert list(r.payload["series"]) == ["rollout"]
+    entry = r.payload["series"]["rollout"]
+    assert entry["n_experts"] == 2
     for side in ("rollout", "reference"):
-        means = r.payload[side]["means"]
+        means = entry[side]["means"]
         assert means, f"{side} produced no bins"
         assert all(abs(sum(row) - 1.0) < 1e-5 for row in means)
 
 
 def test_compute_router_gating_missing_checkpoint_is_unavailable():
     lf = _steps_frame()
-    r = compute_router_gating(None, lf, lf)
+    r = compute_router_gating({"rollout": _side(None, lf)}, lf)
     assert r.kind == "unavailable"
     assert "note" in r.payload
     assert r.title
 
 
+def test_compute_router_gating_two_rollouts_only_moe_ones_included(tmp_path):
+    lf = _steps_frame()
+    ckpt = _write_checkpoint(tmp_path)
+    rollouts = {"flow": _side(None, lf), "moe": _side(ckpt, lf)}
+    r = compute_router_gating(rollouts, lf)
+    assert list(r.payload["series"]) == ["moe"]
+
+
+def test_compute_router_specialization_two_rollouts(tmp_path):
+    lf = _steps_frame()
+    ckpt_a = _write_checkpoint(tmp_path, "a.pt")
+    ckpt_b = _write_checkpoint(tmp_path, "b.pt")
+    rollouts = {"a": _side(ckpt_a, lf), "b": _side(ckpt_b, lf)}
+    r = compute_router_specialization(rollouts, lf)
+    assert r.kind == "router_specialization"
+    assert list(r.payload["series"]) == ["a", "b"]
+    for entry in r.payload["series"].values():
+        assert entry["chance_level"] == 0.5
+        assert len(entry["rollout"]["centers"]) == len(entry["rollout"]["score"])
+
+
 def test_compute_router_share_by_pdg(tmp_path):
     checkpoint = _write_checkpoint(tmp_path)
     lf = _steps_frame()
-    r = compute_router_share_by_pdg(checkpoint, lf, lf, top_pdgs=[11, 22])
+    r = compute_router_share_by_pdg({"rollout": _side(checkpoint, lf)}, lf, top_pdgs=[11, 22])
     assert r.kind == "router_share"
+    entry = r.payload["series"]["rollout"]
     for side in ("rollout", "reference"):
-        assert set(r.payload[side]) == {"e-", "gamma"}
-        for shares in r.payload[side].values():
+        assert set(entry[side]) == {"e-", "gamma"}
+        for shares in entry[side].values():
             assert abs(sum(shares) - 1.0) < 1e-5
 
 
 def test_compute_router_share_by_process(tmp_path):
     checkpoint = _write_checkpoint(tmp_path)
     lf = _steps_frame(process=True)
-    r = compute_router_share_by_process(checkpoint, lf)
+    r = compute_router_share_by_process({"rollout": _side(checkpoint, lf)}, lf)
     assert r.kind == "router_share"
-    assert set(r.payload["categories"]) <= {"eIoni", "compt"}
-    for shares in r.payload["reference"].values():
+    entry = r.payload["series"]["rollout"]
+    assert set(entry["categories"]) <= {"eIoni", "compt"}
+    for shares in entry["reference"].values():
         assert abs(sum(shares) - 1.0) < 1e-5
+
+
+def test_no_moe_rollouts_are_unavailable(tmp_path):
+    lf = _steps_frame()
+    rollouts = {"flow": _side(None, lf), "wgan": _side(None, lf)}
+    assert compute_router_gating(rollouts, lf).kind == "unavailable"
+    assert compute_router_share_by_pdg(rollouts, lf, top_pdgs=[11, 22]).kind == "unavailable"
+    assert compute_router_share_by_process(rollouts, lf).kind == "unavailable"
+    assert compute_router_specialization(rollouts, lf).kind == "unavailable"

tests/test_training_plots.py

@@ -0,0 +1,333 @@
+"""Tests for giant.training.plots (gitea #75) — render smoke tests skipped
+where plotstyle/LaTeX is unavailable, plus pure-function column-classification
+coverage that needs neither."""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("plotstyle")
+
+from giant.training import plots as plots_mod  # noqa: E402
+from giant.training.plots import MetricsTable, derive_metrics_dir, render_metrics  # noqa: E402
+
+# --- fixtures ----------------------------------------------------------
+
+
+_RICH_HEADER = [
+    "epoch",
+    "stage1/train/loss",
+    "stage1/train/loss_gen",
+    "stage1/train/nsec_acc",
+    "stage1/train/grad_norm",
+    "stage1/val/loss",
+    "stage1/val/loss_gen",
+    "stage1/val/nsec_acc",
+    "stage1/lr",
+    "stage1/router/entropy",
+    "stage1/router/util_min",
+    "stage1/router/util_max",
+    "stage1/router/util_std",
+    "stage2/train/d_loss",
+    "stage2/train/g_loss",
+    "stage2/train/wasserstein",
+    "stage2/train/gp_loss",
+    "stage2/train/loss_nsec",
+    "stage2/train/nsec_acc",
+    "stage2/train/grad_norm_d",
+    "stage2/train/grad_norm_g",
+    "stage2/lr",
+    "stage2/critic_lr",
+    "val/loss",
+    "val/marginal_kl",
+    "grad_norm",
+    "gpu_mem_mb",
+    "samples_per_sec",
+    "is_best",
+    "epoch_time_s",
+]
+
+_RICH_ROWS = [
+    [
+        1,
+        1.0,
+        0.8,
+        0.5,
+        1.2,
+        0.9,
+        0.7,
+        0.6,
+        3e-4,
+        1.5,
+        0.05,
+        0.3,
+        0.1,
+        -0.2,
+        0.3,
+        0.5,
+        0.1,
+        0.4,
+        0.4,
+        0.9,
+        1.1,
+        3e-4,
+        1e-4,
+        0.85,
+        0.4,
+        2.1,
+        512.0,
+        100.0,
+        1,
+        5.0,
+    ],
+    [
+        2,
+        0.8,
+        0.6,
+        0.6,
+        1.0,
+        0.7,
+        0.5,
+        0.7,
+        2e-4,
+        1.6,
+        0.06,
+        0.28,
+        0.09,
+        -0.1,
+        0.25,
+        0.4,
+        0.09,
+        0.3,
+        0.5,
+        0.8,
+        1.0,
+        2e-4,
+        8e-5,
+        0.7,
+        0.35,
+        1.9,
+        520.0,
+        105.0,
+        0,
+        5.1,
+    ],
+]
+
+_MINIMAL_HEADER = [
+    "epoch",
+    "stage1/train/loss",
+    "stage1/train/loss_gen",
+    "stage1/val/loss",
+    "stage1/val/loss_gen",
+    "stage1/lr",
+    "val/loss",
+    "grad_norm",
+    "gpu_mem_mb",
+    "samples_per_sec",
+    "is_best",
+    "epoch_time_s",
+]
+
+_MINIMAL_ROWS = [
+    [1, 1.0, 0.8, 0.9, 0.7, 3e-4, 0.85, 0.4, 0.0, 100.0, 0, 5.0],
+    [2, 0.8, 0.6, 0.7, 0.5, 2e-4, 0.7, 0.35, 0.0, 105.0, 1, 5.1],
+]
+
+
+def _write_csv(path: Path, header: list[str], rows: list[list]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, "w", newline="") as f:
+        writer = csv.writer(f)
+        writer.writerow(header)
+        writer.writerows(rows)
+
+
+# --- MetricsTable --------------------------------------------------------
+
+
+def test_metrics_table_load_round_trips(tmp_path: Path):
+    csv_path = tmp_path / "metrics.csv"
+    _write_csv(csv_path, _MINIMAL_HEADER, _MINIMAL_ROWS)
+    table = MetricsTable.load(csv_path)
+    assert table.epochs == [1, 2]
+    assert table.columns["stage1/train/loss"] == [1.0, 0.8]
+    assert "epoch" not in table.columns
+    assert table.best_epochs() == [2]
+
+
+def test_metrics_table_best_epochs_empty_without_is_best_column():
+    table = MetricsTable(epochs=[1, 2], columns={"stage1/train/loss": [1.0, 0.5]})
+    assert table.best_epochs() == []
+
+
+# --- column classification (pure functions, no matplotlib) --------------
+
+
+def _rich_columns() -> dict[str, list]:
+    return {name: [0.0] for name in _RICH_HEADER if name != "epoch"}
+
+
+def test_stages_detects_only_stages_present():
+    assert plots_mod._stages(_rich_columns()) == ["stage1", "stage2"]
+    assert plots_mod._stages({"stage2/train/loss": [0.0]}) == ["stage2"]
+    assert plots_mod._stages({"val/loss": [0.0]}) == []
+
+
+def test_split_matches_stage_and_split_prefix_only():
+    cols = _rich_columns()
+    train = plots_mod._split(cols, "stage1", "train")
+    assert train == {
+        "loss": "stage1/train/loss",
+        "loss_gen": "stage1/train/loss_gen",
+        "nsec_acc": "stage1/train/nsec_acc",
+        "grad_norm": "stage1/train/grad_norm",
+    }
+    assert plots_mod._split(cols, "stage2", "val") == {}
+
+
+def test_point_in_time_excludes_train_val_router():
+    cols = _rich_columns()
+    pit = plots_mod._point_in_time(cols, "stage1")
+    assert pit == {"lr": "stage1/lr"}
+    pit2 = plots_mod._point_in_time(cols, "stage2")
+    assert pit2 == {"lr": "stage2/lr", "critic_lr": "stage2/critic_lr"}
+
+
+def test_router_columns():
+    cols = _rich_columns()
+    assert plots_mod._router(cols, "stage1") == {
+        "entropy": "stage1/router/entropy",
+        "util_min": "stage1/router/util_min",
+        "util_max": "stage1/router/util_max",
+        "util_std": "stage1/router/util_std",
+    }
+    assert plots_mod._router(cols, "stage2") == {}
+
+
+def test_run_level_excludes_stage_prefixed_columns_including_val_loss_lookalike():
+    cols = _rich_columns()
+    run_level = plots_mod._run_level(cols)
+    assert set(run_level) == {
+        "val/loss",
+        "val/marginal_kl",
+        "grad_norm",
+        "gpu_mem_mb",
+        "samples_per_sec",
+        "is_best",
+        "epoch_time_s",
+    }
+    # stage-prefixed "val/loss" lookalike (stage1/val/loss) must not leak in
+    assert "stage1/val/loss" not in run_level
+
+
+def test_loss_keys_excludes_acc_and_wgan_and_grad_norm():
+    train = {"loss": "x", "loss_gen": "x", "nsec_acc": "x", "grad_norm": "x", "d_loss": "x"}
+    val = {"loss": "x", "loss_gen": "x"}
+    assert plots_mod._loss_keys(train, val) == ["loss", "loss_gen"]
+
+
+# --- derive_metrics_dir ---------------------------------------------------
+
+
+def test_derive_metrics_dir_explicit_out_dir_wins():
+    assert derive_metrics_dir("runs/my-run", out_dir="/somewhere") == Path("/somewhere")
+
+
+def test_derive_metrics_dir_default_base():
+    assert derive_metrics_dir("runs/my-run", default_base="/data/analysis_runs") == Path(
+        "/data/analysis_runs/metrics_my-run"
+    )
+
+
+def test_derive_metrics_dir_falls_back_to_cwd_analysis_runs(monkeypatch, tmp_path):
+    monkeypatch.chdir(tmp_path)
+    assert derive_metrics_dir("runs/my-run") == tmp_path / "analysis_runs" / "metrics_my-run"
+
+
+# --- render_metrics end to end -------------------------------------------
+
+
+def _try_render(run_dir: Path, out_dir: Path) -> list[Path]:
+    try:
+        return render_metrics(run_dir, out_dir)
+    except RuntimeError as e:  # LaTeX missing at render time
+        pytest.skip(f"LaTeX rendering unavailable: {e}")
+
+
+def test_render_metrics_rich_run_produces_expected_plots_outside_run_dir(tmp_path: Path):
+    run_dir = tmp_path / "run"
+    out_dir = tmp_path / "out"
+    _write_csv(run_dir / "metrics.csv", _RICH_HEADER, _RICH_ROWS)
+
+    paths = _try_render(run_dir, out_dir)
+
+    names = {p.stem for p in paths}
+    assert names == {
+        "overview",
+        "stage1_loss",
+        "stage2_loss",
+        "lr",
+        "stage1_accuracy",
+        "stage2_accuracy",
+        "grad_norm",
+        "stage1_router",
+        "stage2_wgan_balance",
+        "throughput",
+    }
+    assert all(p.exists() for p in paths)
+    assert all(p.is_relative_to(out_dir) for p in paths)
+    # nothing written into the training run directory itself
+    assert not any(run_dir.rglob("*.pdf"))
+
+
+def test_render_metrics_minimal_run_omits_router_wgan_accuracy(tmp_path: Path):
+    run_dir = tmp_path / "run"
+    out_dir = tmp_path / "out"
+    _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS)
+
+    paths = _try_render(run_dir, out_dir)
+
+    names = {p.stem for p in paths}
+    assert names == {"overview", "stage1_loss", "lr", "grad_norm", "throughput"}
+    assert "stage1_accuracy" not in names
+    assert "stage1_router" not in names
+    assert "stage1_wgan_balance" not in names
+
+
+def test_render_metrics_default_out_dir_uses_analysis_runs_convention(tmp_path: Path):
+    run_dir = tmp_path / "runs" / "my-run"
+    _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS)
+    default_base = tmp_path / "analysis_runs"
+
+    try:
+        paths = render_metrics(run_dir, default_base=default_base)
+    except RuntimeError as e:
+        pytest.skip(f"LaTeX rendering unavailable: {e}")
+
+    assert paths
+    assert all(p.is_relative_to(default_base / "metrics_my-run") for p in paths)
+
+
+# --- CLI -------------------------------------------------------------------
+
+
+def test_cli_analyze_metrics_smoke(tmp_path: Path):
+    from typer.testing import CliRunner
+
+    from giant.cli import app
+
+    run_dir = tmp_path / "run"
+    out_dir = tmp_path / "out"
+    _write_csv(run_dir / "metrics.csv", _MINIMAL_HEADER, _MINIMAL_ROWS)
+
+    runner = CliRunner()
+    result = runner.invoke(app, ["analyze", "metrics", str(run_dir), "--out", str(out_dir)])
+
+    if result.exit_code != 0 and "latex" in (str(result.output) + str(result.exception)).lower():
+        pytest.skip("LaTeX rendering unavailable")
+    assert result.exit_code == 0, result.output or result.exception
+    assert any(out_dir.glob("*.pdf"))

tests/test_type_embedding_distance.py

@@ -3,6 +3,9 @@
 
 from __future__ import annotations
 
+import polars as pl
+
+from giant.analysis.sources import RolloutSide
 from giant.analysis.type_embedding_distance import compute_type_embedding_l1_distance
 
 
@@ -18,26 +21,47 @@ def _summary(n=100):
     }
 
 
+def _side(l1_dist: dict | None) -> RolloutSide:
+    empty = pl.LazyFrame()
+    return RolloutSide(all=empty, phys=empty, type_embedding_l1_dist=l1_dist)
+
+
 def test_none_is_unavailable():
-    r = compute_type_embedding_l1_distance(None)
+    r = compute_type_embedding_l1_distance({"rollout": _side(None)})
     assert r.kind == "unavailable"
     assert r.id == "type_embedding_l1_distance"
     assert r.payload["note"]
 
 
 def test_summary_produces_single_hist():
-    r = compute_type_embedding_l1_distance(_summary())
+    r = compute_type_embedding_l1_distance({"rollout": _side(_summary())})
     assert r.kind == "single_hist"
     assert r.id == "type_embedding_l1_distance"
     assert r.payload["edges"] == [0.0, 1.0, 2.0, 3.0]
-    assert r.payload["rollout"] == [30, 40, 30]
+    assert r.payload["series"]["rollout"] == [30, 40, 30]
     assert r.payload["log_x"] is True
     assert r.payload["log_y"] is True
     assert "n=100" in r.payload["note"]
 
 
 def test_single_hist_payload_shape_matches_render_contract():
-    """_render_single (giant.analysis.render) requires len(rollout) ==
+    """_render_single (giant.analysis.render) requires each series' length ==
     len(edges) - 1."""
-    r = compute_type_embedding_l1_distance(_summary())
-    assert len(r.payload["rollout"]) == len(r.payload["edges"]) - 1
+    r = compute_type_embedding_l1_distance({"rollout": _side(_summary())})
+    assert len(r.payload["series"]["rollout"]) == len(r.payload["edges"]) - 1
+
+
+def test_two_rollouts_both_populated():
+    r = compute_type_embedding_l1_distance({"flow": _side(_summary(50)), "wgan": _side(_summary(80))})
+    assert list(r.payload["series"]) == ["flow", "wgan"]
+    assert "n=50" in r.payload["note"] and "n=80" in r.payload["note"]
+
+
+def test_one_of_two_rollouts_populated_only_that_one_appears():
+    r = compute_type_embedding_l1_distance({"flow": _side(None), "wgan": _side(_summary())})
+    assert list(r.payload["series"]) == ["wgan"]
+
+
+def test_none_populated_across_rollouts_is_unavailable():
+    r = compute_type_embedding_l1_distance({"flow": _side(None), "wgan": _side(None)})
+    assert r.kind == "unavailable"

tests/test_workflow_spec.py

@@ -0,0 +1,148 @@
+"""Workflow spec parsing, validation, and spec hashes (gitea #83)."""
+
+import pytest
+
+from giant.workflow.spec import (
+    WorkflowSpecError,
+    epoch_milestones,
+    load_spec,
+    parse_spec,
+    spec_hash,
+)
+
+MINIMAL = {
+    "workflow": {"name": "wf", "result_dir": "/tmp/wf"},
+    "condor": {"accounting_group": "cms", "repo_dir": "/work/lbogner/giant"},
+    "dataset": {"steps": "/data/train", "reference": "/data/holdout"},
+    "train": [{"name": "a", "epochs": 3}],
+    "rollout": [{"name": "a", "train": "a"}],
+    "analysis": [{"name": "cmp", "rollouts": ["a"], "chunks": 4}],
+}
+
+
+def _spec(**patch):
+    raw = {k: (v.copy() if isinstance(v, dict) else list(v)) for k, v in MINIMAL.items()}
+    raw.update(patch)
+    return parse_spec(raw)
+
+
+def test_parses_minimal_spec():
+    spec = _spec()
+    assert spec.name == "wf"
+    assert spec.log_dir == "/tmp/wf/logs"  # derived from result_dir
+    assert spec.train("a").epochs == 3
+    assert spec.rollout("a").train == "a"
+    assert spec.analysis("cmp").rollouts == ("a",)
+    # defaults come from the dataclasses, not the file
+    assert spec.geometry.method == "slab"
+    assert spec.condor.docker_image_gpu == "mschnepf/slc7-condocker"
+
+
+def test_example_config_is_valid():
+    spec = load_spec("configs/workflow_example.toml")
+    assert {t.name for t in spec.trains} == {"baseline", "router-balanced"}
+    assert spec.analysis("baseline-vs-router").rollouts == ("baseline", "router-balanced")
+
+
+@pytest.mark.parametrize(
+    "patch, message",
+    [
+        ({"train": [{"name": "a"}, {"name": "a"}]}, "unique"),
+        ({"rollout": [{"name": "r", "train": "nope"}]}, "names no"),
+        ({"analysis": [{"name": "c", "rollouts": ["nope"]}]}, "not defined"),
+        ({"analysis": [{"name": "c", "rollouts": []}]}, "at least one"),
+        ({"analysis": [{"name": "c", "rollouts": ["a"], "chunks": 0}]}, "chunks must be"),
+        ({"train": [{"name": "a", "epochs": 0}]}, "epochs must be"),
+        ({"train": [{"name": "a", "epchs": 3}]}, "unknown key"),
+        ({"geometry": {"methd": "slab"}}, "unknown key"),
+    ],
+)
+def test_validation_errors(patch, message):
+    with pytest.raises(WorkflowSpecError, match=message):
+        _spec(**patch)
+
+
+def test_unknown_top_level_table_rejected():
+    with pytest.raises(WorkflowSpecError, match="unknown top-level"):
+        _spec(nonsense={})
+
+
+def test_missing_required_table_rejected():
+    raw = {k: v for k, v in MINIMAL.items() if k != "dataset"}
+    with pytest.raises(WorkflowSpecError, match=r"missing required \[dataset\]"):
+        parse_spec(raw)
+
+
+def test_unknown_lookup_names_are_explicit():
+    spec = _spec()
+    with pytest.raises(WorkflowSpecError, match="no \\[\\[train\\]\\] named 'zzz'"):
+        spec.train("zzz")
+
+
+def test_hash_is_stable_and_order_independent():
+    a = _spec()
+    b = parse_spec(
+        {
+            "dataset": MINIMAL["dataset"],
+            "condor": MINIMAL["condor"],
+            "workflow": MINIMAL["workflow"],
+            "train": MINIMAL["train"],
+            "rollout": MINIMAL["rollout"],
+            "analysis": MINIMAL["analysis"],
+        }
+    )
+    assert a.train_hash("a") == b.train_hash("a")
+    assert a.analysis_hash("cmp") == b.analysis_hash("cmp")
+    assert len(a.train_hash("a")) == 8
+
+
+def test_hash_changes_with_own_settings():
+    base = _spec()
+    changed = _spec(train=[{"name": "a", "epochs": 4}])
+    assert base.train_hash("a") != changed.train_hash("a")
+
+
+def test_hash_propagates_from_parents():
+    """A dataset change must move every downstream task's directory."""
+    base = _spec()
+    changed = _spec(dataset={"steps": "/data/other", "reference": "/data/holdout"})
+    assert base.train_hash("a") != changed.train_hash("a")
+    assert base.rollout_hash("a") != changed.rollout_hash("a")
+    assert base.analysis_hash("cmp") != changed.analysis_hash("cmp")
+
+    # ... and so must a change to a training the analysis transitively uses.
+    retrained = _spec(train=[{"name": "a", "epochs": 9}])
+    assert retrained.analysis_hash("cmp") != base.analysis_hash("cmp")
+    # while an unrelated knob on the analysis leaves the training alone
+    rebinned = _spec(analysis=[{"name": "cmp", "rollouts": ["a"], "chunks": 4, "bins": 99}])
+    assert rebinned.train_hash("a") == base.train_hash("a")
+    assert rebinned.analysis_hash("cmp") != base.analysis_hash("cmp")
+
+
+def test_warm_cache_hash_ignores_epochs():
+    """Epoch count doesn't change the setup cache, so it must not re-warm it."""
+    base = _spec()
+    longer = _spec(train=[{"name": "a", "epochs": 50}])
+    assert base.warm_cache_hash("a") == longer.warm_cache_hash("a")
+    other_cfg = _spec(train=[{"name": "a", "epochs": 3, "config": "configs/router.toml"}])
+    assert base.warm_cache_hash("a") != other_cfg.warm_cache_hash("a")
+
+
+def test_spec_hash_expands_dataclasses():
+    spec = _spec()
+    assert spec_hash(spec.dataset) == spec_hash(spec.dataset)
+    assert spec_hash(spec.dataset) != spec_hash(spec.geometry)
+
+
+@pytest.mark.parametrize(
+    "epochs, per_job, expected",
+    [
+        (3, 1, [1, 2, 3]),
+        (10, 3, [3, 6, 9, 10]),
+        (9, 3, [3, 6, 9]),
+        (1, 5, [1]),
+    ],
+)
+def test_epoch_milestones(epochs, per_job, expected):
+    spec = _spec(train=[{"name": "a", "epochs": epochs, "epochs_per_job": per_job}])
+    assert epoch_milestones(spec.train("a")) == expected

tests/test_workflow_tasks.py

@@ -0,0 +1,172 @@
+"""Workflow task graph: dependencies, output paths, condor settings (gitea #83)."""
+
+import pytest
+
+from giant.analysis.catalog import catalog_ids, get_spec as get_plot_spec
+from giant.workflow import tasks
+from giant.workflow.spec import parse_spec
+
+CONDOR = {
+    "accounting_group": "cms",
+    "repo_dir": "/work/lbogner/giant",
+    "env_script": "/work/lbogner/giant/condor_env.sh",
+}
+
+RAW = {
+    "workflow": {"name": "wf", "result_dir": "/results/wf"},
+    "condor": CONDOR,
+    "dataset": {"steps": "/data/train", "reference": "/data/holdout"},
+    "train": [
+        {"name": "base", "epochs": 3, "gpu_memory_mb": 20000},
+        {"name": "router", "epochs": 2},
+    ],
+    "rollout": [
+        {"name": "base", "train": "base"},
+        {"name": "router", "train": "router"},
+    ],
+    "analysis": [{"name": "cmp", "rollouts": ["base", "router"], "chunks": 4}],
+}
+
+
+@pytest.fixture
+def spec():
+    s = parse_spec(RAW)
+    tasks.set_spec(s)
+    return s
+
+
+def _requires(task):
+    return list(task.requires() or [])
+
+
+def test_epoch_chain_is_linear_and_rooted_at_warm_cache(spec):
+    h = spec.train_hash("base")
+    third = tasks.TrainEpochTask(name="base", spec_hash=h, milestone=3)
+    second = _requires(third)
+    assert [type(t) for t in second] == [tasks.TrainEpochTask]
+    assert second[0].milestone == 2
+    first = _requires(second[0])[0]
+    assert first.milestone == 1
+    root = _requires(first)
+    assert [type(t) for t in root] == [tasks.WarmCacheTask]
+    # the warm cache is keyed by its own hash, not the training's
+    assert root[0].spec_hash == spec.warm_cache_hash("base")
+
+
+def test_epoch_task_outputs_last_pt_per_milestone(spec):
+    h = spec.train_hash("base")
+    path = tasks.TrainEpochTask(name="base", spec_hash=h, milestone=2).output().path
+    assert path == f"/results/wf/train_epoch/name=base/spec_hash={h}/epochs=2/last.pt"
+
+
+def test_train_task_requires_final_epoch_and_publishes_canonical_outputs(spec):
+    h = spec.train_hash("base")
+    train = tasks.TrainTask(name="base", spec_hash=h)
+    (dep,) = _requires(train)
+    assert isinstance(dep, tasks.TrainEpochTask) and dep.milestone == 3
+    out = train.output()
+    assert set(out) == {"best.pt", "last.pt", "metrics.csv"}
+    assert out["best.pt"].path == f"/results/wf/train/name=base/spec_hash={h}/best.pt"
+    # local: it only copies files around, no reason to queue a job for it
+    assert train.batch_system == "local"
+
+
+def test_rollout_requires_training_geometry_and_reference(spec):
+    ro = tasks.RolloutTask(name="base", spec_hash=spec.rollout_hash("base"))
+    deps = _requires(ro)
+    assert [type(d) for d in deps] == [tasks.TrainTask, tasks.GeometryOracleTask, tasks.DatasetTask]
+    assert deps[0].name == "base"
+    assert deps[2].path == "/data/holdout"
+    out = ro.output()
+    assert out["rollout.yaml"].path.endswith("rollout.yaml")
+    # the sidecar sits next to the parquet — the deterministic path
+    # `giant rollout --out` now produces
+    assert out["rollout.yaml"].path[: -len(".yaml")] == out["rollout.parquet"].path[: -len(".parquet")]
+
+
+def test_analysis_prep_requires_every_named_rollout(spec):
+    prep = tasks.AnalysisPrepTask(name="cmp", spec_hash=spec.analysis_hash("cmp"))
+    deps = _requires(prep)
+    assert [d.name for d in deps] == ["base", "router"]
+    assert all(isinstance(d, tasks.RolloutTask) for d in deps)
+    assert prep.batch_system == "local"
+
+
+def test_compute_job_enumeration_collapses_non_chunkable_specs(spec):
+    jobs = tasks.analysis_jobs(spec, "cmp")
+    non_chunkable = [i for i in catalog_ids() if not get_plot_spec(i).chunkable]
+    expected = (len(catalog_ids()) - len(non_chunkable)) * 4 + len(non_chunkable)
+    assert len(jobs) == expected
+    assert non_chunkable, "expected some chunkable=False specs in the catalog"
+    for spec_id in non_chunkable:
+        assert [c for i, c in jobs if i == spec_id] == [0]
+
+
+def test_compute_output_matches_the_on_disk_contract(spec):
+    h = spec.analysis_hash("cmp")
+    task = tasks.AnalysisComputeTask(name="cmp", spec_hash=h, plot_id="event_mean_length", chunk=2)
+    assert task.output().path == (
+        f"/results/wf/analysis/name=cmp/spec_hash={h}/reduced_partial/event_mean_length__2.json"
+    )
+    (dep,) = _requires(task)
+    assert isinstance(dep, tasks.AnalysisPrepTask)
+
+
+def test_render_requires_every_compute_job_and_runs_locally(spec):
+    render = tasks.AnalysisRenderTask(name="cmp", spec_hash=spec.analysis_hash("cmp"))
+    deps = _requires(render)
+    assert len(deps) == len(tasks.analysis_jobs(spec, "cmp"))
+    assert render.batch_system == "local"  # the only step importing plotstyle/LaTeX
+    assert render.output().path.endswith("/plots/metadata.yaml")
+
+
+def test_workflow_task_wraps_every_analysis(spec):
+    deps = _requires(tasks.WorkflowTask(workflow_name="wf"))
+    assert [(type(d), d.name) for d in deps] == [(tasks.AnalysisRenderTask, "cmp")]
+
+
+def test_workflow_without_analysis_falls_back_to_rollouts():
+    raw = {k: v for k, v in RAW.items() if k != "analysis"}
+    tasks.set_spec(parse_spec(raw))
+    deps = _requires(tasks.WorkflowTask(workflow_name="wf"))
+    assert [type(d) for d in deps] == [tasks.RolloutTask, tasks.RolloutTask]
+
+
+def test_gpu_settings_carry_remote_ceph_and_pins(spec):
+    settings = tasks.TrainEpochTask(name="base", spec_hash=spec.train_hash("base"), milestone=1).htcondor_settings
+    assert settings["+RemoteJob"] == "True"
+    assert settings["RequestGPUs"] == 1
+    assert "TARGET.ProvidesEtpCeph =?= True" in settings["requirements"]
+    assert "TARGET.GPUs_GlobalMemoryMb >= 20000" in settings["requirements"]
+    assert settings["accounting_group"] == "cms"
+    assert settings["docker_image"] == "mschnepf/slc7-condocker"
+
+
+def test_cpu_settings_used_for_analysis_compute(spec):
+    task = tasks.AnalysisComputeTask(
+        name="cmp", spec_hash=spec.analysis_hash("cmp"), plot_id="event_mean_length", chunk=0
+    )
+    settings = task.htcondor_settings
+    assert settings["docker_image"] == "cverstege/alma9-gridjob"
+    assert "RequestGPUs" not in settings
+    # no run_meta.json yet (prep hasn't run), so no walltime is claimed
+    assert "+RequestWalltime" not in settings
+
+
+def test_cpu_settings_local_files_use_provides_etp_resources():
+    raw = {**RAW, "condor": {**CONDOR, "remote": False}}
+    spec = parse_spec(raw)
+    tasks.set_spec(spec)
+    settings = tasks.GeometryOracleTask(spec_hash=spec.geometry_hash()).htcondor_settings
+    assert settings["requirements"] == "TARGET.ProvidesETPResources"
+    assert "+RemoteJob" not in settings
+
+
+def test_missing_dataset_fails_immediately(spec):
+    with pytest.raises(FileNotFoundError, match="/ceph"):
+        tasks.DatasetTask(path="/data/train").complete()
+
+
+def test_dataset_that_exists_is_complete(tmp_path, spec):
+    (tmp_path / "steps.parquet").write_text("")
+    assert tasks.DatasetTask(path=str(tmp_path / "steps.parquet")).complete()

uv.lock

@@ -147,6 +147,26 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/44/a1/70ebfffd6c6edc6034a547838ee46287c65ed89f710592ddc39c76b4a5a8/awkward_cpp-53-cp314-cp314t-win_arm64.whl", hash = "sha256:1be0c1d87d9f4fdf94b767a061df849f1bb21579d302b2996fb101527fc80a97", size = 551257, upload-time = "2026-06-08T12:31:56.319Z" },
 ]
 
+[[package]]
+name = "b2luigi"
+version = "1.2.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "cachetools" },
+    { name = "colorama" },
+    { name = "gitpython" },
+    { name = "jinja2" },
+    { name = "luigi" },
+    { name = "parse" },
+    { name = "setuptools" },
+    { name = "tenacity" },
+    { name = "webdavclient3" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/e5/5d/0c3e0602b6cf80a2cfebbe54c330f3623b4de227c2b0f8cbff437dc8d62c/b2luigi-1.2.9.tar.gz", hash = "sha256:3f6734b06970cd5bf6bb62c45c095e9c02b6e283a45b76a29ffabdeaa8fce0c0", size = 786252, upload-time = "2026-04-17T16:38:23.316Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c3/82/1a8d3c20bc235e665300c42783eb8bf9f2aac49c7eae644273191e5e01e8/b2luigi-1.2.9-py3-none-any.whl", hash = "sha256:9ca28b4203f5946394b609da432e53b3c9e35540c840bdc6497cce6ab4c82e2e", size = 115103, upload-time = "2026-04-17T16:38:21.044Z" },
+]
+
 [[package]]
 name = "bracex"
 version = "3.0.1"
@@ -176,6 +196,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b3/0b/5885530f79d4400368b9d4dcb9b39274c0d52e633f7871e7fc6feceea1e3/bump_my_version-1.5.1-py3-none-any.whl", hash = "sha256:df3e2989d0d7fe704718feb24a5880f089b6b6369e427a4445b89c3adebfcff1", size = 65090, upload-time = "2026-08-06T14:26:37.083Z" },
 ]
 
+[[package]]
+name = "cachetools"
+version = "7.1.7"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" },
+]
+
 [[package]]
 name = "certifi"
 version = "2026.7.22"
@@ -605,6 +634,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" },
 ]
 
+[[package]]
+name = "docutils"
+version = "0.23"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/39/a4/5180d9afc57e8fca05601dd652bdff19604c218814037fe90ffc7625a50a/docutils-0.23.tar.gz", hash = "sha256:746f5060322511280a1e50eb76846ed6bf2342984b2ac04dc42caa1a8d78799e", size = 2303823, upload-time = "2026-05-27T17:41:06.934Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/91/30151a39f7570f448ed84529390628a651d7f27c87d73c9b887f8189695e/docutils-0.23-py3-none-any.whl", hash = "sha256:25d013af9bf23bc1c7b2b093dff4208166c53a94786c9e447808335ef1185fea", size = 634701, upload-time = "2026-05-27T17:40:58.442Z" },
+]
+
 [[package]]
 name = "executing"
 version = "2.2.1"
@@ -675,7 +713,7 @@ wheels = [
 
 [[package]]
 name = "giant"
-version = "0.3.6"
+version = "0.3.10"
 source = { editable = "." }
 dependencies = [
     { name = "numpy" },
@@ -728,10 +766,14 @@ geometry = [
 wandb = [
     { name = "wandb" },
 ]
+workflow = [
+    { name = "b2luigi" },
+]
 
 [package.metadata]
 requires-dist = [
     { name = "awkward", marker = "extra == 'convert'", specifier = ">=2.6,<3" },
+    { name = "b2luigi", marker = "extra == 'workflow'", specifier = ">=1.0,<2" },
     { name = "bump-my-version", marker = "extra == 'dev'", specifier = ">=1.2,<2" },
     { name = "giant", extras = ["convert", "analysis", "geometry", "wandb"], marker = "extra == 'dev'" },
     { name = "git-cliff", marker = "extra == 'dev'", specifier = ">=2,<3" },
@@ -757,7 +799,7 @@ requires-dist = [
     { name = "uproot", marker = "extra == 'convert'", specifier = ">=5.3,<6" },
     { name = "wandb", marker = "extra == 'wandb'", specifier = ">=0.16,<1" },
 ]
-provides-extras = ["cpu", "cuda", "dev", "geometry", "wandb", "convert", "analysis"]
+provides-extras = ["cpu", "cuda", "dev", "geometry", "wandb", "convert", "analysis", "workflow"]
 
 [[package]]
 name = "git-cliff"
@@ -779,6 +821,30 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/99/b2/99fac50978b9a90bfec0f1b89354a667ec83f4990301f6c708abce05484e/git_cliff-2.13.1-py3-none-win_amd64.whl", hash = "sha256:856d831a0bede9c258229dbd4d4c2b1c0810d8fce3d3882729669e8dc09c72bf", size = 7714969, upload-time = "2026-04-26T10:33:40.163Z" },
 ]
 
+[[package]]
+name = "gitdb"
+version = "4.0.12"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "smmap" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" },
+]
+
+[[package]]
+name = "gitpython"
+version = "3.1.60"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "gitdb" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/84/14/e6b1a48d831755a53c2029351fcef82e70db4a08f338daefe29d8d0cf31c/gitpython-3.1.60.tar.gz", hash = "sha256:e936431879fa85581b4311fa63492ea52251909e2d655b6529c704c904ddcc24", size = 230793, upload-time = "2026-08-25T18:33:46.102Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/71/63/ba28697918b7c190af9f3f21940d03e8814e25dd4ddd39d6929f3a553995/gitpython-3.1.60-py3-none-any.whl", hash = "sha256:39548bffb8fa0f3a548133348868bb4838e79d73283052207dc97781a569b6b4", size = 221893, upload-time = "2026-08-25T18:33:44.75Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"
@@ -1073,6 +1139,141 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987, upload-time = "2026-03-09T13:15:39.65Z" },
 ]
 
+[[package]]
+name = "lockfile"
+version = "0.12.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/17/47/72cb04a58a35ec495f96984dddb48232b551aafb95bde614605b754fe6f7/lockfile-0.12.2.tar.gz", hash = "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", size = 20874, upload-time = "2015-11-25T18:29:58.279Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c8/22/9460e311f340cb62d26a38c419b1381b8593b0bb6b5d1f056938b086d362/lockfile-0.12.2-py2.py3-none-any.whl", hash = "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa", size = 13564, upload-time = "2015-11-25T18:29:51.462Z" },
+]
+
+[[package]]
+name = "luigi"
+version = "3.7.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "python-daemon" },
+    { name = "python-dateutil" },
+    { name = "tenacity" },
+    { name = "tornado" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b0/df/5235521bd30402e19e469b79c9e91757046014a3d9bc2c91fd341b0edcb6/luigi-3.7.3.tar.gz", hash = "sha256:82e3a5d482e9738f4dcae5acb631ee7ffa748896cb1a1d5d4922c2cac61903fe", size = 1295810, upload-time = "2026-02-12T01:49:34.147Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/63/42/a311f89099c5714dc9c69368d4197a99d453d5f3764713eb5c09b8a8cf18/luigi-3.7.3-py3-none-any.whl", hash = "sha256:f6aaac2417e28a833d30e6b1cd03fe53b2254024dd861fa104ab342cc4f3b843", size = 1186711, upload-time = "2026-02-12T01:49:32.78Z" },
+]
+
+[[package]]
+name = "lxml"
+version = "6.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/ad/a9/970b8fa0ecc4fbf1dfaed0d89bbc1fc1421b25ec26a2038c91e872dc6c8e/lxml-6.1.2.tar.gz", hash = "sha256:1055241852f2b02068af4a625a5d32c087db193c12251928af2562ecd2239f18", size = 4210626, upload-time = "2026-08-19T04:58:15.341Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/ee/a4/55eb54507073089ab27743c5da2113c84f0d0b1715b33175fdd943c9652d/lxml-6.1.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:7d506bdba580ecb1a6ad2e2b5c49445e66d3e1f95894885739094393a1aad237", size = 8602111, upload-time = "2026-08-19T04:58:28.017Z" },
+    { url = "https://files.pythonhosted.org/packages/bc/bf/6332f45d78da385bb01d5cac3fe4acda19f025d1307cbc7ad538355fecbb/lxml-6.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:12acd337d2821cb8b9247dfe4b7aa2f2769a3df5ae8511b7e550df42b8f4d3c3", size = 4638376, upload-time = "2026-08-19T04:58:41.181Z" },
+    { url = "https://files.pythonhosted.org/packages/68/e0/21fba0fe74d417fbe976903ae6bc77e92cdce01aae7b636abd87756f4588/lxml-6.1.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5078ff51e6316c0f75ea8127c2cd24374747fb351f62fb93d1761f8ae5a04a40", size = 4939689, upload-time = "2026-08-19T04:58:48.526Z" },
+    { url = "https://files.pythonhosted.org/packages/de/e5/ce3e885264fdd0bdcb6b49c1ea1842f94281b39e4ff956099e8d57532c60/lxml-6.1.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9477e14217c212e6023c994a71a1a349db19b0e10fd5bf189666b281ae63b1fd", size = 5105185, upload-time = "2026-08-19T04:59:15.533Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/b6/990a8446c488c70fa25681e150de94b7bf2eaaf387e374d195ab3c8faafb/lxml-6.1.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:261d98065326676d7253882db0198d0aa06748d7ee0443367acf10b148273f99", size = 5011863, upload-time = "2026-08-19T04:59:50.58Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/6a/f70f41363dae27e3bfd6224b128f5ba150874bd32ca4938552930ffa33b0/lxml-6.1.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0666943ee1576fa890a6dc6316ef42e8241b5dd56f67bc5475acb2ac298c6ca9", size = 5638234, upload-time = "2026-08-19T05:00:00.802Z" },
+    { url = "https://files.pythonhosted.org/packages/3d/e0/a65b64f34d556925faef2c4f14167d58c571bc15a3e1f2bba71138830562/lxml-6.1.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04cf9e3f4ee9cab9d9ba05401bef8668840fa9620fcd4d8e85a2d2fd0b0fa960", size = 5244532, upload-time = "2026-08-19T05:00:07.516Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/a9/471552e015e954fc9d960aa27c3d67ebf489683d03f033399a790417c67c/lxml-6.1.2-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:9429d2371d406344ed1da5b5686d9412e74137c07b0171278368ff704f470ed5", size = 5358194, upload-time = "2026-08-19T05:00:22.747Z" },
+    { url = "https://files.pythonhosted.org/packages/d7/0f/bc6248fbec2cc416f102b1267f1567e07510f6fa909bbe8cd2a22d6fb78e/lxml-6.1.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:eff128ffdc093cc6317955934ad9751105d37ed8dbca3ff4ccd751af6be37185", size = 4704432, upload-time = "2026-08-19T05:00:51.115Z" },
+    { url = "https://files.pythonhosted.org/packages/a9/3f/cec859f50e63f1fa338fab43d2362d7543e1237f2475960d8ab0769de0eb/lxml-6.1.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ba58574d710b82ead7cbedea01cac3e110bc3ef82d4731519b74a2c11f7cf5e9", size = 5255038, upload-time = "2026-08-19T05:00:58.895Z" },
+    { url = "https://files.pythonhosted.org/packages/7c/d9/2ced0cf2967115f92a1b8b3ae6bd18763abc3ebef88c98cf25145fda396c/lxml-6.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:52f6d4dff133c9778a24e9a2cfc1608930b15869866171aacc5131b5a418a003", size = 5054481, upload-time = "2026-08-19T05:01:10.096Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/f5/4f07386d3c88673daeec3b8cc09a2a4d39fa01c1fc49009791b0746d97fa/lxml-6.1.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:8807998c1023d1e9d60e02500f90e85a0752dbc0b670989806bba87b82dd5b42", size = 4785535, upload-time = "2026-08-19T05:01:18.909Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/5a/f4fe3ecbc189f48fba2547c5db5c940a10151d3e86b856a60a533a77e816/lxml-6.1.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:2170d0a280c877b6e2dc6738217db947be35dd8cf09ca458b355aa1bab2a9e70", size = 5655337, upload-time = "2026-08-19T05:01:41.324Z" },
+    { url = "https://files.pythonhosted.org/packages/92/c4/f586aa1bf27bfbace2dfdbb704da5c52f0bdece8ee440c8fb4946c940b2e/lxml-6.1.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c67f3c1278f942e97d8665c2a690324aaea5137de16f056583a21f0ac706177f", size = 5245778, upload-time = "2026-08-19T05:01:45.227Z" },
+    { url = "https://files.pythonhosted.org/packages/18/a1/677494bbaef4d6db5e4633af817414f478865850b55c03ae4bf70fa7b8ca/lxml-6.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:093fbf547d0f3ca02705381f795a050fbb58988be4aac7f79f99f280c4082313", size = 5267274, upload-time = "2026-08-19T05:01:57.687Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/71/b71425b8764d4cb7c92eb970483be7d5610dce2a6316242b5aaae7d260be/lxml-6.1.2-cp312-cp312-win32.whl", hash = "sha256:be365ce8d2d411cf2fb573747684b4fd470fa6224e0094d9d5a21155acc369d3", size = 3602563, upload-time = "2026-08-19T05:02:01.837Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/fb/909584e16d2148c1a252cc2c32dd99fe0e2682459c586d3d7a192e74a0ae/lxml-6.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:b97153ca609b434b712ddfb92cd6af101a7045a7724c542258bd4727a344472f", size = 4005965, upload-time = "2026-08-19T05:02:07.157Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/8d/41207c9212caad0b52749e34739fb9bfab67486729f52a8fe9bd9266fee6/lxml-6.1.2-cp312-cp312-win_arm64.whl", hash = "sha256:7feb72424f19a893ae4f3373c7aae821b1aacb6076b708915c651f0683a97c49", size = 3666641, upload-time = "2026-08-19T05:02:11.3Z" },
+    { url = "https://files.pythonhosted.org/packages/61/2a/e9651f47a31a60b5cae031abc23391ed9aa30c8fc07571d1a38f58d6d770/lxml-6.1.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:351318f5c0eb7fcab5b4fdb507c6f88fb2c4b5e67784c7e5911448c91fffb5d4", size = 8590165, upload-time = "2026-08-19T04:58:40.489Z" },
+    { url = "https://files.pythonhosted.org/packages/61/87/a8098abaf35118767d1703b84c98940a5d833064e0eca39a00ecfe9840ab/lxml-6.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0edde95e4b4278dcc0175eda06dc8aa2631ad9f83ae5dbdbc4f0925e200b0b0", size = 4632474, upload-time = "2026-08-19T04:58:47.465Z" },
+    { url = "https://files.pythonhosted.org/packages/93/cc/fe74d1def7f4fb967c4a825608a074d4dbdbb871b0d6bd59c6ed07d67868/lxml-6.1.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8326e24ae6c3a6bfb03fa8b4793f9a5d804c125228aa067f652b0428e31b87c", size = 4936196, upload-time = "2026-08-19T04:59:03.477Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/ad/b96e6ca926e26726a99aa643602aac7411ecc1731ddb1b25af8cc57edfcd/lxml-6.1.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7c534ed898413f439b048130011e99a4245ee13d62d431f6b4f7f2484d02a93a", size = 5093290, upload-time = "2026-08-19T04:59:17.498Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/84/616f5d3b7cd086fcfba3e5add6fccda67f976c1c753ae9ed7bbd317cb9be/lxml-6.1.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e37fe49fe2d5aa40a2cb1cc8176673ad7de0d124e6f4a509d9318f5979c7871", size = 4998767, upload-time = "2026-08-19T04:59:28.385Z" },
+    { url = "https://files.pythonhosted.org/packages/80/88/d5b453a8d083483c9442ad7f5ac5c560796022eb5c80d60b65d75e449236/lxml-6.1.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9b52ea73a37fc64aa3357ff8607801d46dd170506d3cf8253a91a1d91639d4f9", size = 5626717, upload-time = "2026-08-19T04:59:40.045Z" },
+    { url = "https://files.pythonhosted.org/packages/71/45/31e5aa4d4bae024908ba1d03480c7425cf027a28b7e5c88d1b7202bd80cc/lxml-6.1.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8b9a92652e75e7731309ea51db5dee892eef414ce70a6ec3441e5d36bf5189f", size = 5232330, upload-time = "2026-08-19T04:59:46.175Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/5a/2627912420df8b2d31ba3014da5539f15ec85add01d42048864ffefda516/lxml-6.1.2-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:9088da25ecd609965f838d89fda0465a905b48f4dd90331db9845518f2177372", size = 5347054, upload-time = "2026-08-19T04:59:52.762Z" },
+    { url = "https://files.pythonhosted.org/packages/16/86/54ac0f529b22a8f12313726dd49e12961bb46471d9028cc28d2a29408f0b/lxml-6.1.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:0349321a0537d4fdbebb2af06dd1b64676132c72e2ae250de8cdb58f8c43019c", size = 4707275, upload-time = "2026-08-19T05:00:04.836Z" },
+    { url = "https://files.pythonhosted.org/packages/3a/42/ffcdc6e4519be90df907cdae7e88409efb25d823ae4de8846f737dae1884/lxml-6.1.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b20440e578d269c5e8a722ab602ddd0f0cedb8b080006b3f936da9991a593d3b", size = 5240071, upload-time = "2026-08-19T05:00:19.604Z" },
+    { url = "https://files.pythonhosted.org/packages/68/49/5b1d7ab35f013f1127ec48f3108319f58b65b00d5cb26f215adbe86eadfb/lxml-6.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7766e525282dd38fd89567311323e441996eb958e8e816d16b38f782e3aecd2a", size = 5050356, upload-time = "2026-08-19T05:00:27.968Z" },
+    { url = "https://files.pythonhosted.org/packages/b0/57/1cf049d054189b55c8fe8012269234f6602256949b69cd3ba80608a88219/lxml-6.1.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:9221442682c27417f10fe11184ea4cce174b25ab52465570b1f3ee3f85f320fa", size = 4780394, upload-time = "2026-08-19T05:00:39.047Z" },
+    { url = "https://files.pythonhosted.org/packages/d5/ad/064488a8fa60e639fd773e421a18bf17541d02a95fbf36238ad7c65f69d4/lxml-6.1.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:75530642d8471327e691ab9b0513a5f9c77f38871014ceda40f51bb51765c0a1", size = 5645854, upload-time = "2026-08-19T05:03:42.697Z" },
+    { url = "https://files.pythonhosted.org/packages/85/bb/120e56f3cf1c149bb3b014278fb86d0a6dd552403981081f0ee0a0a57be7/lxml-6.1.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:678e35f1cbca98f55107511ee21a60568535c950f3c2371819bd64504c980d20", size = 5231132, upload-time = "2026-08-19T05:03:45.466Z" },
+    { url = "https://files.pythonhosted.org/packages/b1/2c/7d49aab893c128671a3276580074cce4c002896145b8dd2893da79633bca/lxml-6.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c2bae42b3a09f977330a08f4a8fe72aec58c4bdb89069d3fe7272a71d885881", size = 5256076, upload-time = "2026-08-19T05:03:48.092Z" },
+    { url = "https://files.pythonhosted.org/packages/72/28/ddea3aa1fa9acfd384fe34d4a2a93eecc07541dd2d922fa9b140c60d8014/lxml-6.1.2-cp313-cp313-win32.whl", hash = "sha256:5848f3de6a8de8a93cff9f068134393ff5fa69ac2a04399f7d49cd67c61c348c", size = 3602177, upload-time = "2026-08-19T05:03:50.571Z" },
+    { url = "https://files.pythonhosted.org/packages/1b/7a/96bac167538748cae2544335855f812fa33e49a9a67bc8b8520dcbd592bd/lxml-6.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:6cb0c87421946030b92b558be416852780a912454e3dcba0998e4497c9c588d5", size = 4004117, upload-time = "2026-08-19T05:03:53.074Z" },
+    { url = "https://files.pythonhosted.org/packages/0a/24/9498fa3c84135956e5ef55ea4d8bd11e999e381f7f210fb6f8c6a980ef03/lxml-6.1.2-cp313-cp313-win_arm64.whl", hash = "sha256:648861c19b775b89ebefa14586f85090b10163367476d77f242c4131c835ce73", size = 3665412, upload-time = "2026-08-19T05:03:55.621Z" },
+    { url = "https://files.pythonhosted.org/packages/27/b5/728b0578791b397ace8d1b101c8b3fe10f36043542f7bb85f82d8bdc3f50/lxml-6.1.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:d50a44113fe6800dcc8a859332b823a4735b1e6ae1b0063882e4cca569ec3e29", size = 8609651, upload-time = "2026-08-19T04:58:42.42Z" },
+    { url = "https://files.pythonhosted.org/packages/3c/6b/49209fa6225c15c48a30061f03d3aba75e3c19634813b88bf83b88c525ed/lxml-6.1.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fa813b0247d0543a563b993ac3dba6168eef59e3a61448432cf5453300c2412b", size = 4639588, upload-time = "2026-08-19T04:59:01.501Z" },
+    { url = "https://files.pythonhosted.org/packages/20/86/80bae4e8bc2eed9d6f017701a3d86fdea56936218efa738911d0b76aa7f4/lxml-6.1.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d858e718b94033ab4b67e4a58fe3114c65bae01ae2314a62fb39ae8897ed4324", size = 4964846, upload-time = "2026-08-19T04:59:08.59Z" },
+    { url = "https://files.pythonhosted.org/packages/70/ce/4782caee7a22959c1ac67cb46495e03912c22a4ba7d20c163496a519e815/lxml-6.1.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e3b666f57a5d81562f38c766c762416b0f6eb58a00590546911514b48412abd", size = 5099288, upload-time = "2026-08-19T04:59:18.649Z" },
+    { url = "https://files.pythonhosted.org/packages/32/21/f120967cc43b54e05512dff0c39726b832c836195d30f41f88733ef36ac8/lxml-6.1.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26ff164c6629e5c4d11c9e55d5ea3d6eed0be2a420eee1f55cbce6e2c23e231a", size = 5036837, upload-time = "2026-08-19T04:59:47.217Z" },
+    { url = "https://files.pythonhosted.org/packages/61/ba/8005e9f47598e3ec5c18312c77f94e889580027616678848405c6aeba5de/lxml-6.1.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:962c12b51d0b164f12569af225dea57568477e24a845b96eaccbef6c07e4cc03", size = 5658569, upload-time = "2026-08-19T04:59:54.078Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/ba/add33b3c7ce51462cf7a4637bcfec2eaa258364d6015b989dd7d1216e6a6/lxml-6.1.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47e367dfe341521426692819803e260d0673899c0ff611f14af978d725e2c999", size = 5246003, upload-time = "2026-08-19T04:59:59.764Z" },
+    { url = "https://files.pythonhosted.org/packages/05/b3/a43012748fb861c914c5eac1c1a3bad44282e767499cd02280d4d1edf092/lxml-6.1.2-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:92c2b366028ac01e90399e6d17734ce6e4f4aeddd8ba75fbaf80ea11d6c6d645", size = 5354047, upload-time = "2026-08-19T05:00:21.657Z" },
+    { url = "https://files.pythonhosted.org/packages/8b/cb/813021d9a445713b8d758b9e5eae2ed392cd598d9f119d9b053b37c2ab93/lxml-6.1.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:7e81fc065ede5d58dd0bf0912025aee1bd04c52c2affd61fdb93226a97ce2fc6", size = 4704382, upload-time = "2026-08-19T05:00:47.067Z" },
+    { url = "https://files.pythonhosted.org/packages/17/c9/1155299f4577bebf3c280497534a73e4b8ad8cab3b96074731ad10949d4e/lxml-6.1.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:633ac039cb32366dd5935868e041e385875c017b8cd54ea56aeee3fe29ca5935", size = 5258530, upload-time = "2026-08-19T05:01:14.893Z" },
+    { url = "https://files.pythonhosted.org/packages/25/6e/d76e58384b378b877e140e25b9a9835da00035f81ff70cbe943a3749bf27/lxml-6.1.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f3194777c0d05945ac91d8594be25d2679d1d826e01e1fc90bae568ff3a547b", size = 5089919, upload-time = "2026-08-19T05:01:33.602Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/b7/898013c0f8891481d0624ab3bd5dd8c8ff827232dfee2a5d1f8bf970a7cc/lxml-6.1.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1133bd969f2bfcc6b0c0cf7cdf5f2631e62b23fa2471ee8bd44f6ab73554ee9a", size = 4741972, upload-time = "2026-08-19T05:01:38.18Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/47/efb53c4d7b655831c03317a450d9da439b0829c61f34d9d4fe7c863445d6/lxml-6.1.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1edca8f4a92b94e873093df959f141d388f2141fcad0c47598442fb4730ef57a", size = 5683241, upload-time = "2026-08-19T05:02:00.731Z" },
+    { url = "https://files.pythonhosted.org/packages/da/0a/0ff36a584cbba14a71326ee8a5300694400f0b97927d1f90a87d95b17d4a/lxml-6.1.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:8512b3775d68994dd1d6d533161e0a214f2ad9c634659d34a99c98e86c6c3d68", size = 5245892, upload-time = "2026-08-19T05:02:06.108Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/9e/303717a1aa56d4bd775c91936717d3c9e8d999a8e8b68b00979c4c1f93d0/lxml-6.1.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5005c0c9e4d749a76a2ff8bd5918a8bb248df8e08e73a55654b9f79c9cd1e2b", size = 5269528, upload-time = "2026-08-19T05:02:09.883Z" },
+    { url = "https://files.pythonhosted.org/packages/ad/c2/2ae7cb97089eb86bf0689516db3cf280a007b6145853d2a0235a1f01683d/lxml-6.1.2-cp314-cp314-win32.whl", hash = "sha256:e17e2c30e27f56da5551e7a425888b45f013e940b99ab07d125a1c33f77a4605", size = 3662743, upload-time = "2026-08-19T05:03:02.513Z" },
+    { url = "https://files.pythonhosted.org/packages/77/13/a3d483230a09201e211ceb1aa208b1374d27d23b8b180d74dba14b30f6b3/lxml-6.1.2-cp314-cp314-win_amd64.whl", hash = "sha256:87e9673cd8a3445024fe38e7f91b55fa3428437eec9b7a7ff7d81979520c0d2d", size = 4073942, upload-time = "2026-08-19T05:03:04.864Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/f1/c1445d4b6ad7c51e39d4e2ebbf015a4880f5b297a4ab0e77e4d0e5b70110/lxml-6.1.2-cp314-cp314-win_arm64.whl", hash = "sha256:878e7c8ada8f92c52f13f35a2ab98ef0adf7fd0211d164fc2af589e4c3cfed63", size = 3749235, upload-time = "2026-08-19T05:03:07.239Z" },
+    { url = "https://files.pythonhosted.org/packages/9d/eb/598c76f4ce19a67c635e86a46d880cc854f308f39a6f1fdf13bbb01813ec/lxml-6.1.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:94162456ed0a64fb1c06915df5bd06af4675ae3966d6048fcb73b0906e0e0222", size = 8860315, upload-time = "2026-08-19T05:02:14.39Z" },
+    { url = "https://files.pythonhosted.org/packages/da/c7/1f9fac7b566a86ad0da13dcc0259164266469c0ad86744c740ccd5c2a081/lxml-6.1.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4b0fa7109b1d0bc1747d8241a0853e135eefb1c978685241b544c46937383efd", size = 4755176, upload-time = "2026-08-19T05:02:18.705Z" },
+    { url = "https://files.pythonhosted.org/packages/3e/1b/cfda9307388d496e7eeb7493d9455896b8137ed95f51f3d6ae6ddcc14a47/lxml-6.1.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:604f4778632588d7c000e7e19430639dc12fca58b5b6e99edffba7631725ef0e", size = 4979444, upload-time = "2026-08-19T05:02:21.262Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/71/f732c8919c45b7f29acf443288c6e90036877a67bfeeb1acceb0fffa011b/lxml-6.1.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a096d6a5f96b776a5b020cb45c17c545effd2a3b6639e6fa97bc95537600923", size = 5115887, upload-time = "2026-08-19T05:02:23.62Z" },
+    { url = "https://files.pythonhosted.org/packages/30/00/121d52b944f41e33ea86c62875f902d24982842dc7231ab154ac5a6c6593/lxml-6.1.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6454d184d556eaf4cb3d6f69e405d21602d6fdcf08b8d57796824275986c6595", size = 5032418, upload-time = "2026-08-19T05:02:26.114Z" },
+    { url = "https://files.pythonhosted.org/packages/70/19/cadb73c7fe48c7563dc8ab62ea53d5b920c8911bfb808507a6daa82e78d2/lxml-6.1.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b68f2548259bb04e0b3d5df0c397abe8b0080f5e1ffe4019fb7a8bf01a9339e", size = 5603304, upload-time = "2026-08-19T05:02:28.694Z" },
+    { url = "https://files.pythonhosted.org/packages/13/32/9de126a14d5a5db8c371c5ec869178417db226707b62a47273a95ae6df7f/lxml-6.1.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c9cc4b6532abe154dbdebb42aaba8d52c852919591e45067f5b7d46a0405e88", size = 5228938, upload-time = "2026-08-19T05:02:30.99Z" },
+    { url = "https://files.pythonhosted.org/packages/ff/9b/22dd9e843629ed04652591fb220eb2bf2394d97be3be377d60d8083405d7/lxml-6.1.2-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:57188e441ab24f906bd5a5c14eb55363ab51aa6c0de549f3dd320043721cc118", size = 5317790, upload-time = "2026-08-19T05:02:33.301Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/2c/b12a1dc121f81c280635c721c7bcaa341441fcbe37397f60b8915048aece/lxml-6.1.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d0bfd719c254bbe60ea022cff0e6ffb799a6fa7d4d72852cebe0257957b32d68", size = 4646468, upload-time = "2026-08-19T05:02:35.504Z" },
+    { url = "https://files.pythonhosted.org/packages/57/41/fd87a41edc531e7969c25ab1d6b52b5b041eb108b88f6394d6afb4374396/lxml-6.1.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:be6f87cd224254a8f81324e34cc655508b83f1d70458a1a39857ad2aa9925852", size = 5240607, upload-time = "2026-08-19T05:02:37.805Z" },
+    { url = "https://files.pythonhosted.org/packages/6e/30/713ba813b6e6673c6dc34733746516017efcd17949b767b154cc50bccf20/lxml-6.1.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:074a88f70a7360a4a0c5be5d898062cd26f898c25b459efb1bdd43ae700c5a1a", size = 5086495, upload-time = "2026-08-19T05:02:40.099Z" },
+    { url = "https://files.pythonhosted.org/packages/33/f8/6532ce0fecd9c326d06b08274ee075cc28dbc9f5e9285355db8504689114/lxml-6.1.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:9031f5f01452681abf39fdd65f84a70cb01a7572a1bbf570042e826b1232d07b", size = 4758801, upload-time = "2026-08-19T05:02:45.434Z" },
+    { url = "https://files.pythonhosted.org/packages/74/b6/5a1f7833ebaa0dd33c28f6f9755ec6ff3891bf63f097634b44e6da1bb65e/lxml-6.1.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:cfeac14425fc7a6fca7864b774d4ee63547926158f4a18c67d77b2c9a948acf1", size = 5626977, upload-time = "2026-08-19T05:02:48.092Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/20/6ae0fc1b45e20877cdcfb1168ceeaf9abb0fba5ed36bd639a260e7b2101e/lxml-6.1.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:8ec111ff8067325f85c08aa9c2b26179ec0537bb89c003fde31127139f85f82d", size = 5235036, upload-time = "2026-08-19T05:02:50.726Z" },
+    { url = "https://files.pythonhosted.org/packages/47/b4/2bc7b37fbb990ccfb7d30393660741592177224a94e07d842c8da70638e8/lxml-6.1.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:48e912f37c99a297175ba955f55a47c0e1c834b506ef162e52a6e4fe276e6e45", size = 5252270, upload-time = "2026-08-19T05:02:53.454Z" },
+    { url = "https://files.pythonhosted.org/packages/4d/0b/07fb8e1dee29a78e2c5fa5c6c914218be76a6406baff27907429566e90ec/lxml-6.1.2-cp314-cp314t-win32.whl", hash = "sha256:7c444c3a6e8e75334879980eed96568f0e12064c8b1913424eac1805e976736b", size = 3902666, upload-time = "2026-08-19T05:02:55.607Z" },
+    { url = "https://files.pythonhosted.org/packages/58/ab/3371527bd9820aae6f511697c93032ed197b0d8dab0f17818f18d3099637/lxml-6.1.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7f35ba7667004ecdafebbe08da7c9fa06ee6195275bb7ef7a29ee1901e69519c", size = 4401011, upload-time = "2026-08-19T05:02:57.899Z" },
+    { url = "https://files.pythonhosted.org/packages/e6/bb/e6de9b2546a4e6df4fb52fb18921906a8b7a041aba06570995759a4d6d8b/lxml-6.1.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d117f39b28ab8a330a74abdbe61c2255b51973b238db25fd6c2448de1eb2a02d", size = 3823384, upload-time = "2026-08-19T05:03:00.371Z" },
+    { url = "https://files.pythonhosted.org/packages/0e/83/7ff98683e14a148191278728d11ba782c3d5137886d49fd95ab4036efa1b/lxml-6.1.2-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:1e3c67b817867c484794d7fe0d73045d7d0c67460c78a0a1249a9e92266e6a0e", size = 8609183, upload-time = "2026-08-19T04:58:32.19Z" },
+    { url = "https://files.pythonhosted.org/packages/24/39/c39f05e8240e98009dd3d4ceb248319d0f36467babc5f90a909ed0c5b68a/lxml-6.1.2-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:d3e97ac4353cca3fbbfa829bc0c6a913771573d1c6d46932d4335c46f2b7796a", size = 4639898, upload-time = "2026-08-19T04:58:39.017Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/bf/25e26b089510940a0777ab334357874569255e50930224c8159cd649e754/lxml-6.1.2-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:827438bf6c8292d22a409bb7990d7cffce410f33e7664e46ca74d2ecc26975ef", size = 5037527, upload-time = "2026-08-19T04:58:46.224Z" },
+    { url = "https://files.pythonhosted.org/packages/65/6d/aed3a58a3d662f7367a537fabe8c549f1446dbd043719e0ae8cd53f47819/lxml-6.1.2-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c470d192e27f97842a068cf12a1c1296b20ca716c56a9249715c6654bc192d19", size = 5661918, upload-time = "2026-08-19T04:59:02.534Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/ca/706d32b6957c0c2e005a9833e8fc528449196b38d5cfcf9e0fd86a96fb00/lxml-6.1.2-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef0b8ba6e13597f681b2b4924ca9c4e8c88420bf0e21d9a9006c757f2fc39d1f", size = 5249359, upload-time = "2026-08-19T05:04:01.956Z" },
+    { url = "https://files.pythonhosted.org/packages/ac/e9/445ff43f56fcffa06f6f3a7189920c216f3eacef68ef834d4111cdbd86ba/lxml-6.1.2-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:65c32ddc5d0750129c7b119fb57d48192b76d334c21e6b690d19dfb06b34af79", size = 4704548, upload-time = "2026-08-19T05:04:04.57Z" },
+    { url = "https://files.pythonhosted.org/packages/69/78/20b8b7e79a1b1d9cd4465c332d62962858562b446692f16a27068fa54b85/lxml-6.1.2-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0aa07065497f191ad26c4b587ce5dbb5a7105285a3789aafd0661750e8bac537", size = 5261170, upload-time = "2026-08-19T05:04:07.336Z" },
+    { url = "https://files.pythonhosted.org/packages/54/ca/84a0e1148bf511e12e0d99732a4e136a3bf1b91622f0a1b197796e2ff984/lxml-6.1.2-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:cde6b8db7d2e5135129eb5e74b7b44dd2053aa767cd5023541fccedddc262453", size = 5090576, upload-time = "2026-08-19T05:04:09.968Z" },
+    { url = "https://files.pythonhosted.org/packages/6f/f8/1ef6fc7070bed8753315f2e4ea66bc0d37620e1444d014db7f0267b8faaf/lxml-6.1.2-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:b28842b30c4bc2e6afe137d98a5d2071a62589471e76d053bea55b0e53298af9", size = 4744614, upload-time = "2026-08-19T05:04:12.717Z" },
+    { url = "https://files.pythonhosted.org/packages/87/f6/3a4824cd1c1b81d996d2d75bbd176ba13fbe9b5d89489290d93ff9558486/lxml-6.1.2-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:11f529062255209a421ae4de5b1bb36b2f0a2e1a700745e675a4bf4084d13c00", size = 5685792, upload-time = "2026-08-19T05:04:15.367Z" },
+    { url = "https://files.pythonhosted.org/packages/64/9a/f133bf16a67149e00ca5d8a8f1ae662c30a86c303aa242693b67f8e19856/lxml-6.1.2-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:f8b89b3be75a37509602b03f9cfa1a28298d4eed4625748148307aeb907901b7", size = 5248972, upload-time = "2026-08-19T05:04:18.491Z" },
+    { url = "https://files.pythonhosted.org/packages/50/63/273e7e8a73a5d183d8552dfdaa131dfda0292ddab7bcddc5a66a0ae525d8/lxml-6.1.2-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:1a2331da06dd55a8184985306eb2afd72d708283ce7e85d67bba77317b785060", size = 5271809, upload-time = "2026-08-19T05:04:21.448Z" },
+    { url = "https://files.pythonhosted.org/packages/49/eb/614117c36a28909e79ff7cdec87008f0bd996478f35cf72309189cf398b1/lxml-6.1.2-cp315-cp315-win32.whl", hash = "sha256:442766b326d9892585a64e8c6c4b5ab81d0e6c0538c9f0fc11a84dc101a5d97f", size = 3662854, upload-time = "2026-08-19T05:05:07.141Z" },
+    { url = "https://files.pythonhosted.org/packages/5c/e9/06aee6107cf8e7b870f10f82539f366cba10dc6053144cca80e838caf8c8/lxml-6.1.2-cp315-cp315-win_amd64.whl", hash = "sha256:a7fd1dd6faa3df9dcd8f1765237362cd885ca62cdf77a7c5f5ea383ae5b6048b", size = 4074590, upload-time = "2026-08-19T05:05:09.697Z" },
+    { url = "https://files.pythonhosted.org/packages/84/bf/dad9b6baf9b26d79584834e15cef2a5dd0a13c7b1df08831e8f18244b494/lxml-6.1.2-cp315-cp315-win_arm64.whl", hash = "sha256:054175250531a5fb102d485743ff16412279c93add12385b3b1c3d7b16d8deaa", size = 3749336, upload-time = "2026-08-19T05:05:12.334Z" },
+    { url = "https://files.pythonhosted.org/packages/5f/9d/cd0c43d45e2eb52df7735c6558f24054ca633499191899b0cb9040fbbc3c/lxml-6.1.2-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:84a2a46b93b789d8acb44cfcb3d967ce9dbe29884ddb93fbb1a33f0e0c8fcd86", size = 8857688, upload-time = "2026-08-19T05:04:24.648Z" },
+    { url = "https://files.pythonhosted.org/packages/0b/26/27093dc1a9edbdd8a54652f237a387f7e63ec0192efe708bc2576d8a1383/lxml-6.1.2-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:4aced3284e0353c798b060fe2c175eb81410e99b9a7e2ae6951be5333732b111", size = 4754422, upload-time = "2026-08-19T05:04:27.645Z" },
+    { url = "https://files.pythonhosted.org/packages/2c/ee/502f7c93507f57eb496744a64da8f4ca86855cf88e48d14584342f1bfd92/lxml-6.1.2-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:47c92dc5167de16e27ace8332454f12ba172dcab04f7a78a9eae14e2e41b6a41", size = 5033396, upload-time = "2026-08-19T05:04:30.054Z" },
+    { url = "https://files.pythonhosted.org/packages/bf/72/c4cbbe72f951650f2afe43a70e51687e111d82b9bec46e3310ea76419d46/lxml-6.1.2-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:40366c23a938008a3bedfcfd80709b3a857c188b4d710b083e978ef5d2c1c715", size = 5615298, upload-time = "2026-08-19T05:04:32.752Z" },
+    { url = "https://files.pythonhosted.org/packages/14/83/a3df966d6d7b6513e9dfb6fbfb041c0619642170359c1b36ab20a83e59eb/lxml-6.1.2-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1c4c6dc1b2485aaa4adfb6ed754f90dddcb2b96a66bbebc9e1ac242b5ce5e818", size = 5236282, upload-time = "2026-08-19T05:04:35.762Z" },
+    { url = "https://files.pythonhosted.org/packages/4f/85/8692ec8173c9f8d295735b9bf410d202317e7b3ed11141e80a30f421f409/lxml-6.1.2-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:3a698fad6f122a9b3e2dc2fb598c1de7329c74a67c7a334c9109a440de2508e5", size = 4650647, upload-time = "2026-08-19T05:04:38.396Z" },
+    { url = "https://files.pythonhosted.org/packages/11/e7/dbe3cece28a5bf82997a091d9dbb0fc49e725a5fa86550897ee2cf6412e6/lxml-6.1.2-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14879fa5eb2b793c040bbfcb62011aa3015c65d6c9875e063ea98ce2029d51fb", size = 5243387, upload-time = "2026-08-19T05:04:41.247Z" },
+    { url = "https://files.pythonhosted.org/packages/99/a9/81a2d27640db0d27200b2f32339a54e74c36d58feb5ad528b87d52a59ecc/lxml-6.1.2-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:b631174cd2e4d9f8a94ef17f911c6ded10ede93b5e7860dee7bbf85961d321e9", size = 5092624, upload-time = "2026-08-19T05:04:43.919Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/f4/0b0304c70c087f618d95b0306738b070bd556afd09c2c92589b78dbe5eb0/lxml-6.1.2-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:ceafa5e0536c62a5cd9f65327fa0b57d6f0b0e3435daf2c98a78d0dde7ecbae1", size = 4758742, upload-time = "2026-08-19T05:04:46.615Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/ec/f9fc45f1d01b632b673e11880e75292dff9953db9f426d1a38201b8eb5f5/lxml-6.1.2-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7c482e87cc86bed78a50462560675bc2c348ef72c47596f9b933346d5a8e920e", size = 5649540, upload-time = "2026-08-19T05:04:49.777Z" },
+    { url = "https://files.pythonhosted.org/packages/6a/0b/d65e0458c2bcce0df68d5cc29ad0006e76446f02d9e50caf188fd1fb8bae/lxml-6.1.2-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:b1c0d2dde8a50520efc51644587f0fc4810e3af7d3e029d7af0be93bf39e2b5c", size = 5234869, upload-time = "2026-08-19T05:04:52.972Z" },
+    { url = "https://files.pythonhosted.org/packages/ee/62/1fee828238badd3bfe9544f5cc9ce6ded421ef38e9634030445dedd78b36/lxml-6.1.2-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:dd7ea3fa47154b9fff90591b961e41b3718bd7fcd5bc2d9bb47e9845c8ace088", size = 5259992, upload-time = "2026-08-19T05:04:56.028Z" },
+    { url = "https://files.pythonhosted.org/packages/20/18/35fb14dd6baccbffa6daeb2369802f04a94e3f73db3c7bb405dbab009729/lxml-6.1.2-cp315-cp315t-win32.whl", hash = "sha256:87534cec6ea325435e4adf2326b0cf3110eee9a47abf73652eb155db639c08c6", size = 3901151, upload-time = "2026-08-19T05:04:58.671Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/b6/07530896ca062bc3d2f09d5cb8a48e799c05b12c496205db03159ba13b6c/lxml-6.1.2-cp315-cp315t-win_amd64.whl", hash = "sha256:4e220a9c297e5d36895d489a08c9a3f1f6193b6414e702c5fb751e4a3767f8d0", size = 4395355, upload-time = "2026-08-19T05:05:01.651Z" },
+    { url = "https://files.pythonhosted.org/packages/31/a8/237d8de1d77085cfd41d0c6049a044d8d01886f3afb7f1eda2f43d900a96/lxml-6.1.2-cp315-cp315t-win_arm64.whl", hash = "sha256:f16a407766bac51c65d605b06d900821751a79aa20e12185f273f14a17180e7b", size = 3822823, upload-time = "2026-08-19T05:05:04.63Z" },
+]
+
 [[package]]
 name = "markdown-it-py"
 version = "4.2.0"
@@ -1518,6 +1719,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" },
 ]
 
+[[package]]
+name = "parse"
+version = "1.22.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a4/f2/0b504486c2a5564798607d3860e48ed19c6443d5e9cc3ec61cc6b8b4ef58/parse-1.22.1.tar.gz", hash = "sha256:d3a4740ec3da338e2b258b2d69741b731eadfddca59e24a14bc4ee5fce38c911", size = 36970, upload-time = "2026-05-26T03:44:52.624Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/6f/c5/7c16e99869e1f422629092cfd23e3b58e461988c3f9c36fd3624bb4142e6/parse-1.22.1-py2.py3-none-any.whl", hash = "sha256:20f0925a46f06602485ac90d751764d0697fd8455aaa97489ba8953a4b66de32", size = 20925, upload-time = "2026-05-26T03:44:51.156Z" },
+]
+
 [[package]]
 name = "parso"
 version = "0.8.7"
@@ -1956,6 +2166,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" },
 ]
 
+[[package]]
+name = "python-daemon"
+version = "2.1.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "docutils" },
+    { name = "lockfile" },
+    { name = "setuptools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b2/fb/a280d65f81e9d69989c8d6c4e0bb18d7280cdcd6d406a2cc3f4eb47d4402/python-daemon-2.1.2.tar.gz", hash = "sha256:261c859be5c12ae7d4286dc6951e87e9e1a70a882a8b41fd926efc1ec4214f73", size = 76176, upload-time = "2016-10-26T10:16:01.909Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/f4/59/816004688f8e8602526553cd96226f34657ce4a86daa2240c3eebb0568a3/python_daemon-2.1.2-py2.py3-none-any.whl", hash = "sha256:53da55aec3bb67b576e13a8091a2181f99b395c2eec32a5a0d91d347a5c420a7", size = 19425, upload-time = "2016-10-26T10:16:05.441Z" },
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -2248,6 +2472,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/89/d3/726bd88f0eece09ddf431bea4c9191c18e7a8d070b854eb0014d447712ee/sentry_sdk-2.66.1-py3-none-any.whl", hash = "sha256:86002793161d9a95ef04bdd8d442e9bfece5d989b755f05d6360215094a7aff6", size = 505555, upload-time = "2026-07-22T12:26:52.71Z" },
 ]
 
+[[package]]
+name = "setuptools"
+version = "84.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" },
+]
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@@ -2266,6 +2499,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
 ]
 
+[[package]]
+name = "smmap"
+version = "5.0.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" },
+]
+
 [[package]]
 name = "stack-data"
 version = "0.6.3"
@@ -2303,6 +2545,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9b/24/84ce997e8ae6296168a74d0d9c4dde572d90fb23fd7c0b219c30ff71e00e/tbb-2021.13.1-py3-none-win_amd64.whl", hash = "sha256:cbf024b2463fdab3ebe3fa6ff453026358e6b903839c80d647e08ad6d0796ee9", size = 286908, upload-time = "2024-08-07T15:09:05.677Z" },
 ]
 
+[[package]]
+name = "tenacity"
+version = "8.5.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/4d/6a19536c50b849338fcbe9290d562b52cbdcf30d8963d3588a68a4107df1/tenacity-8.5.0.tar.gz", hash = "sha256:8bc6c0c8a09b31e6cad13c47afbed1a567518250a9a171418582ed8d9c20ca78", size = 47309, upload-time = "2024-07-05T07:25:31.836Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/d2/3f/8ba87d9e287b9d385a02a7114ddcef61b26f86411e121c9003eb509a1773/tenacity-8.5.0-py3-none-any.whl", hash = "sha256:b594c2a5945830c267ce6b79a166228323ed52718f30302c1359836112346687", size = 28165, upload-time = "2024-07-05T07:25:29.591Z" },
+]
+
 [[package]]
 name = "threadpoolctl"
 version = "3.6.0"
@@ -2596,6 +2847,20 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/bd/6e/95b0e537de1f4d4301f76f944642c6da50d1511cc7b3d64dc418a66c7509/wcwidth-0.8.1-py3-none-any.whl", hash = "sha256:f453740b1e4a4f3291faa37944c555d71056c4da08d59809b307ef4feba695c8", size = 323092, upload-time = "2026-06-08T05:57:21.413Z" },
 ]
 
+[[package]]
+name = "webdavclient3"
+version = "3.14.7"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "lxml" },
+    { name = "python-dateutil" },
+    { name = "requests" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/67/d8/ca3981053ed553363322f71745f543186b93439b6417f5d6ca91d4b4fec7/webdavclient3-3.14.7.tar.gz", hash = "sha256:6c04252b579bc015cec78081480c63eadf1030f382768248777c6203f059b3f5", size = 30836, upload-time = "2026-02-06T17:54:15.506Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/89/5e/0b1c2f494d03c4acbc44567fa68b954cd0fa3f21eb3f9528011da371f9b1/webdavclient3-3.14.7-py3-none-any.whl", hash = "sha256:a904381da8e3ae77b4ca9e11e05058d91a07704254d71c193c797f7c2fb15025", size = 22887, upload-time = "2026-02-06T17:54:14.068Z" },
+]
+
 [[package]]
 name = "xxhash"
 version = "3.7.0"