lars
86fc46b5a8
CI / Lint (ruff check) (push) Successful in 1m1s
CI / Format (ruff format) (push) Successful in 1m5s
CI / Type check (ty) (push) Successful in 1m6s
CI / Tests (push) Successful in 1m52s
CI / Lint (ruff check) (pull_request) Successful in 1m3s
CI / Format (ruff format) (pull_request) Successful in 1m4s
CI / Type check (ty) (pull_request) Successful in 1m4s
CI / Tests (pull_request) Successful in 1m42s
CI / Bump version, build & publish wheel (push) Has been skipped
CI / Bump version, build & publish wheel (pull_request) Has been skipped
Add a second parallelism axis to giant analyze: each plot's data can now be split into a configurable number of event_id-disjoint chunks, each computed as its own HTCondor job, bounding per-job walltime and scan cost on large rollout/reference files instead of one job re-scanning the whole file per plot. Every PlotSpec now splits into compute_partial (runs per (plot, chunk) job against a chunk-filtered Bundle) and finalize (merges chunks - elementwise sum for fixed-edge histograms/species shares, concatenate -then-recompute for specs that derive edges or mean/std from the full per-event/per-secondary array). Router diagnostics stay chunkable=False and always run as a single job. giant analyze render now joins every plot's chunk partials (merge_all) before rendering, transparently. New: --chunks on `analyze prep`/`analyze submit`, --chunk on `analyze compute-one`, and a new `analyze merge-one` command.
66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
"""The compact, self-describing artifact a compute job produces per plot.
|
|
|
|
Serialized as small JSON (no pickle, no per-event arrays) so it is trivially
|
|
transferable off the batch worker and human-inspectable. ``render.py`` dispatches
|
|
on ``kind`` and needs nothing but this file.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import asdict, dataclass, field
|
|
from pathlib import Path
|
|
|
|
# Reduced.kind values:
|
|
# "overlay_hist" rollout vs reference density histogram over shared edges
|
|
# "grouped_hist" one panel per group (energy/pdg/material), each an overlay
|
|
# "profile" edep-weighted mean +/- event-RMS vs depth/radius, two series
|
|
# "bar" per-category rollout vs reference bars (share / counts)
|
|
# "single_hist" one series only (e.g. rollout leakage; reference has none)
|
|
# "router_gating" stacked mean MoE gate weight vs energy, rollout + reference
|
|
# "router_share" stacked bar of MoE top-1 dispatch share by category
|
|
# "unavailable" plot not applicable to this run (e.g. non-MoE checkpoint)
|
|
|
|
|
|
@dataclass
|
|
class Reduced:
|
|
id: str
|
|
family: str
|
|
kind: str
|
|
title: str
|
|
xlabel: str
|
|
payload: dict
|
|
meta: dict = field(default_factory=dict)
|
|
|
|
def save(self, path: str | Path) -> None:
|
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(path).write_text(json.dumps(asdict(self)))
|
|
|
|
@classmethod
|
|
def load(cls, path: str | Path) -> "Reduced":
|
|
return cls(**json.loads(Path(path).read_text()))
|
|
|
|
|
|
@dataclass
|
|
class Partial:
|
|
"""The raw, not-yet-finalized output of one ``(plot, chunk)`` compute job.
|
|
|
|
``data`` holds whatever shape that plot's ``PlotSpec.compute_partial``
|
|
returns — a raw sum-mergeable count dict, or a raw per-event/per-secondary
|
|
array to be concatenated across chunks — never a finished histogram/profile.
|
|
``PlotSpec.finalize`` is the only thing that knows how to interpret it.
|
|
"""
|
|
|
|
id: str
|
|
family: str
|
|
chunk: int
|
|
data: dict
|
|
|
|
def save(self, path: str | Path) -> None:
|
|
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
|
Path(path).write_text(json.dumps(asdict(self)))
|
|
|
|
@classmethod
|
|
def load(cls, path: str | Path) -> "Partial":
|
|
return cls(**json.loads(Path(path).read_text()))
|