From 339d2fb4734984b41e61638528c1d59b859775ac Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Thu, 9 Jul 2026 09:09:14 +0200 Subject: [PATCH] Error on missing secondary lists instead of silently zeroing Stage-2 targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A parquet that carries child_track_ids/e_sec but was never run through the parent->child join lacks the per-secondary columns (sec_E_list/sec_pdg_list/ sec_dir_list). build_features would fall back to all-zero sec_cont/sec_pdg_idx, collapsing every secondary to PDG index 0 and a constant energy fraction — a broken Stage 2 that trained with no error (single-species validation tables). Add an opt-in require_secondaries flag that raises when n_sec > 0 but the lists are absent, and enable it on the training paths (StreamingStepsDataset and the normalizer-fit pass). giant predict keeps the default False for Stage-1-only use. Co-Authored-By: Claude Opus 4.8 --- giant/data/dataset.py | 1 + giant/data/transforms.py | 25 +++++++++++++++++++++ giant/pipeline.py | 2 +- tests/test_transforms.py | 48 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/giant/data/dataset.py b/giant/data/dataset.py index e96b8f5..bbac493 100644 --- a/giant/data/dataset.py +++ b/giant/data/dataset.py @@ -101,6 +101,7 @@ class StreamingStepsDataset(IterableDataset): self.mat_map, cond_normalizer=self.cond_normalizer, target_normalizer=self.target_normalizer, + require_secondaries=True, ) ) buf_cont.append(cond_cont) diff --git a/giant/data/transforms.py b/giant/data/transforms.py index c068d63..fe3a81d 100644 --- a/giant/data/transforms.py +++ b/giant/data/transforms.py @@ -433,6 +433,7 @@ def build_features( cond_normalizer: Normalizer | None = None, target_normalizer: Normalizer | None = None, fit: bool = False, + require_secondaries: bool = False, ) -> tuple[ np.ndarray, np.ndarray, @@ -450,6 +451,11 @@ def build_features( sec_cont: (N, K_MAX, 4) continuous secondary targets [stick_logit, dir_local] sec_pdg_idx: (N, K_MAX) integer PDG model-indices; used to look up embedding targets in the training loop + + require_secondaries: when True, raise if any step has n_sec > 0 but the + per-secondary list columns are absent (a mis-converted file that would + otherwise silently zero all Stage-2 targets). Training paths set this; + Stage-1-only callers (e.g. `giant predict`) leave it False. """ from giant.constants import K_MAX @@ -512,6 +518,25 @@ def build_features( sec_pdg_list ).astype(np.int64) else: + # Guard against silently training Stage 2 on zeroed targets: if any step + # actually spawned secondaries (n_sec > 0, from child_track_ids) but the + # per-secondary columns are absent, the file was never run through the + # parent->child join (steps_to_parquet._add_secondary_attributes / + # `dwarf convert`). Zero-filling here would collapse every secondary to + # PDG index 0 and a constant energy fraction — a broken Stage 2 with no + # error. Callers that only need Stage-1 (e.g. `giant predict`) keep the + # default require_secondaries=False. + if require_secondaries and n_sec_raw.max(initial=0) > 0: + n_with_sec = int((n_sec_raw > 0).sum()) + raise ValueError( + f"{n_with_sec} step(s) have secondaries (n_sec > 0) but the " + "per-secondary columns (sec_E_list / sec_pdg_list / sec_dx_list " + "…) are missing. This parquet was not run through the " + "parent->child join (steps_to_parquet._add_secondary_attributes " + "/ `dwarf convert`); training on it would silently zero all " + "Stage-2 targets. Re-convert the file, or pass " + "require_secondaries=False for Stage-1-only use." + ) N = len(n_sec) sec_cont = np.zeros((N, K_MAX, 4), dtype=np.float32) sec_pdg_idx = np.zeros((N, K_MAX), dtype=np.int64) diff --git a/giant/pipeline.py b/giant/pipeline.py index 10b5746..c542584 100644 --- a/giant/pipeline.py +++ b/giant/pipeline.py @@ -64,7 +64,7 @@ def run_train_job( continue chunk_tr = {k: v[mask] for k, v in chunk.items()} cond_cont, _, target_s1, _n_sec, _sec_cont, _sec_pdg, _, _ = build_features( - chunk_tr, pdg_map, mat_map + chunk_tr, pdg_map, mat_map, require_secondaries=True ) cond_acc.update(cond_cont) tgt_acc.update(target_s1) diff --git a/tests/test_transforms.py b/tests/test_transforms.py index d92f347..45cfc04 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -239,3 +239,51 @@ def test_build_features_clamps_n_sec_label_to_k_max(): assert n_sec.max() <= K_MAX np.testing.assert_array_equal(n_sec, [0, 5, K_MAX]) + + +def _step_data_no_sec_lists(n_sec: np.ndarray) -> dict: + """Minimal build_features input with n_sec but no per-secondary list columns + (mimics a parquet that skipped the parent->child join).""" + N = len(n_sec) + rng = np.random.default_rng(0) + return { + "pdg": np.full(N, 11, dtype=np.int32), + "material": np.full(N, "PbWO4", dtype=object), + "pre_pos": rng.standard_normal((N, 3)).astype(np.float32), + "pre_E": np.full(N, 10.0, dtype=np.float32), + "pre_dir": np.tile(np.array([0, 0, 1], dtype=np.float32), (N, 1)), + "layer_id": np.zeros(N, dtype=np.int32), + "n_sec": np.asarray(n_sec, dtype=np.int32), + "e_sec": np.full(N, 1.0, dtype=np.float32), + "step_length": np.full(N, 1.0, dtype=np.float32), + "post_E": np.full(N, 9.0, dtype=np.float32), + "edep": np.full(N, 1.0, dtype=np.float32), + "post_dir": np.tile(np.array([0, 0, 1], dtype=np.float32), (N, 1)), + "post_pos": rng.standard_normal((N, 3)).astype(np.float32), + } + + +def test_build_features_require_secondaries_raises_when_lists_missing(): + """A parquet with n_sec > 0 but no per-secondary list columns was never run + through the parent->child join; require_secondaries must catch it instead of + silently zeroing every Stage-2 target (regression: this collapsed the + secondary species to a single PDG index during training).""" + data = _step_data_no_sec_lists(np.array([0, 2, 1])) + pdg_map, mat_map = {11: 0}, {"PbWO4": 0} + + with pytest.raises(ValueError, match="per-secondary columns"): + build_features(data, pdg_map, mat_map, require_secondaries=True) + + +def test_build_features_require_secondaries_ok_when_no_secondaries(): + """require_secondaries only fires when secondaries actually exist; a file + with n_sec == 0 everywhere (e.g. Stage-1-only) must still load.""" + data = _step_data_no_sec_lists(np.zeros(3, dtype=np.int32)) + pdg_map, mat_map = {11: 0}, {"PbWO4": 0} + + _, _, _, _, sec_cont, sec_pdg_idx, _, _ = build_features( + data, pdg_map, mat_map, require_secondaries=True + ) + + assert not sec_cont.any() + assert not sec_pdg_idx.any()