Error on missing secondary lists instead of silently zeroing Stage-2 targets

A parquet that carries child_track_ids/e_sec but was never run through the
parent->child join lacks the per-secondary columns (sec_E_list/sec_pdg_list/
sec_dir_list). build_features would fall back to all-zero sec_cont/sec_pdg_idx,
collapsing every secondary to PDG index 0 and a constant energy fraction — a
broken Stage 2 that trained with no error (single-species validation tables).

Add an opt-in require_secondaries flag that raises when n_sec > 0 but the lists
are absent, and enable it on the training paths (StreamingStepsDataset and the
normalizer-fit pass). giant predict keeps the default False for Stage-1-only use.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

giant/data/dataset.py

@@ -101,6 +101,7 @@ class StreamingStepsDataset(IterableDataset):
                         self.mat_map,
                         cond_normalizer=self.cond_normalizer,
                         target_normalizer=self.target_normalizer,
+                        require_secondaries=True,
                     )
                 )
                 buf_cont.append(cond_cont)

giant/data/transforms.py

@@ -433,6 +433,7 @@ def build_features(
     cond_normalizer: Normalizer | None = None,
     target_normalizer: Normalizer | None = None,
     fit: bool = False,
+    require_secondaries: bool = False,
 ) -> tuple[
     np.ndarray,
     np.ndarray,
@@ -450,6 +451,11 @@ def build_features(
     sec_cont: (N, K_MAX, 4) continuous secondary targets [stick_logit, dir_local]
     sec_pdg_idx: (N, K_MAX) integer PDG model-indices; used to look up embedding
                  targets in the training loop
+
+    require_secondaries: when True, raise if any step has n_sec > 0 but the
+        per-secondary list columns are absent (a mis-converted file that would
+        otherwise silently zero all Stage-2 targets). Training paths set this;
+        Stage-1-only callers (e.g. `giant predict`) leave it False.
     """
     from giant.constants import K_MAX
 
@@ -512,6 +518,25 @@ def build_features(
             sec_pdg_list
         ).astype(np.int64)
     else:
+        # Guard against silently training Stage 2 on zeroed targets: if any step
+        # actually spawned secondaries (n_sec > 0, from child_track_ids) but the
+        # per-secondary columns are absent, the file was never run through the
+        # parent->child join (steps_to_parquet._add_secondary_attributes /
+        # `dwarf convert`). Zero-filling here would collapse every secondary to
+        # PDG index 0 and a constant energy fraction — a broken Stage 2 with no
+        # error. Callers that only need Stage-1 (e.g. `giant predict`) keep the
+        # default require_secondaries=False.
+        if require_secondaries and n_sec_raw.max(initial=0) > 0:
+            n_with_sec = int((n_sec_raw > 0).sum())
+            raise ValueError(
+                f"{n_with_sec} step(s) have secondaries (n_sec > 0) but the "
+                "per-secondary columns (sec_E_list / sec_pdg_list / sec_dx_list "
+                "…) are missing. This parquet was not run through the "
+                "parent->child join (steps_to_parquet._add_secondary_attributes "
+                "/ `dwarf convert`); training on it would silently zero all "
+                "Stage-2 targets. Re-convert the file, or pass "
+                "require_secondaries=False for Stage-1-only use."
+            )
         N = len(n_sec)
         sec_cont = np.zeros((N, K_MAX, 4), dtype=np.float32)
         sec_pdg_idx = np.zeros((N, K_MAX), dtype=np.int64)

giant/pipeline.py

@@ -64,7 +64,7 @@ def run_train_job(
                 continue
             chunk_tr = {k: v[mask] for k, v in chunk.items()}
             cond_cont, _, target_s1, _n_sec, _sec_cont, _sec_pdg, _, _ = build_features(
-                chunk_tr, pdg_map, mat_map
+                chunk_tr, pdg_map, mat_map, require_secondaries=True
             )
             cond_acc.update(cond_cont)
             tgt_acc.update(target_s1)

tests/test_transforms.py

@@ -239,3 +239,51 @@ def test_build_features_clamps_n_sec_label_to_k_max():
 
     assert n_sec.max() <= K_MAX
     np.testing.assert_array_equal(n_sec, [0, 5, K_MAX])
+
+
+def _step_data_no_sec_lists(n_sec: np.ndarray) -> dict:
+    """Minimal build_features input with n_sec but no per-secondary list columns
+    (mimics a parquet that skipped the parent->child join)."""
+    N = len(n_sec)
+    rng = np.random.default_rng(0)
+    return {
+        "pdg": np.full(N, 11, dtype=np.int32),
+        "material": np.full(N, "PbWO4", dtype=object),
+        "pre_pos": rng.standard_normal((N, 3)).astype(np.float32),
+        "pre_E": np.full(N, 10.0, dtype=np.float32),
+        "pre_dir": np.tile(np.array([0, 0, 1], dtype=np.float32), (N, 1)),
+        "layer_id": np.zeros(N, dtype=np.int32),
+        "n_sec": np.asarray(n_sec, dtype=np.int32),
+        "e_sec": np.full(N, 1.0, dtype=np.float32),
+        "step_length": np.full(N, 1.0, dtype=np.float32),
+        "post_E": np.full(N, 9.0, dtype=np.float32),
+        "edep": np.full(N, 1.0, dtype=np.float32),
+        "post_dir": np.tile(np.array([0, 0, 1], dtype=np.float32), (N, 1)),
+        "post_pos": rng.standard_normal((N, 3)).astype(np.float32),
+    }
+
+
+def test_build_features_require_secondaries_raises_when_lists_missing():
+    """A parquet with n_sec > 0 but no per-secondary list columns was never run
+    through the parent->child join; require_secondaries must catch it instead of
+    silently zeroing every Stage-2 target (regression: this collapsed the
+    secondary species to a single PDG index during training)."""
+    data = _step_data_no_sec_lists(np.array([0, 2, 1]))
+    pdg_map, mat_map = {11: 0}, {"PbWO4": 0}
+
+    with pytest.raises(ValueError, match="per-secondary columns"):
+        build_features(data, pdg_map, mat_map, require_secondaries=True)
+
+
+def test_build_features_require_secondaries_ok_when_no_secondaries():
+    """require_secondaries only fires when secondaries actually exist; a file
+    with n_sec == 0 everywhere (e.g. Stage-1-only) must still load."""
+    data = _step_data_no_sec_lists(np.zeros(3, dtype=np.int32))
+    pdg_map, mat_map = {11: 0}, {"PbWO4": 0}
+
+    _, _, _, _, sec_cont, sec_pdg_idx, _, _ = build_features(
+        data, pdg_map, mat_map, require_secondaries=True
+    )
+
+    assert not sec_cont.any()
+    assert not sec_pdg_idx.any()