Offset event_id per file to avoid cross-file collisions
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s

Each input parquet file is one Geant4 job (scripts/steps_to_parquet.py),
and a job's event_id numbering always restarts from 0 — so loading
multiple files together (a directory or .manifest) let same-numbered
events from different files collapse into one during the event index
scan and train/val split, corrupting both. Every per-file event_id now
gets offset by file index * EVENT_ID_FILE_STRIDE (giant/data/loader.py),
threaded through the setup-cache event index, the streaming dataset,
and predict/rollout seeding. Bumps the setup-cache format version so
stale sidecars computed pre-fix are invalidated.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-30 11:36:48 +02:00
parent e288c3fe21
commit b944bba8fb
8 changed files with 268 additions and 24 deletions
+87
View File
@@ -3,10 +3,16 @@ import pandas as pd
import pytest
from giant.data.loader import (
EVENT_ID_FILE_STRIDE,
build_index_maps,
build_index_maps_from_files,
build_process_map_from_files,
event_id_offset,
find_parquet_files,
iter_cond_chunks,
iter_file_chunks,
load_event_ids,
load_steps,
)
@@ -279,3 +285,84 @@ def test_build_index_maps_from_files_matches_build_index_maps(tmp_path):
from_files = build_index_maps_from_files([path])
from_memory = build_index_maps({"pdg": pdg, "material": material})
assert from_files == from_memory
# ── event_id_offset / per-file event_id offsetting ─────────────────────────
def _steps_df(event_ids):
"""Minimal schema-complete steps rows (no secondaries) for _df_to_dict."""
return pd.DataFrame(
[
{
"event_id": eid,
"pdg": 11,
"pre_x": 0.0,
"pre_y": 0.0,
"pre_z": 0.0,
"pre_E": 100.0,
"pre_dx": 0.0,
"pre_dy": 0.0,
"pre_dz": 1.0,
"material": "G4_AIR",
"layer_id": 0,
"child_track_ids": [],
"e_sec": 0.0,
"step_length": 1.0,
"post_E": 90.0,
"edep": 10.0,
"post_dx": 0.0,
"post_dy": 0.0,
"post_dz": 1.0,
"post_x": 0.0,
"post_y": 0.0,
"post_z": 1.0,
}
for eid in event_ids
]
)
def test_event_id_offset_scales_by_file_index():
assert event_id_offset(0) == 0
assert event_id_offset(1) == EVENT_ID_FILE_STRIDE
assert event_id_offset(3) == 3 * EVENT_ID_FILE_STRIDE
def test_load_event_ids_default_offset_is_zero(tmp_path):
path = tmp_path / "a.parquet"
pd.DataFrame({"event_id": [5, 6, 7]}).to_parquet(path)
np.testing.assert_array_equal(load_event_ids(path), [5, 6, 7])
def test_load_event_ids_applies_offset(tmp_path):
path = tmp_path / "a.parquet"
pd.DataFrame({"event_id": [0, 1, 2]}).to_parquet(path)
offset = event_id_offset(1)
np.testing.assert_array_equal(
load_event_ids(path, offset=offset), [offset, offset + 1, offset + 2]
)
def test_load_steps_applies_offset_to_event_id(tmp_path):
path = tmp_path / "a.parquet"
_steps_df([0, 1]).to_parquet(path)
offset = event_id_offset(2)
d = load_steps(path, offset=offset)
np.testing.assert_array_equal(d["event_id"], [offset, offset + 1])
def test_iter_file_chunks_applies_offset(tmp_path):
path = tmp_path / "a.parquet"
_steps_df([0, 1, 2]).to_parquet(path)
offset = event_id_offset(1)
ids = np.concatenate([c["event_id"] for c in iter_file_chunks(path, offset=offset)])
np.testing.assert_array_equal(sorted(ids), [offset, offset + 1, offset + 2])
def test_iter_cond_chunks_applies_offset(tmp_path):
path = tmp_path / "a.parquet"
_steps_df([0, 1]).to_parquet(path)
offset = event_id_offset(5)
ids = np.concatenate([c["event_id"] for c in iter_cond_chunks(path, offset=offset)])
np.testing.assert_array_equal(sorted(ids), [offset, offset + 1])