Offset event_id per file to avoid cross-file collisions
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s
CI / Format (ruff format) (push) Successful in 25s
CI / Lint (ruff check) (push) Successful in 27s
CI / Sync project version with tag (push) Has been skipped
CI / Type check (ty) (push) Successful in 23s
CI / Lint (ruff check) (pull_request) Successful in 27s
CI / Tests (push) Successful in 1m14s
CI / Format (ruff format) (pull_request) Successful in 27s
CI / Sync project version with tag (pull_request) Has been skipped
CI / Type check (ty) (pull_request) Successful in 31s
CI / Tests (pull_request) Successful in 59s
Each input parquet file is one Geant4 job (scripts/steps_to_parquet.py), and a job's event_id numbering always restarts from 0 — so loading multiple files together (a directory or .manifest) let same-numbered events from different files collapse into one during the event index scan and train/val split, corrupting both. Every per-file event_id now gets offset by file index * EVENT_ID_FILE_STRIDE (giant/data/loader.py), threaded through the setup-cache event index, the streaming dataset, and predict/rollout seeding. Bumps the setup-cache format version so stale sidecars computed pre-fix are invalidated. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,10 +3,16 @@ import pandas as pd
|
||||
import pytest
|
||||
|
||||
from giant.data.loader import (
|
||||
EVENT_ID_FILE_STRIDE,
|
||||
build_index_maps,
|
||||
build_index_maps_from_files,
|
||||
build_process_map_from_files,
|
||||
event_id_offset,
|
||||
find_parquet_files,
|
||||
iter_cond_chunks,
|
||||
iter_file_chunks,
|
||||
load_event_ids,
|
||||
load_steps,
|
||||
)
|
||||
|
||||
|
||||
@@ -279,3 +285,84 @@ def test_build_index_maps_from_files_matches_build_index_maps(tmp_path):
|
||||
from_files = build_index_maps_from_files([path])
|
||||
from_memory = build_index_maps({"pdg": pdg, "material": material})
|
||||
assert from_files == from_memory
|
||||
|
||||
|
||||
# ── event_id_offset / per-file event_id offsetting ─────────────────────────
|
||||
|
||||
|
||||
def _steps_df(event_ids):
|
||||
"""Minimal schema-complete steps rows (no secondaries) for _df_to_dict."""
|
||||
return pd.DataFrame(
|
||||
[
|
||||
{
|
||||
"event_id": eid,
|
||||
"pdg": 11,
|
||||
"pre_x": 0.0,
|
||||
"pre_y": 0.0,
|
||||
"pre_z": 0.0,
|
||||
"pre_E": 100.0,
|
||||
"pre_dx": 0.0,
|
||||
"pre_dy": 0.0,
|
||||
"pre_dz": 1.0,
|
||||
"material": "G4_AIR",
|
||||
"layer_id": 0,
|
||||
"child_track_ids": [],
|
||||
"e_sec": 0.0,
|
||||
"step_length": 1.0,
|
||||
"post_E": 90.0,
|
||||
"edep": 10.0,
|
||||
"post_dx": 0.0,
|
||||
"post_dy": 0.0,
|
||||
"post_dz": 1.0,
|
||||
"post_x": 0.0,
|
||||
"post_y": 0.0,
|
||||
"post_z": 1.0,
|
||||
}
|
||||
for eid in event_ids
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_event_id_offset_scales_by_file_index():
|
||||
assert event_id_offset(0) == 0
|
||||
assert event_id_offset(1) == EVENT_ID_FILE_STRIDE
|
||||
assert event_id_offset(3) == 3 * EVENT_ID_FILE_STRIDE
|
||||
|
||||
|
||||
def test_load_event_ids_default_offset_is_zero(tmp_path):
|
||||
path = tmp_path / "a.parquet"
|
||||
pd.DataFrame({"event_id": [5, 6, 7]}).to_parquet(path)
|
||||
np.testing.assert_array_equal(load_event_ids(path), [5, 6, 7])
|
||||
|
||||
|
||||
def test_load_event_ids_applies_offset(tmp_path):
|
||||
path = tmp_path / "a.parquet"
|
||||
pd.DataFrame({"event_id": [0, 1, 2]}).to_parquet(path)
|
||||
offset = event_id_offset(1)
|
||||
np.testing.assert_array_equal(
|
||||
load_event_ids(path, offset=offset), [offset, offset + 1, offset + 2]
|
||||
)
|
||||
|
||||
|
||||
def test_load_steps_applies_offset_to_event_id(tmp_path):
|
||||
path = tmp_path / "a.parquet"
|
||||
_steps_df([0, 1]).to_parquet(path)
|
||||
offset = event_id_offset(2)
|
||||
d = load_steps(path, offset=offset)
|
||||
np.testing.assert_array_equal(d["event_id"], [offset, offset + 1])
|
||||
|
||||
|
||||
def test_iter_file_chunks_applies_offset(tmp_path):
|
||||
path = tmp_path / "a.parquet"
|
||||
_steps_df([0, 1, 2]).to_parquet(path)
|
||||
offset = event_id_offset(1)
|
||||
ids = np.concatenate([c["event_id"] for c in iter_file_chunks(path, offset=offset)])
|
||||
np.testing.assert_array_equal(sorted(ids), [offset, offset + 1, offset + 2])
|
||||
|
||||
|
||||
def test_iter_cond_chunks_applies_offset(tmp_path):
|
||||
path = tmp_path / "a.parquet"
|
||||
_steps_df([0, 1]).to_parquet(path)
|
||||
offset = event_id_offset(5)
|
||||
ids = np.concatenate([c["event_id"] for c in iter_cond_chunks(path, offset=offset)])
|
||||
np.testing.assert_array_equal(sorted(ids), [offset, offset + 1])
|
||||
|
||||
Reference in New Issue
Block a user