print commit hash on import; fix event_id offset in multi-core steps
G4Calo.py prints the commit at import time, resolved via a minicalo_version.py file (baked in by Docker) or git rev-parse from the file's directory as fallback. event_id in the Steps ntuple restarts from 0 in each subprocess; _assemble_results_to_mini_df now accepts per-file event_id_offsets so IDs are globally unique after merging. run_batch passes cumulative nevents as offsets when loading the Steps tree. Dockerfiles write minicalo_version.py before copying Python files. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+33
-4
@@ -171,6 +171,27 @@ from IPython.display import Image, display
|
|||||||
from minipandas import MiniFrame
|
from minipandas import MiniFrame
|
||||||
|
|
||||||
|
|
||||||
|
def _get_commit():
|
||||||
|
try:
|
||||||
|
from minicalo_version import commit as _c
|
||||||
|
return _c
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
['git', 'rev-parse', '--short', 'HEAD'],
|
||||||
|
capture_output=True, text=True,
|
||||||
|
cwd=os.path.dirname(os.path.abspath(__file__))
|
||||||
|
)
|
||||||
|
if r.returncode == 0:
|
||||||
|
return r.stdout.strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return 'unknown'
|
||||||
|
|
||||||
|
print(f"miniCaloSim: commit {_get_commit()}")
|
||||||
|
|
||||||
|
|
||||||
#### helpers #####
|
#### helpers #####
|
||||||
|
|
||||||
class Stopwatch(object):
|
class Stopwatch(object):
|
||||||
@@ -238,11 +259,14 @@ def _count_total_entries(root_paths, tree_name: str = "Hits") -> int:
|
|||||||
total += int(f[tree_name].num_entries)
|
total += int(f[tree_name].num_entries)
|
||||||
return total
|
return total
|
||||||
|
|
||||||
def _assemble_results_to_mini_df(root_paths, tree_name: str = "Hits"):
|
def _assemble_results_to_mini_df(root_paths, tree_name: str = "Hits", event_id_offsets=None):
|
||||||
"""
|
"""
|
||||||
Returns:
|
Returns:
|
||||||
scalars: dict[col, np.ndarray] # shape (N,)
|
scalars: dict[col, np.ndarray] # shape (N,)
|
||||||
arrays: dict[col, np.ndarray] # shape (N, M[col])
|
arrays: dict[col, np.ndarray] # shape (N, M[col])
|
||||||
|
|
||||||
|
event_id_offsets: optional list[int], one per file — added to the
|
||||||
|
'event_id' column so that IDs are globally unique across batches.
|
||||||
"""
|
"""
|
||||||
# 1) learn schema & sizes from first file
|
# 1) learn schema & sizes from first file
|
||||||
scalar_cols, array_cols, array_sizes, dtypes_scalar, dtypes_array, _ = \
|
scalar_cols, array_cols, array_sizes, dtypes_scalar, dtypes_array, _ = \
|
||||||
@@ -257,14 +281,17 @@ def _assemble_results_to_mini_df(root_paths, tree_name: str = "Hits"):
|
|||||||
|
|
||||||
# 4) fill by slices
|
# 4) fill by slices
|
||||||
pos = 0
|
pos = 0
|
||||||
for p in root_paths:
|
for file_idx, p in enumerate(root_paths):
|
||||||
with uproot.open(p) as f:
|
with uproot.open(p) as f:
|
||||||
arr = f[tree_name].arrays(library="ak")
|
arr = f[tree_name].arrays(library="ak")
|
||||||
n = len(arr)
|
n = len(arr)
|
||||||
sl = slice(pos, pos + n)
|
sl = slice(pos, pos + n)
|
||||||
|
|
||||||
for c in scalar_cols:
|
for c in scalar_cols:
|
||||||
scalars[c][sl] = np.asarray(arr[c])
|
data = np.asarray(arr[c])
|
||||||
|
if c == 'event_id' and event_id_offsets is not None:
|
||||||
|
data = data + event_id_offsets[file_idx]
|
||||||
|
scalars[c][sl] = data
|
||||||
for c in array_cols:
|
for c in array_cols:
|
||||||
arrays[c][sl, :] = ak.to_numpy(arr[c]) # (n, fixed_size)
|
arrays[c][sl, :] = ak.to_numpy(arr[c]) # (n, fixed_size)
|
||||||
|
|
||||||
@@ -412,6 +439,7 @@ Example
|
|||||||
#print(f"Running {nEventsLastCore} events on last core")
|
#print(f"Running {nEventsLastCore} events on last core")
|
||||||
|
|
||||||
nevents = [nEventsPerCore if i < nCores - 1 else nEventsLastCore for i in range(nCores)]
|
nevents = [nEventsPerCore if i < nCores - 1 else nEventsLastCore for i in range(nCores)]
|
||||||
|
event_id_offsets = [sum(nevents[:i]) for i in range(nCores)]
|
||||||
|
|
||||||
if manual_seed >= 0:
|
if manual_seed >= 0:
|
||||||
seed = manual_seed
|
seed = manual_seed
|
||||||
@@ -466,7 +494,8 @@ Example
|
|||||||
try:
|
try:
|
||||||
df = _assemble_results_to_mini_df(rp)
|
df = _assemble_results_to_mini_df(rp)
|
||||||
if return_steps:
|
if return_steps:
|
||||||
steps_df = _assemble_results_to_mini_df(rp, tree_name="Steps")
|
steps_df = _assemble_results_to_mini_df(rp, tree_name="Steps",
|
||||||
|
event_id_offsets=event_id_offsets)
|
||||||
finally:
|
finally:
|
||||||
for f in tmpfile:
|
for f in tmpfile:
|
||||||
if os.path.exists(f):
|
if os.path.exists(f):
|
||||||
|
|||||||
+3
-1
@@ -101,7 +101,9 @@ RUN cd /root/minicalosim && git checkout $COMMIT && \
|
|||||||
mkdir -p build && cd build && rm -rf * && cmake ../ && make -j4
|
mkdir -p build && cd build && rm -rf * && cmake ../ && make -j4
|
||||||
|
|
||||||
|
|
||||||
RUN cp /root/minicalosim/build/minicalo* /root/minicalosim/bind/G4Calo.py /root/minicalosim/bind/minicalo_tools.py /root/minicalosim/bind/minipandas.py /usr/local/lib/python3.10/dist-packages/
|
RUN echo "commit = '${COMMIT}'" > /root/minicalosim/bind/minicalo_version.py
|
||||||
|
|
||||||
|
RUN cp /root/minicalosim/build/minicalo* /root/minicalosim/bind/G4Calo.py /root/minicalosim/bind/minicalo_tools.py /root/minicalosim/bind/minipandas.py /root/minicalosim/bind/minicalo_version.py /usr/local/lib/python3.10/dist-packages/
|
||||||
|
|
||||||
RUN cp /root/minicalosim/bind/G4Calo_exec.py /usr/local/bin/
|
RUN cp /root/minicalosim/bind/G4Calo_exec.py /usr/local/bin/
|
||||||
|
|
||||||
|
|||||||
@@ -99,7 +99,8 @@ ADD minicalosim /root/minicalosim
|
|||||||
RUN cd /root/minicalosim && git checkout $COMMIT && \
|
RUN cd /root/minicalosim && git checkout $COMMIT && \
|
||||||
git submodule update --init lib/pybind11 && \
|
git submodule update --init lib/pybind11 && \
|
||||||
mkdir -p build && cd build && rm -rf * && cmake ../ && make -j$(nproc) &&\
|
mkdir -p build && cd build && rm -rf * && cmake ../ && make -j$(nproc) &&\
|
||||||
cp minicalo* ../bind/G4Calo.py /usr/local/lib/python3.8/dist-packages/
|
echo "commit = '${COMMIT}'" > ../bind/minicalo_version.py && \
|
||||||
|
cp minicalo* ../bind/G4Calo.py ../bind/minicalo_version.py /usr/local/lib/python3.8/dist-packages/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -52,9 +52,11 @@ RUN cd /root/minicalosim && git checkout $COMMIT && \
|
|||||||
cmake --build build --parallel $(nproc) && \
|
cmake --build build --parallel $(nproc) && \
|
||||||
cp build/minicalo*.so /usr/local/lib/python3.10/dist-packages/
|
cp build/minicalo*.so /usr/local/lib/python3.10/dist-packages/
|
||||||
|
|
||||||
RUN cp /root/minicalosim/bind/G4Calo.py \
|
RUN echo "commit = '${COMMIT}'" > /root/minicalosim/bind/minicalo_version.py && \
|
||||||
|
cp /root/minicalosim/bind/G4Calo.py \
|
||||||
/root/minicalosim/bind/minicalo_tools.py \
|
/root/minicalosim/bind/minicalo_tools.py \
|
||||||
/root/minicalosim/bind/minipandas.py \
|
/root/minicalosim/bind/minipandas.py \
|
||||||
|
/root/minicalosim/bind/minicalo_version.py \
|
||||||
/usr/local/lib/python3.10/dist-packages/ && \
|
/usr/local/lib/python3.10/dist-packages/ && \
|
||||||
install -m 755 /root/minicalosim/bind/G4Calo_exec.py /usr/local/bin/G4Calo_exec.py
|
install -m 755 /root/minicalosim/bind/G4Calo_exec.py /usr/local/bin/G4Calo_exec.py
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user