From 4d6101dcd70389ee67a55014ea3f80298e05b628 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Mon, 27 Jul 2026 11:06:40 +0200 Subject: [PATCH] analyze: recalibrate condor walltime model from real cluster timings The prior _COST_MODEL/_FIXED_OVERHEAD_S were fit only against local synthetic benchmarks (up to 2M rows/side), which can't see docker pull or real /ceph read latency and wildly overestimated real jobs (~1200-1800s predicted vs 50-320s median actual, from condor_history on production run 563f5ee3, --chunks 4, ~254M total rows). Refit each spec's per-row rate through the origin against its median real wall-clock time (not max, to avoid baking a few /ceph-contention spikes into a rate that would then wrongly scale with dataset size), and raised RUNTIME_SAFETY_MARGIN to compensate for that same contention risk instead. --- giant/analysis/runtime_estimate.py | 143 ++++++++++++++++------------- 1 file changed, 78 insertions(+), 65 deletions(-) diff --git a/giant/analysis/runtime_estimate.py b/giant/analysis/runtime_estimate.py index 29a7f95..7f6cc37 100644 --- a/giant/analysis/runtime_estimate.py +++ b/giant/analysis/runtime_estimate.py @@ -3,23 +3,34 @@ Each catalog spec's compute cost is close to linear in the number of input rows a `compute-one` job streams over — every spec is one (or a couple of) streaming `group_by` pass(es) over the chunk (see `catalog.py`/`reduce.py`). -`_COST_MODEL` below is ``spec_id -> (intercept_s, seconds_per_row)``, fit by -least squares against wall-clock timings of `compute_reduced` on synthetic -mock data of increasing size, run on a local dev machine (see -`scripts/profile_analysis_costs.py` — rerun it and paste the new numbers in -here if the catalog changes or this needs recalibrating). ``n_rows`` is the -combined rollout+reference row count of the job's input: the chunk's row -count for `chunkable=True` specs, the whole dataset's for the three -`chunkable=False` router specs (they always run as a single job regardless of -chunk count). +`_COST_MODEL` below is ``spec_id -> (intercept_s, seconds_per_row)``. +``n_rows`` is the combined rollout+reference row count of the job's input: +the chunk's row count for `chunkable=True` specs, the whole dataset's for the +three `chunkable=False` router specs (they always run as a single job +regardless of chunk count). -The fitted numbers only capture *local, in-memory compute* — they don't (and -from a laptop with no `/ceph` access, can't) capture the real condor job's -docker pull, `uv run` cold start, or shared-filesystem read latency, which in -practice likely dominate total wall time for anything but a huge single-chunk -job. `_FIXED_OVERHEAD_S` is a deliberately generous placeholder for all of -that combined; recalibrate it from real `condor_q`/log timings once some are -available, rather than trusting it as measured. +Calibrated 2026-07-27 from real HTCondor timings (`condor_history` +``RemoteWallClockTime``) of a production run: prediction ``563f5ee3`` +(PbWO4, 50 GeV) analyzed with ``--chunks 4`` against +``giant/analysis/runtime_estimate.py``'s prior (local-synthetic-only) model — +see the ``analysis-rollout-plots`` branch history for the raw data. That run's +4 chunks came out at nearly identical row counts (~63-64M rows each, ~254M +total), so this real data has no genuine row-count spread to fit a slope +against — instead each spec's ``per_row`` here is a single line through the +origin (``intercept=0``) hitting that spec's *median* wall-clock time across +its 4 chunks at that run's row count. A handful of (spec, chunk) pairs showed +3-8x spikes in one chunk only (e.g. ``marginal_edep_by_material``: 88, 88, 90, +722s) — almost certainly shared ``/ceph`` contention from ~130 jobs landing on +the filesystem at once right after submission, not a real per-row cost, so +the median (not the max) was fit to avoid baking that noise into a rate that +would then wrongly scale up with a bigger dataset. `RUNTIME_SAFETY_MARGIN` is +deliberately generous (4x total) specifically to absorb that kind of +contention spike instead. Rerun this calibration (pull fresh +`condor_history`/`run_meta.json`, refit) if the catalog changes or timings +drift — a synthetic local rebaseline via `scripts/profile_analysis_costs.py` +is a reasonable fallback when no real cluster data is available yet, but +undershoots real wall time badly (it can't see docker pull / `/ceph` I/O +latency), which is exactly why this file moved off it. """ from __future__ import annotations @@ -27,62 +38,64 @@ from __future__ import annotations import math # Multiplicative pad applied to every job's estimated walltime. The one knob -# this feature was asked to expose. -RUNTIME_SAFETY_MARGIN = 1.00 +# this feature was asked to expose. Set generously (4x total, i.e. 3.0 here) +# to absorb the shared-/ceph-contention spikes described above rather than +# encoding them into individual specs' per-row rates. +RUNTIME_SAFETY_MARGIN = 3.00 -# Docker image pull + `uv run` startup + shared (/ceph, ETP) filesystem read -# latency — not measurable on a machine with no /ceph access, so this is a -# conservative placeholder rather than a fit. Recalibrate from real job logs. -_FIXED_OVERHEAD_S = 600.0 +# Fixed per-job overhead (docker start, `.venv/bin/giant` startup, initial +# `/ceph` read latency) — calibrated as the fastest observed real spec +# (`leakage_fraction`, median 51s) rounded up, since even the cheapest spec +# streams the whole chunk once. +_FIXED_OVERHEAD_S = 60.0 -# Router diagnostics need a live torch checkpoint to do any real work; this -# machine has none, so their cost (torch.load + a bounded inference pass over -# <= 200k subsampled rows, independent of chunk size) couldn't be profiled -# either. Fixed budget, on top of _FIXED_OVERHEAD_S, instead of a row-based fit. -_ROUTER_FIXED_S = 300.0 +# Router diagnostics run a live torch checkpoint (bounded inference over +# <=200k subsampled rows, independent of chunk size) instead of a row-based +# scan. Calibrated from the 3 real router jobs' observed wall times (119, 66, +# 124s) — max minus _FIXED_OVERHEAD_S, on top of it. +_ROUTER_FIXED_S = 64.0 _ROUTER_IDS = frozenset({"router_gating", "router_share_by_pdg", "router_share_by_process"}) # Conservative fallback for any catalog id not in _COST_MODEL (e.g. a plot -# added after the last profiling run) — the most expensive fitted (intercept, -# seconds/row) pair observed, rounded up. -_DEFAULT_COST = (0.02, 2.0e-6) +# added after the last calibration run) — the most expensive fitted per-row +# rate observed, plus a small constant pad. +_DEFAULT_COST = (5.0, 3.0e-6) -# spec_id -> (intercept_s, seconds_per_row), fit on this machine 2026-07-27 -# via `scripts/profile_analysis_costs.py` against SIDE_ROW_COUNTS up to 2M -# rows/side (4M combined). +# spec_id -> (intercept_s, seconds_per_row), fit 2026-07-27 from real +# HTCondor `RemoteWallClockTime` (see module docstring for methodology). _COST_MODEL: dict[str, tuple[float, float]] = { - "marginal_step_length": (0.003581, 0.000000042), - "marginal_step_length_by_energy": (0.010336, 0.000000081), - "marginal_step_length_by_pdg": (0.007231, 0.000000042), - "marginal_step_length_by_material": (0.004662, 0.000000048), - "marginal_edep": (0.003996, 0.000000042), - "marginal_edep_by_energy": (0.007977, 0.000000085), - "marginal_edep_by_pdg": (0.003535, 0.000000042), - "marginal_edep_by_material": (0.003893, 0.000000047), - "marginal_delta_e": (0.001380, 0.000000053), - "marginal_delta_e_by_energy": (0.006543, 0.000000093), - "marginal_delta_e_by_pdg": (0.001761, 0.000000054), - "marginal_delta_e_by_material": (0.000000, 0.000000168), - "marginal_post_E": (0.000000, 0.000000146), - "marginal_post_E_by_energy": (0.000000, 0.000000509), - "marginal_post_E_by_pdg": (0.000000, 0.000000149), - "marginal_post_E_by_material": (0.000000, 0.000000168), - "marginal_cos_scatter": (0.000000, 0.000000249), - "marginal_cos_scatter_by_energy": (0.000000, 0.000000553), - "marginal_cos_scatter_by_pdg": (0.000000, 0.000000270), - "marginal_cos_scatter_by_material": (0.000000, 0.000000260), - "event_total_edep": (0.000000, 0.000000512), - "event_total_edep_by_energy": (0.000000, 0.000000241), - "event_mean_length": (0.000000, 0.000000090), - "event_n_steps": (0.002021, 0.000000066), - "shower_longitudinal": (0.000000, 0.000001804), - "shower_transverse": (0.000000, 0.000001731), - "species_edep_share": (0.007505, 0.000000017), - "leakage_fraction": (0.003798, 0.000000026), - "sec_count_per_event": (0.006551, 0.000000054), - "sec_count_per_species": (0.006708, 0.000000042), - "sec_energy": (0.006856, 0.000000047), - "sec_cos_angle": (0.001780, 0.000000234), + "marginal_step_length": (0.0, 5.199e-07), + "marginal_step_length_by_energy": (0.0, 1.678e-06), + "marginal_step_length_by_pdg": (0.0, 4.569e-07), + "marginal_step_length_by_material": (0.0, 2.269e-06), + "marginal_edep": (0.0, 2.804e-06), + "marginal_edep_by_energy": (0.0, 1.386e-06), + "marginal_edep_by_pdg": (0.0, 4.490e-07), + "marginal_edep_by_material": (0.0, 4.333e-07), + "marginal_delta_e": (0.0, 5.042e-07), + "marginal_delta_e_by_energy": (0.0, 1.678e-06), + "marginal_delta_e_by_pdg": (0.0, 4.727e-07), + "marginal_delta_e_by_material": (0.0, 4.490e-07), + "marginal_post_E": (0.0, 4.805e-07), + "marginal_post_E_by_energy": (0.0, 1.284e-06), + "marginal_post_E_by_pdg": (0.0, 4.569e-07), + "marginal_post_E_by_material": (0.0, 4.490e-07), + "marginal_cos_scatter": (0.0, 5.436e-07), + "marginal_cos_scatter_by_energy": (0.0, 1.363e-06), + "marginal_cos_scatter_by_pdg": (0.0, 4.727e-07), + "marginal_cos_scatter_by_material": (0.0, 4.727e-07), + "event_total_edep": (0.0, 4.490e-07), + "event_total_edep_by_energy": (0.0, 4.411e-07), + "event_mean_length": (0.0, 4.333e-07), + "event_n_steps": (0.0, 4.569e-07), + "shower_longitudinal": (0.0, 2.348e-06), + "shower_transverse": (0.0, 2.899e-06), + "species_edep_share": (0.0, 4.333e-07), + "leakage_fraction": (0.0, 0.0), + "sec_count_per_event": (0.0, 4.727e-07), + "sec_count_per_species": (0.0, 4.963e-07), + "sec_energy": (0.0, 4.727e-07), + "sec_cos_angle": (0.0, 2.749e-06), }