lars
a482b04761
One workflow TOML now parameterises a whole experiment and `giant workflow run <spec.toml>` turns it into a b2luigi DAG whose targets are files on /ceph: nothing already produced is recomputed, every step waits for its inputs, and HTCondor submission/polling is b2luigi's job. - spec.py: workflow TOML -> frozen dataclasses with name-uniqueness and cross-reference validation, unknown keys rejected the way giant.config rejects them, and a short spec_hash per task that folds in its transitive parents — so an edited spec re-runs exactly the affected subtree. - htcondor.py: the CPU/GPU submit settings. The GPU requirement strings (ProvidesEtpCeph + optional device/memory pins) are ported from the condor-gpu-train-rollout branch rather than rewritten. - tasks.py: DatasetTask, WarmCacheTask, GeometryOracleTask, TrainEpochTask (one short GPU job per epoch, chained via --resume, which the training loop already supports unchanged), TrainTask (publishes best.pt/last.pt and a concatenated metrics.csv so downstream never sees the epoch fan-out), RolloutTask, AnalysisPrepTask, AnalysisComputeTask (one job per plot x chunk, walltime sized from run_meta.json at submit time), AnalysisRenderTask (always local — the only step importing plotstyle/LaTeX), WorkflowTask. Task bodies call the existing entry points; none of them reimplement anything. - run.py + `giant workflow run`: settings wiring and the script b2luigi re-executes on workers. add_filename_to_cmd is off because b2luigi passes only the script's basename, and --spec is forwarded via task_cmd_additional_args so a worker resolves the identical task graph. configs/workflow_example.toml is the documented starting point. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
120 lines
2.5 KiB
TOML
120 lines
2.5 KiB
TOML
[project]
|
|
name = "giant"
|
|
version = "0.3.10"
|
|
description = "Geant4 step-function surrogate via conditional flow matching"
|
|
readme = "README.md"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"numpy>=1.26,<3",
|
|
"pandas>=2.2,<4",
|
|
"pyarrow>=16,<25",
|
|
"tqdm>=4.60,<5",
|
|
"typer>=0.12,<1",
|
|
"pyyaml>=6,<7",
|
|
"particle>=1.0,<2",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
cpu = [
|
|
"torch>=2.3,<2.4",
|
|
]
|
|
cuda = [
|
|
"torch>=2.3,<2.4",
|
|
]
|
|
dev = [
|
|
"pytest>=8,<10",
|
|
"pytest-cov>=5,<8",
|
|
"ruff>=0.15,<1",
|
|
"ty>=0.0.50,<0.1",
|
|
"bump-my-version>=1.2,<2",
|
|
"git-cliff>=2,<3",
|
|
"giant[convert,analysis,geometry,wandb,workflow]",
|
|
]
|
|
geometry = [
|
|
"scikit-learn>=1.4,<2",
|
|
]
|
|
wandb = [
|
|
"wandb>=0.16,<1",
|
|
]
|
|
convert = [
|
|
"uproot>=5.3,<6",
|
|
"awkward>=2.6,<3",
|
|
"polars>=1.0,<2",
|
|
]
|
|
analysis = [
|
|
"matplotlib>=3.8,<4",
|
|
"polars>=1.0,<2",
|
|
"ipykernel>=7.3.0",
|
|
# KIT matplotlib theme, published from git.larsbogner.de. Only the local
|
|
# `giant analyze render` step imports it; compute workers never do.
|
|
"plotstyle>=1.0.0",
|
|
]
|
|
# b2luigi pulls luigi + tenacity; the only sanctioned way to chain a
|
|
# multi-step pipeline (see giant/workflow/).
|
|
workflow = [
|
|
"b2luigi>=1.0,<2",
|
|
]
|
|
|
|
[project.scripts]
|
|
giant = "giant.cli:app"
|
|
dwarf = "giant.tools.dwarf:app"
|
|
|
|
[tool.ruff]
|
|
line-length = 120
|
|
|
|
[tool.coverage.run]
|
|
source = ["giant"]
|
|
omit = ["*/legacy/*"]
|
|
|
|
[tool.coverage.report]
|
|
exclude_also = [
|
|
"if TYPE_CHECKING:",
|
|
"raise NotImplementedError",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[tool.hatch.build.targets.wheel]
|
|
packages = ["giant"]
|
|
|
|
[tool.uv]
|
|
conflicts = [
|
|
[
|
|
{ extra = "cpu" },
|
|
{ extra = "cuda" },
|
|
],
|
|
]
|
|
|
|
[tool.uv.sources]
|
|
torch = [
|
|
{ index = "pytorch-cpu", extra = "cpu" },
|
|
{ index = "pytorch-cu118", extra = "cuda" },
|
|
]
|
|
plotstyle = { index = "larsbogner" }
|
|
|
|
[[tool.uv.index]]
|
|
name = "pytorch-cpu"
|
|
url = "https://download.pytorch.org/whl/cpu"
|
|
explicit = true
|
|
|
|
[[tool.uv.index]]
|
|
name = "pytorch-cu118"
|
|
url = "https://download.pytorch.org/whl/cu118"
|
|
explicit = true
|
|
|
|
[[tool.uv.index]]
|
|
name = "larsbogner"
|
|
url = "https://git.larsbogner.de/api/packages/lars/pypi/simple/"
|
|
explicit = true
|
|
|
|
# luigi builds task constructors from class-level Parameter descriptors, so a
|
|
# static checker sees no keyword parameters at all on `Task(name=..., ...)`.
|
|
# The workflow code is written against that API; nothing else in the repo is.
|
|
[[tool.ty.overrides]]
|
|
include = ["giant/workflow/**", "tests/test_workflow_tasks.py"]
|
|
|
|
[tool.ty.overrides.rules]
|
|
unknown-argument = "ignore"
|