Fix CLI/tooling robustness gaps and dedupe the Conditioning enum
CI / Lint (ruff check) (push) Successful in 35s
CI / Format (ruff format) (push) Failing after 31s
CI / Type check (ty) (push) Successful in 30s
CI / Sync project version with tag (push) Has been skipped
CI / Tests (push) Successful in 1m7s

- run_create_manifest gains --force; it previously overwrote an
  existing manifest (including holdout.manifest, which
  check_holdout_overlap exists specifically to protect) with no
  warning or backup on a second run.
- _git_user_name only caught OSError, not subprocess.TimeoutExpired (a
  SubprocessError, not an OSError) — a slow/loaded shared portal
  machine could crash `dwarf bump-gen`/`bump-schema` instead of
  degrading to by=None as intended.
- `dwarf convert --jobs`/`make-root --jobs` now warn (never block) when
  the requested count exceeds ~1/4 of the machine's CPUs, matching the
  same shared-machine etiquette check added to giant train in the
  previous commit.
- The Conditioning enum was independently redefined in both
  giant/cli.py and scripts/dwarf.py; moved to a single
  giant.config.Conditioning both now import, removing the drift risk
  of a third conditioning mode being added to one but not the other.

Each fix has a regression test.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-03 13:49:09 +02:00
co-authored by Claude Sonnet 5
parent ad1b8e7835
commit ca3a2a3462
6 changed files with 124 additions and 10 deletions
+40
View File
@@ -1,4 +1,6 @@
import os
import subprocess
from scripts import bump_dataset_version
plan_bump_gen = bump_dataset_version.plan_bump_gen
@@ -11,6 +13,14 @@ apply_create_manifest = bump_dataset_version.apply_create_manifest
check_holdout_overlap = bump_dataset_version.check_holdout_overlap
def test_git_user_name_returns_none_on_timeout(monkeypatch):
def _raise_timeout(*args, **kwargs):
raise subprocess.TimeoutExpired(cmd=["git"], timeout=2)
monkeypatch.setattr(subprocess, "run", _raise_timeout)
assert bump_dataset_version._git_user_name() is None
def test_bump_gen_starts_at_gen1_when_none_exist(tmp_path):
dirs, log_line = plan_bump_gen(
tmp_path, "steps", "first generation", None, "2026-01-01"
@@ -401,6 +411,36 @@ def test_create_manifest_creates_parent_dirs(tmp_path):
assert output.exists()
def test_run_create_manifest_refuses_to_overwrite_existing_output(tmp_path):
pq = tmp_path / "a.parquet"
pq.touch()
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
output.parent.mkdir(parents=True)
output.write_text("original contents\n")
try:
bump_dataset_version.run_create_manifest(
[str(pq)], execute=True, output=str(output)
)
assert False, "expected SystemExit"
except SystemExit:
pass
assert output.read_text() == "original contents\n"
def test_run_create_manifest_force_overwrites_existing_output(tmp_path):
pq = tmp_path / "a.parquet"
pq.touch()
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
output.parent.mkdir(parents=True)
output.write_text("original contents\n")
bump_dataset_version.run_create_manifest(
[str(pq)], execute=True, output=str(output), force=True
)
assert output.read_text() != "original contents\n"
# ---------------------------------------------------------------------------
# check_holdout_overlap
# ---------------------------------------------------------------------------