Fix CLI/tooling robustness gaps and dedupe the Conditioning enum
CI / Lint (ruff check) (push) Successful in 35s
CI / Format (ruff format) (push) Failing after 31s
CI / Type check (ty) (push) Successful in 30s
CI / Sync project version with tag (push) Has been skipped
CI / Tests (push) Successful in 1m7s

- run_create_manifest gains --force; it previously overwrote an
  existing manifest (including holdout.manifest, which
  check_holdout_overlap exists specifically to protect) with no
  warning or backup on a second run.
- _git_user_name only caught OSError, not subprocess.TimeoutExpired (a
  SubprocessError, not an OSError) — a slow/loaded shared portal
  machine could crash `dwarf bump-gen`/`bump-schema` instead of
  degrading to by=None as intended.
- `dwarf convert --jobs`/`make-root --jobs` now warn (never block) when
  the requested count exceeds ~1/4 of the machine's CPUs, matching the
  same shared-machine etiquette check added to giant train in the
  previous commit.
- The Conditioning enum was independently redefined in both
  giant/cli.py and scripts/dwarf.py; moved to a single
  giant.config.Conditioning both now import, removing the drift risk
  of a third conditioning mode being added to one but not the other.

Each fix has a regression test.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-03 13:49:09 +02:00
parent ad1b8e7835
commit ca3a2a3462
6 changed files with 124 additions and 10 deletions
+40
View File
@@ -1,4 +1,6 @@
import os
import subprocess
from scripts import bump_dataset_version
plan_bump_gen = bump_dataset_version.plan_bump_gen
@@ -11,6 +13,14 @@ apply_create_manifest = bump_dataset_version.apply_create_manifest
check_holdout_overlap = bump_dataset_version.check_holdout_overlap
def test_git_user_name_returns_none_on_timeout(monkeypatch):
def _raise_timeout(*args, **kwargs):
raise subprocess.TimeoutExpired(cmd=["git"], timeout=2)
monkeypatch.setattr(subprocess, "run", _raise_timeout)
assert bump_dataset_version._git_user_name() is None
def test_bump_gen_starts_at_gen1_when_none_exist(tmp_path):
dirs, log_line = plan_bump_gen(
tmp_path, "steps", "first generation", None, "2026-01-01"
@@ -401,6 +411,36 @@ def test_create_manifest_creates_parent_dirs(tmp_path):
assert output.exists()
def test_run_create_manifest_refuses_to_overwrite_existing_output(tmp_path):
pq = tmp_path / "a.parquet"
pq.touch()
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
output.parent.mkdir(parents=True)
output.write_text("original contents\n")
try:
bump_dataset_version.run_create_manifest(
[str(pq)], execute=True, output=str(output)
)
assert False, "expected SystemExit"
except SystemExit:
pass
assert output.read_text() == "original contents\n"
def test_run_create_manifest_force_overwrites_existing_output(tmp_path):
pq = tmp_path / "a.parquet"
pq.touch()
output = tmp_path / "pools" / "pbwo4" / "holdout.manifest"
output.parent.mkdir(parents=True)
output.write_text("original contents\n")
bump_dataset_version.run_create_manifest(
[str(pq)], execute=True, output=str(output), force=True
)
assert output.read_text() != "original contents\n"
# ---------------------------------------------------------------------------
# check_holdout_overlap
# ---------------------------------------------------------------------------
+23
View File
@@ -1,12 +1,35 @@
from typer.testing import CliRunner
from giant import cli as giant_cli
from giant.config import Conditioning
from giant.data import setup_cache
from scripts import dwarf
from scripts.dwarf import app
from test_pipeline import _make_synthetic_steps
runner = CliRunner()
def test_conditioning_enum_shared_across_both_clis():
"""giant.cli and scripts.dwarf must use the one giant.config.Conditioning
enum, not independently redefined copies that could silently drift apart
on valid --conditioning values."""
assert dwarf.Conditioning is Conditioning
assert giant_cli.Conditioning is Conditioning
def test_warn_if_exceeds_shared_quota_warns_over_quarter_cpu(monkeypatch, capsys):
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
dwarf._warn_if_exceeds_shared_quota(3, "--jobs")
assert "warning: --jobs=3 exceeds" in capsys.readouterr().err
def test_warn_if_exceeds_shared_quota_silent_within_quota(monkeypatch, capsys):
monkeypatch.setattr(dwarf.os, "cpu_count", lambda: 8) # quota = 2
dwarf._warn_if_exceeds_shared_quota(2, "--jobs")
assert capsys.readouterr().err == ""
def test_convert_rejects_jobs_below_one(tmp_path):
root_file = tmp_path / "shard.root"
root_file.touch()