From caaf3c61029954ec689757489fda4b32554a5263 Mon Sep 17 00:00:00 2001 From: Lars Bogner Date: Fri, 26 Jun 2026 11:24:05 +0200 Subject: [PATCH] Restrict holdout overlap check to holdout vs dev/full only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev and full are allowed to share files — only holdout must be strictly isolated. When creating dev or full, only compare against holdout.manifest; when creating holdout, compare against all other manifests in the dir. Co-Authored-By: Claude Sonnet 4.6 --- scripts/bump_dataset_version.py | 8 +++++++- tests/test_bump_dataset_version.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/bump_dataset_version.py b/scripts/bump_dataset_version.py index befc90a..5bc5b2a 100644 --- a/scripts/bump_dataset_version.py +++ b/scripts/bump_dataset_version.py @@ -263,7 +263,13 @@ def check_holdout_overlap( new_set = set(resolved_new_files) overlaps: list[tuple[str, Path]] = [] - for existing in sorted(manifest_dir.glob("*.manifest")): + # When creating holdout, check against all other manifests (dev, full, …). + # When creating dev/full, only check against holdout — dev vs full overlap is allowed. + if output_resolved.name == "holdout.manifest": + candidates = sorted(manifest_dir.glob("*.manifest")) + else: + candidates = [holdout_path] + for existing in candidates: if existing.resolve() == output_resolved: continue try: diff --git a/tests/test_bump_dataset_version.py b/tests/test_bump_dataset_version.py index 948e5f8..6f3bc30 100644 --- a/tests/test_bump_dataset_version.py +++ b/tests/test_bump_dataset_version.py @@ -293,6 +293,21 @@ def test_overlap_detected_when_holdout_already_exists(tmp_path): assert overlaps[0][0] == "holdout.manifest" +def test_dev_full_overlap_not_checked(tmp_path): + pool_dir = tmp_path / "pools" / "pbwo4" + pool_dir.mkdir(parents=True) + pq = tmp_path / "a.parquet" + pq.touch() + + # dev.manifest lists the same file — should not trigger a warning for full + dev = pool_dir / "dev.manifest" + dev.write_text(os.path.relpath(pq.resolve(), start=pool_dir) + "\n") + + output = pool_dir / "full.manifest" + overlaps = check_holdout_overlap(output, [pq.resolve()]) + assert overlaps == [] + + def test_no_overlap_when_files_are_disjoint(tmp_path): pool_dir = tmp_path / "pools" / "pbwo4" pool_dir.mkdir(parents=True)