Add --copy mode to migrate_geant_steps.py

Lets the migration run while another process still has the original files
open for reading: --copy uses shutil.copy2 instead of move, and skips the
now-empty-directory cleanup since the legacy train/ etc. dirs stay populated
by design.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-06-25 17:20:13 +02:00
parent 5be91e0d17
commit 6b85e2c90b
+23 -10
View File
@@ -13,7 +13,9 @@ up pool membership from POOL_ASSIGNMENT — decoupling "where it sits today" fro
"which pool it belongs to".
Defaults to a dry run (prints the planned moves and manifest contents). Pass
--execute to actually move files and write manifests.
--execute to actually move files and write manifests. Pass --copy as well to
copy instead of move, leaving the original files in place — e.g. if another
process is still reading them from their current location.
"""
import argparse
@@ -192,6 +194,13 @@ def main() -> None:
action="store_true",
help="Actually move files and write manifests (default: dry run / print plan only)",
)
parser.add_argument(
"--copy",
action="store_true",
help="Copy instead of move, leaving the originals in place "
"(e.g. if another process is still reading them). Implies the legacy "
"train/ etc. directories are left as-is too, since they won't be empty.",
)
args = parser.parse_args()
src_root = Path(args.root)
@@ -201,9 +210,10 @@ def main() -> None:
moves, unrecognized = plan_moves(src_root)
manifests = plan_manifests(src_root)
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'}: {src_root} ===\n")
verb = "COPY" if args.copy else "MOVE"
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
print(f"-- {len(moves)} file move(s) --")
print(f"-- {len(moves)} file {'copy' if args.copy else 'move'}(s) --")
for src, dst in moves:
print(f" {src.relative_to(src_root)} -> {dst.relative_to(src_root)}")
@@ -222,11 +232,12 @@ def main() -> None:
print("\nDry run only — pass --execute to apply.")
return
transfer = shutil.copy2 if args.copy else shutil.move
for src, dst in moves:
if dst.exists():
raise FileExistsError(f"refusing to overwrite existing file: {dst}")
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.move(str(src), str(dst))
transfer(str(src), str(dst))
for manifest_path, lines in manifests.items():
manifest_path.parent.mkdir(parents=True, exist_ok=True)
@@ -244,12 +255,14 @@ def main() -> None:
# Legacy pool dirs (train/, sampling_train/, sampling_train/small/) are now
# empty since their contents were classified by filename, not location —
# remove them, but only if a move actually emptied them.
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
d = src_root / stale_dir
if d.is_dir() and not any(d.iterdir()):
d.rmdir()
print(f"removed now-empty directory: {d.relative_to(src_root)}")
# remove them, but only if a move actually emptied them. In --copy mode the
# originals are still there by design, so leave these alone entirely.
if not args.copy:
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
d = src_root / stale_dir
if d.is_dir() and not any(d.iterdir()):
d.rmdir()
print(f"removed now-empty directory: {d.relative_to(src_root)}")
print("\nDone.")