Add --copy mode to migrate_geant_steps.py
Lets the migration run while another process still has the original files open for reading: --copy uses shutil.copy2 instead of move, and skips the now-empty-directory cleanup since the legacy train/ etc. dirs stay populated by design. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -13,7 +13,9 @@ up pool membership from POOL_ASSIGNMENT — decoupling "where it sits today" fro
|
||||
"which pool it belongs to".
|
||||
|
||||
Defaults to a dry run (prints the planned moves and manifest contents). Pass
|
||||
--execute to actually move files and write manifests.
|
||||
--execute to actually move files and write manifests. Pass --copy as well to
|
||||
copy instead of move, leaving the original files in place — e.g. if another
|
||||
process is still reading them from their current location.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -192,6 +194,13 @@ def main() -> None:
|
||||
action="store_true",
|
||||
help="Actually move files and write manifests (default: dry run / print plan only)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--copy",
|
||||
action="store_true",
|
||||
help="Copy instead of move, leaving the originals in place "
|
||||
"(e.g. if another process is still reading them). Implies the legacy "
|
||||
"train/ etc. directories are left as-is too, since they won't be empty.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
src_root = Path(args.root)
|
||||
@@ -201,9 +210,10 @@ def main() -> None:
|
||||
moves, unrecognized = plan_moves(src_root)
|
||||
manifests = plan_manifests(src_root)
|
||||
|
||||
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'}: {src_root} ===\n")
|
||||
verb = "COPY" if args.copy else "MOVE"
|
||||
print(f"=== {'EXECUTING' if args.execute else 'DRY RUN'} ({verb}): {src_root} ===\n")
|
||||
|
||||
print(f"-- {len(moves)} file move(s) --")
|
||||
print(f"-- {len(moves)} file {'copy' if args.copy else 'move'}(s) --")
|
||||
for src, dst in moves:
|
||||
print(f" {src.relative_to(src_root)} -> {dst.relative_to(src_root)}")
|
||||
|
||||
@@ -222,11 +232,12 @@ def main() -> None:
|
||||
print("\nDry run only — pass --execute to apply.")
|
||||
return
|
||||
|
||||
transfer = shutil.copy2 if args.copy else shutil.move
|
||||
for src, dst in moves:
|
||||
if dst.exists():
|
||||
raise FileExistsError(f"refusing to overwrite existing file: {dst}")
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), str(dst))
|
||||
transfer(str(src), str(dst))
|
||||
|
||||
for manifest_path, lines in manifests.items():
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -244,12 +255,14 @@ def main() -> None:
|
||||
|
||||
# Legacy pool dirs (train/, sampling_train/, sampling_train/small/) are now
|
||||
# empty since their contents were classified by filename, not location —
|
||||
# remove them, but only if a move actually emptied them.
|
||||
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
|
||||
d = src_root / stale_dir
|
||||
if d.is_dir() and not any(d.iterdir()):
|
||||
d.rmdir()
|
||||
print(f"removed now-empty directory: {d.relative_to(src_root)}")
|
||||
# remove them, but only if a move actually emptied them. In --copy mode the
|
||||
# originals are still there by design, so leave these alone entirely.
|
||||
if not args.copy:
|
||||
for stale_dir in ("train", "sampling_train/small", "sampling_train"):
|
||||
d = src_root / stale_dir
|
||||
if d.is_dir() and not any(d.iterdir()):
|
||||
d.rmdir()
|
||||
print(f"removed now-empty directory: {d.relative_to(src_root)}")
|
||||
|
||||
print("\nDone.")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user