diff --git a/giant/cli.py b/giant/cli.py index 2e1d6c2..327ec5e 100644 --- a/giant/cli.py +++ b/giant/cli.py @@ -1,5 +1,5 @@ from collections import Counter -from datetime import date, datetime, timezone +from datetime import datetime, timezone from enum import Enum import math from pathlib import Path @@ -388,7 +388,10 @@ def train( out: Annotated[ Optional[Path], typer.Option( - "--out", "-o", help="Checkpoint dir (default: auto from hyperparams)" + "--out", + "-o", + help="Checkpoint dir (default: timestamped dir from hyperparams, " + "or the --resume checkpoint's own dir when resuming)", ), ] = None, device: Annotated[ @@ -513,16 +516,28 @@ def train( f"batch_size: {t['batch_size']} (auto-estimated from free GPU memory)" ) - out_dir = out or Path( - f"checkpoints/{date.today().strftime('%Y%m%d')}" - f"_{t['mode']}" - f"_h{m['hidden_dim']}" - f"_b{m['n_blocks']}" - f"_e{m['emb_dim']}" - f"_c{m['conditioning']}" - f"_lr{t['lr']}" - f"_bs{t['batch_size']}" - ) + if out is not None: + out_dir = out + elif resume is not None: + # Continue writing into the resumed checkpoint's own directory + # rather than recomputing a hyperparam-derived name — the latter + # would (a) collide with the original run's dir only by accident + # (same day, unchanged hyperparams) and now never collides at all + # since the fresh-run name below is timestamped to the second, and + # (b) silently start a fresh directory if a resumed run tweaks any + # hyperparam baked into the name (e.g. --lr for a fine-tune). + out_dir = resume.parent + else: + out_dir = Path( + f"checkpoints/{datetime.now().strftime('%Y%m%d_%H%M%S')}" + f"_{t['mode']}" + f"_h{m['hidden_dim']}" + f"_b{m['n_blocks']}" + f"_e{m['emb_dim']}" + f"_c{m['conditioning']}" + f"_lr{t['lr']}" + f"_bs{t['batch_size']}" + ) typer.echo(f"device: {_device}") typer.echo(f"out_dir: {out_dir}")