diff --git a/scripts/bump_dataset_version.py b/scripts/bump_dataset_version.py index 9def3ef..0b61ce3 100644 --- a/scripts/bump_dataset_version.py +++ b/scripts/bump_dataset_version.py @@ -120,11 +120,46 @@ def apply_bump(root: Path, new_dirs: list[Path], log_line: str) -> None: f.write(log_line + "\n") +def _du(path: Path) -> int: + """Total size in bytes of all regular files under *path* (0 if missing).""" + if not path.is_dir(): + return 0 + total = 0 + for dirpath, _dirnames, filenames in os.walk(path): + for name in filenames: + fp = Path(dirpath) / name + try: + total += fp.stat().st_size + except OSError: + pass + return total + + +def _human_size(n: int) -> str: + size = float(n) + for unit in ("B", "KB", "MB", "GB", "TB"): + if size < 1024 or unit == "TB": + return f"{size:.1f} {unit}" if unit != "B" else f"{int(size)} {unit}" + size /= 1024 + return f"{size:.1f} TB" + + +_ROW_WIDTH = 48 + + +def _row(label: str, size_bytes: int, indent: int = 0) -> str: + text = " " * indent + label + size_str = _human_size(size_bytes) + pad = max(1, _ROW_WIDTH - len(text) - len(size_str)) + return text + " " * pad + size_str + + def print_status(root: Path) -> None: raw_root = root / "raw" if not raw_root.is_dir(): print(f"no raw/ tree found under {root}") return + grand_total = 0 for kind_dir in sorted(p for p in raw_root.iterdir() if p.is_dir()): kind = kind_dir.name gens = sorted( @@ -132,9 +167,11 @@ def print_status(root: Path) -> None: for m in (GEN_RE.match(p.name) for p in kind_dir.iterdir() if p.is_dir()) if m ) - print(f"kind={kind}") + print(f"{kind}/") + kind_total = 0 for gen in gens: gen_tag = f"gen{gen}" + raw_size = _du(raw_root / kind / gen_tag) schema_dir = root / "processed" / kind / gen_tag schemas = sorted( int(m.group(1)) @@ -145,8 +182,30 @@ def print_status(root: Path) -> None: ) if m ) - schema_str = ", ".join(f"schema{s}" for s in schemas) or "(none)" - print(f" {gen_tag}: {schema_str}") + schema_sizes = {s: _du(schema_dir / f"schema{s}") for s in schemas} + processed_size = sum(schema_sizes.values()) + gen_total = raw_size + processed_size + kind_total += gen_total + + print(_row(gen_tag, gen_total, indent=1)) + print(_row("raw", raw_size, indent=2)) + print(_row("processed", processed_size, indent=2)) + if schemas: + for s in schemas: + print(_row(f"schema{s}", schema_sizes[s], indent=3)) + else: + print(" (none)") + print(_row(f"{kind} total", kind_total, indent=1)) + print() + grand_total += kind_total + + derived_size = _du(root / "derived") + pools_size = _du(root / "pools") + grand_total += derived_size + pools_size + print(_row("derived/", derived_size)) + print(_row("pools/", pools_size)) + print("-" * _ROW_WIDTH) + print(_row("grand total", grand_total)) # ---------------------------------------------------------------------------