v1 of data fetching

Code/python/src/adsbpy/fetch.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+import os
+import subprocess
+import urllib.request
+from pathlib import Path
+import tarfile
+import shutil
+
+# --- configuration ---
+BASE_DIR = Path(__file__).parent.resolve() / Path("../../../..")  # root of project
+DATA_DIR = BASE_DIR / Path("data")           # where extracted tars go
+CSV_DIR = BASE_DIR / Path("data/csv")             # where final csvs go
+EXTRACT_ADSB = BASE_DIR / Path("Code/cpp/extract-adsb")  # external binary
+# -----------------------
+
+
+def download(url: str, dest: Path):
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    print(f"Downloading {url} → {dest}")
+    with urllib.request.urlopen(url) as r, open(dest, "wb") as f:
+        shutil.copyfileobj(r, f)
+
+def get_name_from_url(url: str) -> str:
+    return url.split("/")[-1]
+
+def extract_adsb_data(cmd):
+    print(f"Running: {' '.join(map(str, cmd))}")
+    subprocess.run(cmd, check=True)
+
+def extract_tar(tar_paths: list[Path], extract_dir: Path):
+    extract_dir.mkdir(parents=True, exist_ok=True)
+    if len(tar_paths) == 1:
+        tar_path = tar_paths[0]
+    else:
+        # concatenate parts
+        # Remove all files without .tar.aa, .tar.ab, ... suffixes
+        tar_paths = [p for p in tar_paths if len(p.suffix) == 3]
+        tar_path = tar_paths[0].stem # files are named foo.tar.aa, foo.tar.ab, ...
+        tar_path = extract_dir / tar_path
+        print(f"Concatenating {len(tar_paths)} parts → {tar_path}")
+        with open(tar_path, "wb") as out:
+            for part in tar_paths:
+                with open(part, "rb") as pf:
+                    shutil.copyfileobj(pf, out)
+    print(f"Extracting {tar_path} → {extract_dir}")
+    with tarfile.open(tar_path) as tf:
+        tf.extractall(path=extract_dir)
+    return extract_dir
+
+def move_csv_files(src_dir: Path, dest_dir: Path):
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    for p in src_dir.rglob("*.csv"):
+        dest = dest_dir / p.name
+        shutil.move(str(p), dest)
+
+def remove_dir(dir_path: Path):
+    for p in dir_path.rglob("*"):
+        if p.is_file():
+            p.unlink()
+    for p in dir_path.rglob("*"):
+        if p.is_dir():
+            if not any(p.iterdir()):
+                p.rmdir()
+            else:
+                remove_dir(p)
+    dir_path.rmdir()
+
+def fetch_lines(lines: list[str]):
+    for line in lines:
+        parts = line.strip().split(",")
+        if not parts:
+            continue
+        tar_files = []
+        for url in parts:
+            filename = DATA_DIR / get_name_from_url(url)
+            tar_files.append(filename)
+            download(url, filename)
+
+        extract_dir = tar_files[0].parent / tar_files[0].stem
+        extract_tar(tar_files, extract_dir)
+        traces_dir = extract_dir / "traces"
+        if traces_dir.exists():
+            extract_adsb_data([EXTRACT_ADSB, str(traces_dir)])
+        csv_files_dir = CSV_DIR / extract_dir.name
+        move_csv_files(extract_dir, csv_files_dir)
+        # clean up
+        for tar_file in tar_files:
+            if tar_file.exists():
+                tar_file.unlink()
+        remove_dir(extract_dir)
+
+def fetch_file(file: Path):
+    with open(file) as f:
+        lines = f.readlines()
+    fetch_lines(lines)
+