Succesful data exploration

Code/python/src/adsbpy/csvutil.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import pathlib
+
+def read_csv(file_path: pathlib.Path) -> pd.DataFrame:
+    """Reads a CSV file into a pandas DataFrame."""
+    return pd.read_csv(file_path)
+
+def read_basics(file_path: pathlib.Path) -> pd.Series:
+    """Reads first and last row of a CSV file into a pandas Series."""
+    df = pd.read_csv(file_path, nrows=1)
+    first_row = df.iloc[0]
+    data = {
+        "file": str(file_path),
+        "icao": first_row["icao"],
+        "reg": first_row["r"],
+        "type": first_row["t"],
+        "first_seen": first_row["timestamp"],
+        "first_lat": first_row["lat"],
+        "first_lon": first_row["lon"],
+        "first_alt": first_row["alt"],
+        "first_airspeed": first_row["ias"],
+    }
+    try:
+        df_last = pd.read_csv(file_path, skiprows=lambda x: x != 0 and x != sum(1 for _ in open(file_path)) - 1)
+        last_row = df_last.iloc[-1]
+        data.update({
+            "last_seen": last_row["timestamp"],
+            "last_lat": last_row["lat"],
+            "last_lon": last_row["lon"],
+            "last_alt": last_row["alt"],
+            "last_airspeed": last_row["ias"],
+        })
+    except Exception:
+        pass
+    return pd.Series(data)
+
+def get_basics_from_dir(directory: pathlib.Path, go_into_subdirs: bool = False) -> pd.DataFrame:
+    """Aggregates basics from all CSV files in a directory into a DataFrame."""
+    all_basics = []
+    pattern = "**/*.csv" if go_into_subdirs else "*.csv"
+    for csv_file in directory.glob(pattern):
+        basics = read_basics(csv_file)
+        all_basics.append(basics)
+    return pd.DataFrame(all_basics)

Code/python/src/adsbpy/fetch.py

@@ -122,24 +122,27 @@ async def extract_adsb_data(cmd: List[str]) -> None:
 def extract_tar_sync(tar_paths: List[Path], extract_dir: Path) -> Path:
     """Synchronous tar extraction (CPU-bound, run in thread pool)."""
     extract_dir.mkdir(parents=True, exist_ok=True)
-    
+
     if len(tar_paths) == 1:
         tar_path = tar_paths[0]
     else:
-        # Concatenate multi-part tar files
-        tar_paths = [p for p in tar_paths if len(p.suffix) == 3]
-        tar_path = extract_dir / tar_paths[0].stem
-        
-        logger.info(f"Concatenating {len(tar_paths)} parts → {pretty_path(tar_path)}")
-        with open(tar_path, 'wb') as outfile:
-            for part in tar_paths:
-                with open(part, 'rb') as infile:
+        # Keep only multipart fragments like .aa, .ab, .ac ...
+        parts = [p for p in tar_paths if len(p.suffix) == 3]
+
+        # Derive correct base archive name: strip only the multipart suffix
+        base = parts[0].with_suffix("")  # "foo.tar.aa" → "foo.tar"
+        tar_path = extract_dir / base.name
+
+        logger.info(f"Concatenating {len(parts)} parts → {pretty_path(tar_path)}")
+        with open(tar_path, "wb") as outfile:
+            for part in sorted(parts):
+                with open(part, "rb") as infile:
                     shutil.copyfileobj(infile, outfile)
 
     logger.info(f"Extracting {pretty_path(tar_path)} → {pretty_path(extract_dir)}")
     with tarfile.open(tar_path) as tf:
         tf.extractall(path=extract_dir)
-    
+
     return extract_dir
 
 

Code/python/src/adsbpy/prettylogging.py

@@ -0,0 +1,94 @@
+import logging
+import sys
+from datetime import datetime
+
+# Color codes
+RESET = "\033[0m"
+BOLD = "\033[1m"
+RED = "\033[31m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+BLUE = "\033[34m"
+MAGENTA = "\033[35m"
+CYAN = "\033[36m"
+WHITE = "\033[37m"
+GRAY = "\033[90m"
+
+# Color mapping for different log levels
+LOG_COLORS = {
+    logging.DEBUG: GRAY,
+    logging.INFO: CYAN,
+    logging.WARNING: YELLOW,
+    logging.ERROR: RED,
+    logging.CRITICAL: RED + BOLD,
+}
+
+class ColoredFormatter(logging.Formatter):
+    def __init__(self, fmt=None, datefmt=None):
+        super().__init__(fmt, datefmt)
+    
+    def format(self, record):
+        # Get the level color
+        level_color = LOG_COLORS.get(record.levelno, WHITE)
+        
+        # Create colored level name
+        levelname = f"{level_color}{record.levelname:8}{RESET}"
+        
+        # Format time as HH:mm
+        time_str = datetime.now().strftime("%H:%M")
+        time_formatted = f"{GRAY}{time_str}{RESET}"
+        
+        # Format origin (logger name)
+        origin = f"{BLUE}{record.name}{RESET}"
+        
+        # Format message
+        message = f"{WHITE}{record.getMessage()}{RESET}"
+        
+        # Construct the final log line
+        log_line = f"{time_formatted} {levelname} {origin} → {message}"
+        
+        # Format exception if present
+        if record.exc_info:
+            log_line += f"\n{RED}{self.formatException(record.exc_info)}{RESET}"
+        
+        return log_line
+
+def setup_logger(level=logging.INFO):
+    # Create logger
+    logger = logging.getLogger()
+    logger.setLevel(level)
+    
+    # Remove existing handlers
+    for handler in logger.handlers[:]:
+        logger.removeHandler(handler)
+    
+    # Create console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_handler.setLevel(logging.DEBUG)
+    
+    # Create formatter and add it to the handler
+    formatter = ColoredFormatter()
+    console_handler.setFormatter(formatter)
+    
+    # Add handler to the logger
+    logger.addHandler(console_handler)
+    
+    return logger
+
+# Example usage
+if __name__ == "__main__":
+    logger = setup_logger()
+    
+    # Test different log levels
+    logger.debug("This is a debug message")
+    logger.info("This is an info message")
+    logger.warning("This is a warning message")
+    logger.error("This is an error message")
+    logger.critical("This is a critical message")
+    
+    # Test with different origins
+    module_logger = logging.getLogger("my_module")
+    module_logger.info("This comes from a specific module")
+    
+    another_logger = logging.getLogger("another.component")
+    another_logger.warning("This comes from another component")

Code/python/src/aiRNN/dataloader.py

@@ -0,0 +1,5 @@
+import pandas as pd
+import tensorflow as tf
+import numpy as np
+import pathlib
+