Succesful data exploration

2025-11-19 11:57:59 +01:00
parent 3e47dcd52f
commit dfdd200867
18 changed files with 40849 additions and 12 deletions
@@ -0,0 +1,63 @@
+#include "csv-helpers.hpp"
+#include "argparse/argparse.hpp"
+#include <iostream>
+
+
+int main(int argc, char* argv[]) {
+    argparse::ArgumentParser program("filter-csv");
+
+    program.add_argument("folder")
+        .help("Folder containing CSV files to filter");
+    program.add_argument("out_file")
+        .help("Output file for filtered CSV file names");
+    program.add_argument("--type-filter", "-t")
+        .help("Filter CSV files by type")
+        .flag();
+    program.add_argument("--departure-airport", "-d")
+        .help("Filter CSV files by proximity to departure airport (lat, lon)")
+        .flag();
+    program.add_argument("--arrival-airport", "-a")
+        .help("Filter CSV files by proximity to arrival airport (lat, lon)")
+        .flag();
+
+    try {
+        program.parse_args(argc, argv);
+    } catch (const std::exception& e) {
+        std::cerr << e.what() << std::endl;
+        std::cerr << program.help().str() << std::endl;
+        return 1;
+    }
+
+    std::string folder = program.get<std::string>("folder");
+    std::string out_file = program.get<std::string>("out_file");
+    bool use_type_filter = program.is_used("--type-filter");
+    bool use_departure_filter = program.is_used("--departure-airport");
+    bool use_arrival_filter = program.is_used("--arrival-airport");
+
+    std::ofstream ofs(out_file);
+    if (!ofs.is_open()) {
+        std::cerr << "Could not open output file: " << out_file << std::endl;
+        return 1;
+    }
+
+    std::vector<std::string> csv_files = list_csv_files_in_folder(folder);
+    for (const auto& csv_file : csv_files) {
+        bool passes_filters = true;
+        CSVRow first_row = read_first_csv_row(csv_file);
+        if(use_type_filter) {
+            passes_filters &= aircraft_type_filter(first_row.t);
+        }
+        if(use_departure_filter) {
+            passes_filters &= near_airport_filter(first_row.lat, first_row.lon, first_row.alt);
+        }
+        if(use_arrival_filter) {
+            CSVRow last_row = read_last_csv_row(csv_file);
+            passes_filters &= near_airport_filter(last_row.lat, last_row.lon, last_row.alt);
+        }
+        if (passes_filters) {
+            ofs << csv_file << std::endl;
+        }
+    }
+    ofs.close();
+    return 0;
+}
@@ -0,0 +1,38 @@
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <cmath>
+#include <stdexcept>
+#include <vector>
+#include <filesystem>
+
+#ifndef CSV_HELPERS_HPP
+#define CSV_HELPERS_HPP
+
+struct CSVRow {
+    std::string icao;
+    std::string r;
+    std::string t;
+    double timestamp;
+    double lat;
+    double lon;
+    double alt;
+    double ias;
+};
+
+const double AIRPORT_PROXIMITY_THRESHOLD_KM = 20.0;
+const double AIRPORT_MAX_ALTITUDE_FT = 5000.0;
+
+CSVRow parse_csv_line(const std::string& line);
+CSVRow read_first_csv_row(const std::string& filepath);
+CSVRow read_last_csv_row(const std::string& filepath);
+
+double haversine_distance(double lat1, double lon1, double lat2, double lon2);
+
+bool aircraft_type_filter(const std::string& t);
+bool near_airport_filter(const double lat, const double lon, const double alt);
+
+std::vector<std::string> list_csv_files_in_folder(const std::string& folder_path);
+
+
+#endif // CSV_HELPERS_HPP
@@ -0,0 +1,117 @@
+#include "csv-helpers.hpp"
+
+// Parse a single CSV line into a CSVRow struct
+CSVRow parse_csv_line(const std::string& line) {
+    CSVRow row;
+    std::istringstream ss(line);
+    std::string token;
+    std::getline(ss, row.icao, ',');
+    std::getline(ss, row.r, ',');
+    std::getline(ss, row.t, ',');
+    std::getline(ss, token, ','); row.timestamp = std::stod(token);
+    std::getline(ss, token, ','); row.lat = std::stod(token);
+    std::getline(ss, token, ','); row.lon = std::stod(token);
+    std::getline(ss, token, ','); row.alt = std::stod(token);
+    std::getline(ss, token, ','); row.ias = std::stod(token);
+    return row;
+}
+
+// Read the first row of a CSV file
+CSVRow read_first_csv_row(const std::string& filepath) {
+    std::ifstream ifs(filepath);
+    if (!ifs.is_open()) {
+        throw std::runtime_error("Could not open file: " + filepath);
+    }
+    std::string line;
+    std::getline(ifs, line); // Skip header
+    std::getline(ifs, line); // Read first data row
+    ifs.close();
+    return parse_csv_line(line);
+}
+
+// Read the last row of a CSV file
+CSVRow read_last_csv_row(const std::string& filepath) {
+    std::ifstream ifs(filepath);
+    if (!ifs.is_open()) {
+        throw std::runtime_error("Could not open file: " + filepath);
+    }
+    std::string line;
+    std::string last_line;
+    std::getline(ifs, line); // Skip header
+    while (std::getline(ifs, line)) {
+        last_line = line;
+    }
+    ifs.close();
+    return parse_csv_line(last_line);
+}
+
+// Haversine distance calculation
+double haversine_distance(double lat1, double lon1, double lat2, double lon2) {
+    const double R = 6371.0; // Earth radius in kilometers
+    double dlat = (lat2 - lat1) * M_PI / 180.0;
+    double dlon = (lon2 - lon1) * M_PI / 180.0;
+    double a = std::sin(dlat / 2) * std::sin(dlat / 2) +
+               std::cos(lat1 * M_PI / 180.0) * std::cos(lat2 * M_PI / 180.0) *
+               std::sin(dlon / 2) * std::sin(dlon / 2);
+    double c = 2 * std::atan2(std::sqrt(a), std::sqrt(1 - a));
+    return R * c;
+}
+
+// Filter by aircraft type
+bool aircraft_type_filter(const std::string& t) {
+    // Return true if type starts with 'A[2-3]' or 'B7' or 'A19N' or 'E'
+    if (t.size() < 2) return false;
+    if (t[0] == 'A' && (t[1] == '2' || t[1] == '3')) return true;
+    if (t[0] == 'B' && t[1] == '7') return true;
+    if (t.find("A19N") != std::string::npos) return true;
+    if (t[0] == 'E') return true;
+    return false;
+}
+
+// Filter by proximity to known airports
+bool near_airport_filter(const double lat, const double lon, const double alt) {
+    // Known airports (lon, lat)
+    /*
+    OSLO_GARDERMOEN_AIRPORT = (11.0859718, 60.1902228)
+    BERGEN_FLESLAND_AIRPORT = (5.2201954, 60.293512)
+    STAVANGER_SOLA_AIRPORT = (5.667972, 58.876667)
+    UMEA_AIRPORT = (20.2840218, 63.7906151)
+    GOTHENBURG_LANDVETTER_AIRPORT = (12.2850909, 57.669275)
+    STOCKHOLM_ARLANDA_AIRPORT = (17.9185743, 59.6519444)
+    TRONDHEIM_VAERNES_AIRPORT = (10.9110182, 63.4577224)
+    TROMSO_LANGNES_AIRPORT = (18.910825, 69.683911)
+    */
+    const std::vector<std::pair<double, double>> known_airports = {
+        {11.0859718, 60.1902228},
+        {5.2201954, 60.293512},
+        {5.667972, 58.876667},
+        {20.2840218, 63.7906151},
+        {12.2850909, 57.669275},
+        {17.9185743, 59.6519444},
+        {10.9110182, 63.4577224},
+        {18.910825, 69.683911}
+    };
+    for (const auto& airport : known_airports) {
+        double distance = haversine_distance(lat, lon, airport.second, airport.first);
+        if (distance <= AIRPORT_PROXIMITY_THRESHOLD_KM) {
+            if (std::isnan(alt) || alt <= AIRPORT_MAX_ALTITUDE_FT) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+// List all CSV files in a folder
+std::vector<std::string> list_csv_files_in_folder(const std::string& folder_path) {
+    std::vector<std::string> csv_files;
+    for (const auto& entry : std::filesystem::directory_iterator(folder_path)) {
+        if (entry.path().extension() == ".csv") {
+            csv_files.push_back(entry.path().string());
+        } else if (std::filesystem::is_directory(entry.path())) {
+            auto nested_files = list_csv_files_in_folder(entry.path().string());
+            csv_files.insert(csv_files.end(), nested_files.begin(), nested_files.end());
+        }
+    }
+    return csv_files;
+}