v1 of data fetching

2025-11-12 12:35:45 +01:00
commit bc6d436b85
16 changed files with 26636 additions and 0 deletions
@@ -0,0 +1,134 @@
+#include "adsb-helpers.hpp"
+
+// Extracts the first ts + offset, lat, lon, alt, ias from trace entries
+std::vector<std::vector<double>> extract_trace_values(const json& data, double base_ts) {
+    std::vector<std::vector<double>> result;
+
+    if (!data.contains("trace") || !data["trace"].is_array())
+        return result;
+
+    for (const auto& entry : data["trace"]) {
+        if (!entry.is_array() || entry.size() < 4) 
+            continue;
+
+        std::vector<double> values;
+        // Extract first four values: ts, lat, lon, alt
+        for (size_t i = 0; i < 4; ++i) {
+            if (entry[i].is_number()) {
+                double val = entry[i].get<double>();
+                if (i == 0) val += base_ts;  // apply offset to first value
+                values.push_back(val);
+            } else if (i == 3) {
+                // altitude can be null or "ground"
+                if (entry[i].is_string()) {
+                    std::string alt_str = entry[i].get<std::string>();
+                    if (alt_str == "ground") {
+                        values.push_back(0.0);
+                    } else {
+                        values.push_back(std::numeric_limits<double>::quiet_NaN());
+                    }
+                } else {
+                    values.push_back(std::numeric_limits<double>::quiet_NaN());
+                }
+            } else {
+                values.push_back(std::numeric_limits<double>::quiet_NaN());
+            }
+        }
+        // Extract ias if present
+        if (entry.size() > 12 && entry[12].is_number()) {
+            values.push_back(entry[12].get<double>());
+        } else {
+            values.push_back(std::numeric_limits<double>::quiet_NaN());
+        }
+        result.push_back(std::move(values));
+    }
+    return result;
+}
+
+// Extracts icao, r, t, timestamp into a struct
+FlightInfo extract_flight_info(const json& data) {
+    FlightInfo info;
+    info.icao = data.value("icao", "");
+    info.r = data.value("r", "");
+    info.t = data.value("t", "");
+    info.timestamp = data.value("timestamp", 0.0);
+    return info;
+}
+
+
+// Parse JSON from filepath
+
+json parse_json_file(const std::string& filepath) {
+    gzFile gz_file = gzopen(filepath.c_str(), "rb");
+    if (!gz_file) {
+        throw std::runtime_error("Could not open gzipped file: " + filepath);
+    }
+
+    constexpr size_t BUFFER_SIZE = 4096;
+    char buffer[BUFFER_SIZE];
+    std::string json_str;
+
+    int bytes_read;
+    while ((bytes_read = gzread(gz_file, buffer, BUFFER_SIZE)) > 0) {
+        json_str.append(buffer, bytes_read);
+    }
+
+    gzclose(gz_file);
+
+    try {
+        return json::parse(json_str);
+    } catch (const std::exception& e) {
+        throw std::runtime_error(std::string("JSON parse error: ") + e.what());
+    }
+}
+
+// Write CSV file
+void write_csv(const FlightInfo& info, const std::vector<std::vector<double>>& trace_values, const std::string& output_filepath) {
+    std::ofstream ofs(output_filepath);
+    if (!ofs.is_open()) {
+        throw std::runtime_error("Could not open output file: " + output_filepath);
+    }
+    // Write header
+    ofs << "icao,r,t,timestamp,lat,lon,alt,ias\n";
+
+    for (const auto& values : trace_values) {
+        ofs << info.icao << "," << info.r << "," << info.t;
+        for (const auto& val : values) {
+            ofs << "," << std::fixed << std::setprecision(5) << val;
+        }
+        ofs << "\n";
+    }
+    ofs.close();
+}
+
+
+// Check if values are within realistic bounds
+bool check_bounds(const std::vector<double>& values) {
+    if (values.size() < 4) return false;
+    double lat = values[1];
+    double lon = values[2];
+    double alt = values[3];
+
+    // BBOX for Norway
+    // 3.076172,57.562995,31.728516,71.357067
+    if (lat < 57.562995 || lat > 71.357067) return false;
+    if (lon < 3.076172 || lon > 31.728516) return false;
+    if (alt < 0.0) return false;
+    return true;
+}
+
+void process_adsb_file(const std::string& input_filepath, const std::string& output_filepath) {
+    json data = parse_json_file(input_filepath);
+    FlightInfo info = extract_flight_info(data);
+    std::vector<std::vector<double>> trace_values = extract_trace_values(data, info.timestamp);
+
+    // Check if either start or end point is in bounds
+    bool start_in_bounds = !trace_values.empty() && check_bounds(trace_values.front());
+    bool end_in_bounds = !trace_values.empty() && check_bounds(trace_values.back());
+
+    if (!start_in_bounds && !end_in_bounds) {
+        return; // Skip writing if neither point is in bounds
+    }
+
+    write_csv(info, trace_values, output_filepath);
+}