v1 of data fetching
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
▄████ ██▓ ▒█████ ▄▄▄▄ ▓█████ ██░ ██ ██▓ █████▄▄▄█████▒█████ ██▀███▓ ██ ██
|
||||
██▒ ▀█▓██▒ ▒██▒ █▓█████▄▓█ ▀▓██░ ██▓██▒██ ▓ ██▒ ▒██▒ █▓██ ▒ ██▒ ██ ██
|
||||
▒██░▄▄▄▒██░ ▒██░ █▒██▒ ▄█▒███ ▒██▀▀██▒██░ ▓██▄ ▒ ▓██░ ▒██░ █▓██ ░▄█ ▒██ ██
|
||||
░▓█ ██▒██░ ▒██ █▒██░█▀ ▒▓█ ▄░▓█ ░██░██░ ▒ █░ ▓██▓ ▒██ █▒██▀▀█▄ ░ ▐██▓
|
||||
░▒▓███▀░██████░ ████▓░▓█ ▀█░▒████░▓█▒░██░██▒██████▒ ▒██▒ ░ ████▓░██▓ ▒██ ░ ██▒▓
|
||||
░▒ ▒░ ▒░▓ ░ ▒░▒░▒░▒▓███▀░░ ▒░ ░▒ ░░▒░░▓ ▒ ▒▓▒ ▒ ▒ ░░ ░ ▒░▒░▒░ ▒▓ ░▒▓ ██▒▒▒
|
||||
░ ░░ ░ ▒ ░ ░ ▒ ▒▒░▒ ░ ░ ░ ░▒ ░▒░ ░▒ ░ ░▒ ░ ░ ░ ▒ ▒░ ░▒ ░ ▒ ▓██ ░▒░
|
||||
░ ░ ░ ░ ░ ░ ░ ░ ▒ ░ ░ ░ ░ ░░ ░▒ ░ ░ ░ ░ ░ ░ ░ ▒ ░░ ░ ▒ ▒ ░░
|
||||
░ ░ ░ ░ ░ ░ ░ ░░ ░ ░░ ░ ░ ░ ░ ░ ░
|
||||
░
|
||||
|
||||
https://github.com/adsblol/globe_history_2024/
|
||||
|
||||
This is a database for the day of 2024-12-29 of all aircraft known to adsb.lol.
|
||||
GitHub Release: v2024.12.29-planes-readsb-prod-0
|
||||
GitHub Download Link: https://github.com/adsblol/globe_history_2024/releases/tag/v2024.12.29-planes-readsb-prod-0
|
||||
Uploaded on: 2024-12-30
|
||||
Original Path: /var/globe_history/2024/12/29
|
||||
Pod of origin: planes-readsb-prod-0
|
||||
|
||||
It was made by readsb (https://github.com/wiedehopf/readsb)
|
||||
|
||||
|
||||
This database is made available under the Open Database License:
|
||||
http://opendatacommons.org/licenses/odbl/1.0/.
|
||||
Attached locally: LICENSE-ODbL.txt
|
||||
|
||||
This is made possible by the adsb.lol feeders.
|
||||
If you want to help, please consider increasing coverage by adding a feeder:
|
||||
https://adsb.lol/feed
|
||||
|
||||
By feeding adsb.lol, you agree to the extent possible under law,
|
||||
to waive all copyright and related or neighboring rights to your data, associating your work with the CC0 license.
|
||||
https://creativecommons.org/publicdomain/zero/1.0/
|
||||
Attached locally: LICENSE-CC0.txt
|
||||
@@ -0,0 +1,43 @@
|
||||
#include <iostream>
|
||||
#include "adsb-helpers.hpp"
|
||||
|
||||
int N_THREADS = 16;
|
||||
|
||||
std::vector<std::string> scan_directory_for_json(const std::string& directory_path) {
|
||||
std::vector<std::string> json_files;
|
||||
for (const auto& entry : std::filesystem::directory_iterator(directory_path)) {
|
||||
if (entry.path().extension() == ".json" || entry.path().extension() == ".json.gz") {
|
||||
json_files.push_back(entry.path().string());
|
||||
}
|
||||
if (entry.is_directory()) {
|
||||
auto nested_files = scan_directory_for_json(entry.path().string());
|
||||
json_files.insert(json_files.end(), nested_files.begin(), nested_files.end());
|
||||
}
|
||||
}
|
||||
return json_files;
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
if (argc != 2) {
|
||||
std::cerr << "Usage: " << argv[0] << " <input_adsb_directory>" << std::endl;
|
||||
return 1;
|
||||
}
|
||||
std::string input_directory = argv[1];
|
||||
|
||||
// Get list of all .json files in the input directory
|
||||
std::vector<std::string> json_files = scan_directory_for_json(input_directory);
|
||||
std::cout << "Found " << json_files.size() << " JSON files." << std::endl;
|
||||
|
||||
// OpenMP for parallel processing
|
||||
#pragma omp parallel for num_threads(N_THREADS)
|
||||
for (size_t i = 0; i < json_files.size(); ++i) {
|
||||
const std::string& input_filepath = json_files[i];
|
||||
std::string output_filepath = input_filepath + ".csv"; // Example output path
|
||||
try {
|
||||
process_adsb_file(input_filepath, output_filepath);
|
||||
} catch (const std::exception& e) {
|
||||
std::cerr << "Error processing file " << input_filepath << ": " << e.what() << std::endl;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include <fstream>
|
||||
#include <zlib.h>
|
||||
#include <stdexcept>
|
||||
#include <iomanip>
|
||||
|
||||
#ifndef ADSB_HELPERS_HPP
|
||||
#define ADSB_HELPERS_HPP
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
struct FlightInfo {
|
||||
std::string icao;
|
||||
std::string r;
|
||||
std::string t;
|
||||
double timestamp;
|
||||
};
|
||||
|
||||
std::vector<std::vector<double>> extract_trace_values(const json& data, double base_ts);
|
||||
FlightInfo extract_flight_info(const json& data);
|
||||
json parse_json_file(const std::string& filepath);
|
||||
void write_csv(const FlightInfo& info, const std::vector<std::vector<double>>& trace_values, const std::string& output_filepath);
|
||||
bool check_bounds(const std::vector<double>& values);
|
||||
void process_adsb_file(const std::string& input_filepath, const std::string& output_filepath);
|
||||
#endif
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,134 @@
|
||||
#include "adsb-helpers.hpp"
|
||||
|
||||
// Extracts the first ts + offset, lat, lon, alt, ias from trace entries
|
||||
std::vector<std::vector<double>> extract_trace_values(const json& data, double base_ts) {
|
||||
std::vector<std::vector<double>> result;
|
||||
|
||||
if (!data.contains("trace") || !data["trace"].is_array())
|
||||
return result;
|
||||
|
||||
for (const auto& entry : data["trace"]) {
|
||||
if (!entry.is_array() || entry.size() < 4)
|
||||
continue;
|
||||
|
||||
std::vector<double> values;
|
||||
// Extract first four values: ts, lat, lon, alt
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
if (entry[i].is_number()) {
|
||||
double val = entry[i].get<double>();
|
||||
if (i == 0) val += base_ts; // apply offset to first value
|
||||
values.push_back(val);
|
||||
} else if (i == 3) {
|
||||
// altitude can be null or "ground"
|
||||
if (entry[i].is_string()) {
|
||||
std::string alt_str = entry[i].get<std::string>();
|
||||
if (alt_str == "ground") {
|
||||
values.push_back(0.0);
|
||||
} else {
|
||||
values.push_back(std::numeric_limits<double>::quiet_NaN());
|
||||
}
|
||||
} else {
|
||||
values.push_back(std::numeric_limits<double>::quiet_NaN());
|
||||
}
|
||||
} else {
|
||||
values.push_back(std::numeric_limits<double>::quiet_NaN());
|
||||
}
|
||||
}
|
||||
// Extract ias if present
|
||||
if (entry.size() > 12 && entry[12].is_number()) {
|
||||
values.push_back(entry[12].get<double>());
|
||||
} else {
|
||||
values.push_back(std::numeric_limits<double>::quiet_NaN());
|
||||
}
|
||||
result.push_back(std::move(values));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Extracts icao, r, t, timestamp into a struct
|
||||
FlightInfo extract_flight_info(const json& data) {
|
||||
FlightInfo info;
|
||||
info.icao = data.value("icao", "");
|
||||
info.r = data.value("r", "");
|
||||
info.t = data.value("t", "");
|
||||
info.timestamp = data.value("timestamp", 0.0);
|
||||
return info;
|
||||
}
|
||||
|
||||
|
||||
// Parse JSON from filepath
|
||||
|
||||
json parse_json_file(const std::string& filepath) {
|
||||
gzFile gz_file = gzopen(filepath.c_str(), "rb");
|
||||
if (!gz_file) {
|
||||
throw std::runtime_error("Could not open gzipped file: " + filepath);
|
||||
}
|
||||
|
||||
constexpr size_t BUFFER_SIZE = 4096;
|
||||
char buffer[BUFFER_SIZE];
|
||||
std::string json_str;
|
||||
|
||||
int bytes_read;
|
||||
while ((bytes_read = gzread(gz_file, buffer, BUFFER_SIZE)) > 0) {
|
||||
json_str.append(buffer, bytes_read);
|
||||
}
|
||||
|
||||
gzclose(gz_file);
|
||||
|
||||
try {
|
||||
return json::parse(json_str);
|
||||
} catch (const std::exception& e) {
|
||||
throw std::runtime_error(std::string("JSON parse error: ") + e.what());
|
||||
}
|
||||
}
|
||||
|
||||
// Write CSV file
|
||||
void write_csv(const FlightInfo& info, const std::vector<std::vector<double>>& trace_values, const std::string& output_filepath) {
|
||||
std::ofstream ofs(output_filepath);
|
||||
if (!ofs.is_open()) {
|
||||
throw std::runtime_error("Could not open output file: " + output_filepath);
|
||||
}
|
||||
// Write header
|
||||
ofs << "icao,r,t,timestamp,lat,lon,alt,ias\n";
|
||||
|
||||
for (const auto& values : trace_values) {
|
||||
ofs << info.icao << "," << info.r << "," << info.t;
|
||||
for (const auto& val : values) {
|
||||
ofs << "," << std::fixed << std::setprecision(5) << val;
|
||||
}
|
||||
ofs << "\n";
|
||||
}
|
||||
ofs.close();
|
||||
}
|
||||
|
||||
|
||||
// Check if values are within realistic bounds
|
||||
bool check_bounds(const std::vector<double>& values) {
|
||||
if (values.size() < 4) return false;
|
||||
double lat = values[1];
|
||||
double lon = values[2];
|
||||
double alt = values[3];
|
||||
|
||||
// BBOX for Norway
|
||||
// 3.076172,57.562995,31.728516,71.357067
|
||||
if (lat < 57.562995 || lat > 71.357067) return false;
|
||||
if (lon < 3.076172 || lon > 31.728516) return false;
|
||||
if (alt < 0.0) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
void process_adsb_file(const std::string& input_filepath, const std::string& output_filepath) {
|
||||
json data = parse_json_file(input_filepath);
|
||||
FlightInfo info = extract_flight_info(data);
|
||||
std::vector<std::vector<double>> trace_values = extract_trace_values(data, info.timestamp);
|
||||
|
||||
// Check if either start or end point is in bounds
|
||||
bool start_in_bounds = !trace_values.empty() && check_bounds(trace_values.front());
|
||||
bool end_in_bounds = !trace_values.empty() && check_bounds(trace_values.back());
|
||||
|
||||
if (!start_in_bounds && !end_in_bounds) {
|
||||
return; // Skip writing if neither point is in bounds
|
||||
}
|
||||
|
||||
write_csv(info, trace_values, output_filepath);
|
||||
}
|
||||
Reference in New Issue
Block a user