with '#' will be ignored, and an empty message aborts the commit.

Minor changes, run autoencoder on ml node

Code/python/src/aiRNN/dataloader.py

@@ -1,10 +1,12 @@
 import pandas as pd
 import torch
+import torch.nn.functional as F
 from torch.utils.data import Dataset, DataLoader
 import numpy as np
 import pathlib
 from typing import Sequence
 import random
+from aiRNN import preprocessors
 
 class BaseDataset(Dataset):
     def __init__(
@@ -15,7 +17,7 @@ class BaseDataset(Dataset):
         n_windows_per_file,
         step=1,
         feature_columns=("lat", "lon", "alt", "ias"),
-        context_columns=("r","t"),
+        context_columns=(f"type_encoding_{i}" for i in range(4), "last_lat", "last_lon", "last_alt", "last_ias"),
         time_columns=("timestamp",),
         target_columns=("lat", "lon", "alt"),
         device="cpu",
@@ -38,7 +40,6 @@ class BaseDataset(Dataset):
 
         # Total samples = windows_per_file * number_of_files
         self.total_windows = n_windows_per_file * len(self.filepaths)
-
         for col_list in [self.feature_cols, self.context_cols, self.time_cols, self.target_cols]:
             assert all(
                 col in sample.columns for col in col_list
@@ -68,6 +69,28 @@ class BaseDataset(Dataset):
 
     def _modify_df(self, df):
         """Hook for subclasses to modify dataframe before slicing windows."""
+        df.loc[:, ["lat", "lon", "alt"]] = preprocessors.norm_coords(
+            df["lat"].values, df["lon"].values, df["alt"].values
+        )
+        df.loc[:, "ias"] = preprocessors.norm_ias(df["ias"].values)
+        df.loc[:, "timestamp"] = preprocessors.norm_time(df["timestamp"].values)
+        for col in ["lat", "lon", "alt", "ias"]:
+            df.loc[:, col] = preprocessors.fillna_with_mean(df[col])
+
+        mapped_df = preprocessors.map_categories(df, preprocessors.category_mappings)
+        X_a_1 = F.one_hot(torch.tensor(mapped_df["R_1_IDX"].values)).float()
+        X_a_2 = F.one_hot(torch.tensor(mapped_df["R_2_IDX"].values)).float()
+        X_t = F.one_hot(torch.tensor(mapped_df["T_IDX"].values)).float()
+        df.loc[:, [f"type_encoding_{i}" for i in range(4)]] = preprocessors.encode_features(
+            X_a_1, X_a_2, X_t
+        )
+        last_row = df.iloc[-1][["lat", "lon", "alt", "ias"]]
+
+        df.loc[:, "last_lat"] = last_row["lat"]
+        df.loc[:, "last_lon"] = last_row["lon"]
+        df.loc[:, "last_alt"] = last_row["alt"]
+        df.loc[:, "last_ias"] = last_row["ias"]
+
         return df
 
 
@@ -108,6 +131,7 @@ class EvenlySpacedDataset(BaseDataset):
                 try:
                     start_idx, end_idx = self._get_start_end_indices(w, n_rows)
                     window = df.iloc[start_idx:end_idx:self.step]
+                    window = self._modify_df(window)
                     self.data.append(window)
                 except ValueError:
                     self.total_windows -= 1
@@ -184,6 +208,7 @@ class EvenlySpacedStreamingDataset(BaseDataset):
         # Read only the required rows
         skip = list(set(range(1, start_idx + 1)))
         df = pd.read_csv(fp, skiprows=skip, nrows=self.window_size * self.step).iloc[::self.step]
+        df = self._modify_df(df)
 
         # Slice into input / decoder-input / targets
         X_feat = df.iloc[: self.n_input][self.feature_cols].to_numpy(np.float32)
@@ -272,4 +297,15 @@ def get_datasets(
     )
 
     return train_dataset, val_dataset, test_dataset
-    
+    
+
+class EncodingDataset(Dataset):
+    def __init__(self, X_a_1, X_a_2, X_b):
+        super().__init__()
+        self.X_a_1 = X_a_1
+        self.X_a_2 = X_a_2
+        self.X_b = X_b
+    def __len__(self):
+        return self.X_a_1.shape[0]
+    def __getitem__(self, idx):
+        return self.X_a_1[idx], self.X_a_2[idx], self.X_b[idx]