fix primary keys, do not insert items for which no run can be found.

Northeastern-Electric-Racing · Dec 20, 2024 · e6dcbc0 · e6dcbc0
1 parent 53f2559
commit e6dcbc0
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 29 deletions.
diff --git a/scylla-server/migrations/2024-11-10-031516_create_all/up.sql b/scylla-server/migrations/2024-11-10-031516_create_all/up.sql
@@ -1,26 +1,30 @@
+CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+
 -- CreateTable
 CREATE TABLE "run" (
     "id" SERIAL NOT NULL,
     "locationName" TEXT,
     "latitude" DOUBLE PRECISION,
     "longitude" DOUBLE PRECISION,
     "driverName" TEXT,
-    "notes" TEXT DEFAULT '',
+    "notes" TEXT NOT NULL DEFAULT '',
     "time" TIMESTAMPTZ NOT NULL,
 
     CONSTRAINT "run_pkey" PRIMARY KEY ("id")
 );
 
 -- CreateTable
 CREATE TABLE "data" (
-    "id" SERIAL NOT NULL,
     "values" DOUBLE PRECISION[],
     "dataTypeName" TEXT NOT NULL,
     "time" TIMESTAMPTZ NOT NULL,
     "runId" INTEGER NOT NULL,
 
-    CONSTRAINT "data_pkey" PRIMARY KEY ("id")
+    PRIMARY KEY("time", "dataTypeName")
 );
+-- SELECT * FROM create_hypertable("data", by_range("time"));
+-- SELECT * FROM add_dimension("data", by_hash("dataTypeNmae", 4));
+
 
 -- CreateTable
 CREATE TABLE "dataType" (
@@ -34,9 +38,6 @@ CREATE TABLE "dataType" (
 -- CreateIndex
 CREATE UNIQUE INDEX "run_id_key" ON "run"("id");
 
--- CreateIndex
-CREATE UNIQUE INDEX "data_id_time_key" ON "data"("id", "time");
-
 -- CreateIndex
 CREATE UNIQUE INDEX "dataType_name_key" ON "dataType"("name");
 

diff --git a/scylla-server/src/controllers/file_insertion_controller.rs b/scylla-server/src/controllers/file_insertion_controller.rs
@@ -66,38 +66,28 @@ pub async fn insert_file(
             }
         }
 
+        let size_before_filter = insertable_data.len();
         debug!("Mapping data to ClientData type, with inferred run IDs!");
-        let new_data: Vec<ClientData> = insertable_data
+        let insertable_data: Vec<ClientData> = insertable_data
             .into_iter()
-            .map(|f| match run_rng.get(&f.time_us) {
-                Some(a) => ClientData {
+            .filter_map(|f| match run_rng.get(&f.time_us) {
+                Some(a) => Some(ClientData {
                     run_id: *a,
                     name: f.topic.clone(),
                     unit: f.unit,
                     values: f.values,
                     timestamp: DateTime::from_timestamp_micros(f.time_us as i64).unwrap(),
                     node: f.topic.split_once('/').unwrap_or_default().0.to_owned(),
-                },
-                None => ClientData {
-                    run_id: -1,
-                    name: f.topic.clone(),
-                    unit: f.unit,
-                    values: f.values,
-                    timestamp: DateTime::from_timestamp_micros(f.time_us as i64).unwrap(),
-                    node: f.topic.split_once('/').unwrap_or_default().0.to_owned(),
-                },
+                }),
+                None => None,
             })
             .collect();
         info!(
-            "Inserting {} points, {} of which have no run ID (-1).",
-            new_data.len(),
-            new_data
-                .iter()
-                .filter(|x| x.run_id == -1)
-                .collect::<Vec<_>>()
-                .len()
+            "Inserting {} points. {} points could not be assigned IDs.",
+            insertable_data.len(),
+            size_before_filter - insertable_data.len()
         );
-        if let Err(err) = batcher.send(new_data).await {
+        if let Err(err) = batcher.send(insertable_data).await {
             warn!("Error sending file insert data to batcher! {}", err);
         };
     }

diff --git a/scylla-server/src/models.rs b/scylla-server/src/models.rs
@@ -6,8 +6,8 @@ use serde::Serialize;
 #[diesel(table_name = crate::schema::data)]
 #[diesel(belongs_to(DataType, foreign_key = dataTypeName))]
 #[diesel(check_for_backend(diesel::pg::Pg))]
+#[diesel(primary_key(dataTypeName, time))]
 pub struct Data {
-    pub id: i32,
     pub values: Option<Vec<Option<f64>>>,
     pub dataTypeName: String,
     pub time: DateTime<Utc>,
@@ -26,6 +26,7 @@ pub struct DataType {
 
 #[derive(Queryable, Debug, Identifiable, Insertable, Selectable, Serialize, AsChangeset)]
 #[diesel(table_name = crate::schema::run)]
+#[diesel(primary_key(id))]
 #[diesel(check_for_backend(diesel::pg::Pg))]
 pub struct Run {
     pub id: i32,

diff --git a/scylla-server/src/schema.rs b/scylla-server/src/schema.rs
@@ -1,8 +1,7 @@
 // @generated automatically by Diesel CLI.
 
 diesel::table! {
-    data (id) {
-        id -> Int4,
+    data (time, dataTypeName) {
         values -> Nullable<Array<Nullable<Float8>>>,
         dataTypeName -> Text,
         time -> Timestamptz,