not null script, and minor adjustment to yarn scripts

charybdis/README.md

@@ -1,14 +1,18 @@
 # Charybdis
 
-To initialize the databases run `yarn database:setup`
+make sure you [install yarn](https://classic.yarnpkg.com/lang/en/docs/install/#mac-stable) if you don't have it already.
 
-You can also individually teardown and bringup the databases by running `yarn database:cloud:setup` or `yarn: database:local:setup` and `yarn database:cloud:teardown` or `yarn database:local:teardown`. 
+and install prisma globally... it's just easier that way(if you have trouble just follow this: [prisma install instructions](https://www.prisma.io/docs/orm/tools/prisma-cli)).
+
+To initialize the databases run `yarn database:setup` (if this command fails, you should run `yarn database:teardown` before running again.)
+
+You can also individually teardown and bringup the databases by running `yarn database:cloud:setup` or `yarn database:local:setup` and `yarn database:cloud:teardown` or `yarn database:local:teardown`.
 
 To teardown all the databases run `yarn database:teardown`
 
-When updating the local database schema, *Delete the existing migrations files in local-prisma* and then run `yarn prisma:migrate:local:dev`. Ensure to name the file `init`. This will create the migration file, copy it to scylla and rerun the diesel migration onto the database to ensure consistency between the two projects.
+When updating the local database schema, _Delete the existing migrations files in local-prisma_ and then run `yarn prisma:local:migrate:dev`. Ensure to name the file `init`. This will create the migration file, then run a script to enforce NOT NULL constraints that prisma gurantees put refuses to support in migration sql, copy it to scylla and rerun the diesel migration onto the database to ensure consistency between the two projects, it will also update the diesel schema.rs file according to the new migration (this should be checked to ensure it is based on the expected types, made in your prisma schema).
 
-You will have to manually add `NOT NULL` after the values Real[] since prisma and diesel interpret these values in different ways. 
+IMPORTANT: the above will fail if you have more than the two expect migration folder in /scylla-server/migrations : _00000000000000_diesel_initial_setup_, and the working migration folder: _2024-11-10-031516_create_all_ <--- this will we be where the diesel migration file that is getting updated.
 
 When updating the cloud schema, only run `yarn prisma:migrate:cloud:dev` and name it appropriately to whatever changes were made. Ensure that when deploying migrations to the cloud you only run `yarn prisma:migrate:cloud`. You will also have to change the cloud database url in your .env file to point to the cloud database
 

charybdis/package.json

@@ -4,15 +4,15 @@
   "description": "",
   "main": "index.ts",
   "scripts": {
-    "prisma:local:migrate:dev": "prisma migrate dev --schema ./local-prisma/schema.prisma && cp ./local-prisma/migrations/*_init/migration.sql ../scylla-server/migrations/*_create_all/up.sql && cd ../scylla-server && diesel database reset --database-url postgresql://postgres:docker@localhost:8000/charybdis-local && diesel database setup --database-url postgresql://postgres:docker@localhost:8000/charybdis-local",
+    "prisma:local:migrate:dev": "prisma migrate dev --schema ./local-prisma/schema.prisma && node scripts/prisma-sql-not-null-enforcment.js && cp ./local-prisma/migrations/*_init/migration.sql ../scylla-server/migrations/*_create_all/up.sql && cd ../scylla-server && diesel database reset --database-url postgresql://postgres:docker@localhost:8000/charybdis-local && diesel database setup --database-url postgresql://postgres:docker@localhost:8000/charybdis-local && diesel print-schema --database-url postgresql://postgres:docker@localhost:8000/charybdis-local > src/schema.rs",
     "prisma:cloud:migrate:dev": "prisma migrate dev --schema ./cloud-prisma/schema.prisma",
     "prisma:cloud:migrate": "prisma migrate deploy --schema ./cloud-prisma/schema.prisma",
     "prisma:local:migrate": "prisma migrate deploy --schema ./local-prisma/schema.prisma",
     "prisma:cloud:reset": "prisma migrate reset --schema ./cloud-prisma/schema.prisma",
     "prisma:local:reset": "prisma migrate reset --schema ./local-prisma/schema.prisma",
     "database:setup": "yarn database:setup:script && yarn database:local:setup && yarn database:cloud:setup",
     "database:local:setup": "docker run --name charybdis_local -e POSTGRES_PASSWORD=docker -p 8000:5432 -d postgres && sleep 5 && docker exec charybdis_local psql -U postgres -c \"CREATE DATABASE charybdis_local;\" && yarn prisma:local:reset",
-    "database:cloud:setup": "docker run --name charybdis_cloud -e POSTGRES_PASSWORD=docker -p 8001:5432 -d timescale/timescaledb:latest-pg17 && sleep 5 && yarn prisma:cloud:reset",
+    "database:cloud:setup": "docker run --name charybdis_cloud -e POSTGRES_PASSWORD=docker -p 8001:5432 -d timescale/timescaledb:latest-pg17 && sleep 5 && docker exec charybdis_cloud psql -U postgres -c \"CREATE DATABASE charybdis_cloud;\" && yarn prisma:cloud:migrate",
     "database:setup:script": "node scripts/database-setup.js",
     "database:cloud:teardown": "docker stop charybdis_cloud && docker rm charybdis_cloud",
     "database:local:teardown": "docker stop charybdis_local && docker rm charybdis_local",
@@ -23,7 +23,6 @@
   "license": "MIT",
   "devDependencies": {
     "@types/node": "^22.10.6",
-    "prisma": "^6.2.1",
     "tsx": "^4.19.2",
     "typescript": "^5.7.3"
   },

charybdis/scripts/prisma-sql-not-null-enforcment.js

@@ -0,0 +1,103 @@
+const fs = require("fs");
+const path = require("path");
+
+const prismaSchemaPath = "././local-prisma/schema.prisma"; // Path to schema.prisma
+const migrationsDir = "././local-prisma/migrations/"; // Path to migrations directory
+
+// Helper to parse schema.prisma and find fields that are required
+function parsePrismaSchema() {
+  const schemaContent = fs.readFileSync(prismaSchemaPath, "utf8");
+
+  const models = {};
+  const modelRegex = /model (\w+) {([\s\S]*?)}/g;
+  let match;
+
+  while ((match = modelRegex.exec(schemaContent)) !== null) {
+    const modelName = match[1];
+    const modelBody = match[2];
+
+    const fields = [];
+    const fieldRegex = /^\s*(\w+)\s+([\w\[\]]+)(.*?(@.+)?)$/gm;
+    let fieldMatch;
+
+    while ((fieldMatch = fieldRegex.exec(modelBody)) !== null) {
+      const [, fieldName, fieldType, modifiers] = fieldMatch;
+      const isRequired =
+        !modifiers.includes("?") && !modifiers.includes("@default");
+      const isArray = fieldType.includes("[]");
+      fields.push({ fieldName, fieldType, isRequired, isArray });
+    }
+
+    models[modelName] = fields;
+  }
+
+  return models;
+}
+
+// Modify the latest migration to add NOT NULL constraints for required array fields
+function modifyLatestMigration(models) {
+  const migrationDirs = fs.readdirSync(migrationsDir).filter((file) => {
+    const fullPath = path.join(migrationsDir, file);
+    return fs.statSync(fullPath).isDirectory();
+  });
+
+  if (migrationDirs.length === 0) {
+    console.error(
+      "No migrations found. Run `prisma migrate dev` to create a migration first."
+    );
+    process.exit(1);
+  }
+
+  const latestMigration = migrationDirs[migrationDirs.length - 1];
+  const migrationFile = path.join(
+    migrationsDir,
+    latestMigration,
+    "migration.sql"
+  );
+
+  if (!fs.existsSync(migrationFile)) {
+    console.error(`Migration file not found: ${migrationFile}`);
+    process.exit(1);
+  }
+
+  console.log(`Modifying migration: ${migrationFile}`);
+  let migrationSQL = fs.readFileSync(migrationFile, "utf8");
+
+  for (const [modelName, fields] of Object.entries(models)) {
+    fields
+      .filter(({ isRequired, isArray }) => isRequired && isArray)
+      .forEach(({ fieldName }) => {
+        // Match the specific table and column definition, ensuring correct scope
+        const tableRegex = new RegExp(
+          `CREATE TABLE "${modelName}" \\(([^)]*?)"${fieldName}"\\s+([^,]*)`, // Match the correct table and field
+          "s"
+        );
+
+        migrationSQL = migrationSQL.replace(
+          tableRegex,
+          (match, before, columnDef) => {
+            if (columnDef.includes("NOT NULL")) {
+              console.warn(
+                `Field ${modelName}.${fieldName} already has NOT NULL.`
+              );
+              return match;
+            }
+            console.log(
+              `Adding NOT NULL constraint to ${modelName}.${fieldName}`
+            );
+            return match.replace(columnDef, `${columnDef} NOT NULL`);
+          }
+        );
+      });
+  }
+
+  fs.writeFileSync(migrationFile, migrationSQL, "utf8");
+  console.log("Migration modified successfully.");
+}
+
+function main() {
+  const models = parsePrismaSchema();
+  modifyLatestMigration(models);
+}
+
+main();

scylla-server/integration_test.sh

@@ -10,7 +10,7 @@ docker rm -f odyssey-timescale 2>/dev/null || echo "No existing container to rem
 
 # Start a new odyssey-timescale container
 echo "Starting a new odyssey-timescale container..."
-docker-compose up -d odyssey-timescale || { echo "Failed to start odyssey-timescale"; exit 1; }
+docker compose up -d odyssey-timescale || { echo "Failed to start odyssey-timescale"; exit 1; }
 
 # Wait for the database to initialize
 echo "Waiting for the database to initialize..."