refactor: migrate models to postgres via pg-orm (#188)

.ameba.yml

@@ -0,0 +1,6 @@
+Lint/NotNil:
+  Enabled: false
+Style/PredicateName:
+  Enabled: false
+Style/ParenthesesAroundCondition:
+  Enabled: false

README.md

@@ -16,17 +16,14 @@ We use [RethinkDB](https://rethinkdb.com) to unify our database and event bus, g
 | Key                       | Description                                    | Default     |
 | ------------------------- | ---------------------------------------------- | ----------- |
 | `PLACE_MAX_VERSIONS`      | Number of versions to keep of versioned models | 20          |
-| `RETHINKDB_HOST`          | RethinkDB host                                 | "localhost" |
-| `RETHINKDB_PORT`          | RethinkDB port                                 | 28015       |
-| `RETHINKDB_DB`            | Database name                                  | "test"      |
-| `RETHINKDB_USER`          | Database user                                  | "admin"     |
-| `RETHINKDB_PASSWORD`      | Database password                              | ""          |
-| `RETHINKDB_TIMEOUT`       | Retry interval in seconds                      | 2           |
-| `RETHINKDB_RETRIES`       | Times to reattempt failed driver operations    | 10          |
-| `RETHINKDB_QUERY_RETRIES` | Times to reattempt failed queries              | 10          |
-| `RETHINKDB_LOCK_EXPIRE`   | Expiry on locks in seconds                     | 30          |
-| `RETHINKDB_LOCK_TIMEOUT`  | Timeout on retrying a lock in seconds          | 5           |
-
+| `PG_HOST`                 | Postgresql host                                | "localhost" |
+| `PG_PORT`                 | Postgresql port                                | 5432        |
+| `PG_DB`                   | Database name  or `PG_DATABASE`                | "test"      |
+| `PG_USER`                 | Database user                                  | "postgres"  |
+| `PG_PASSWORD`             | Database password                              | ""          |
+| `PG_QUERY`                | Query string, that can be used to configure pooling | ""     |
+| `PG_LOCK_TIMEOUT`         | Timeout on retrying Advisory lock in seconds   | 5           |
+| `PG_DATABASE_URL`         | Or provide a Database DSN                      |             |
 ## Contributing
 
 See [`CONTRIBUTING.md`](./CONTRIBUTING.md).

docker-compose.yml

@@ -1,12 +1,4 @@
 version: "3.7"
-
-# YAML Anchors
-
-x-rethinkdb-client-env: &rethinkdb-client-env
-  RETHINKDB_HOST: ${RETHINKDB_HOST:-rethink}
-  RETHINKDB_PORT: ${RETHINKDB_PORT:-28015}
-  RETHINKDB_DB: ${RETHINKDB_DB:-place_development}
-
 services:
   test:
     image: placeos/service-spec-runner:latest
@@ -18,12 +10,37 @@ services:
       - ${PWD}/shard.yml:/app/shard.yml.input
       - ${PWD}/coverage:/app/coverage
     depends_on:
-      - rethink
+      - migrator
+      - postgres
     environment:
       GITHUB_ACTION: ${GITHUB_ACTION:-}
-      <<: *rethinkdb-client-env
+      PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/model_dev}
+
+  postgres:
+    hostname: postgres
+    image: postgres:13-alpine
+    container_name: postgres
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 30s
+      timeout: 30s
+      retries: 3
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: model_dev
+    # volumes:
+    #   - ${PWD}/models_init.sql:/docker-entrypoint-initdb.d/models_init.sql
+    ports:
+      - "5432:5432"
 
-  rethink:
-    image: rethinkdb:${RETHINKDB_VERSION:-2.4}
-    restart: always
-    hostname: rethink
+  migrator:
+    build:
+      context: ${PWD}/migration
+    container_name: migrator
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      GITHUB_ACTION: ${GITHUB_ACTION:-}
+      PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/model_dev}

migration/.dockerignore

@@ -0,0 +1,2 @@
+bin
+lib

migration/.editorconfig

@@ -0,0 +1,9 @@
+root = true
+
+[*.cr]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true

migration/.gitignore

@@ -0,0 +1,5 @@
+/docs/
+/lib/
+/bin/
+/.shards/
+*.dwarf

migration/Dockerfile

@@ -0,0 +1,11 @@
+FROM placeos/crystal:latest
+
+WORKDIR /app
+
+COPY . /app
+
+ENV PATH /app/bin:$PATH
+
+RUN shards build
+
+ENTRYPOINT [ "/app/run.sh" ]

migration/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Ali Naqvi <[email protected]>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

migration/README.md

@@ -0,0 +1,3 @@
+#PlaceOS Models Migration Script
+
+This folder contains source code for scaffolding PlaceOS models schema and running migration.

migration/db/migrations/20221103144813_add_doorkeeper_table.sql

@@ -0,0 +1,92 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+-- Table for model Auth::OAuthApplications
+CREATE TABLE IF NOT EXISTS "oauth_applications"(
+    id bigint PRIMARY KEY,
+    name character varying NOT NULL,
+    uid character varying NOT NULL,
+    secret character varying NOT NULL,
+    redirect_uri text NOT NULL,
+    scopes character varying DEFAULT ''::character varying NOT NULL,
+    confidential boolean DEFAULT true NOT NULL,
+    owner_id text NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL
+);
+
+CREATE SEQUENCE public.oauth_applications_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.oauth_applications_id_seq OWNED BY "oauth_applications".id;
+ALTER TABLE ONLY "oauth_applications" ALTER COLUMN id SET DEFAULT nextval('public.oauth_applications_id_seq'::regclass);
+CREATE UNIQUE INDEX index_oauth_applications_on_uid ON "oauth_applications" USING btree (uid);
+
+
+CREATE TABLE IF NOT EXISTS "oauth_access_grants" (
+    id bigint PRIMARY KEY,
+    resource_owner_id TEXT NOT NULL,
+    application_id bigint NOT NULL,
+    token character varying NOT NULL,
+    expires_in integer NOT NULL,
+    redirect_uri text NOT NULL,
+    created_at timestamp(6) without time zone NOT NULL,
+    revoked_at timestamp(6) without time zone,
+    scopes character varying DEFAULT ''::character varying NOT NULL
+);
+
+CREATE SEQUENCE public.oauth_access_grants_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.oauth_access_grants_id_seq OWNED BY "oauth_access_grants".id;
+ALTER TABLE ONLY "oauth_access_grants" ALTER COLUMN id SET DEFAULT nextval('public.oauth_access_grants_id_seq'::regclass);
+CREATE INDEX index_oauth_access_grants_on_application_id ON "oauth_access_grants" USING btree (application_id);
+CREATE INDEX index_oauth_access_grants_on_resource_owner_id ON "oauth_access_grants" USING btree (resource_owner_id);
+CREATE UNIQUE INDEX index_oauth_access_grants_on_token ON "oauth_access_grants" USING btree (token);
+ALTER TABLE ONLY "oauth_access_grants"
+    ADD CONSTRAINT fk_oauth_access_grants_on_oauth_applications_id FOREIGN KEY (application_id) REFERENCES "oauth_applications"(id);
+
+
+CREATE TABLE "oauth_access_tokens" (
+    id bigint PRIMARY KEY,
+    resource_owner_id TEXT,
+    application_id bigint NOT NULL,
+    token character varying NOT NULL,
+    refresh_token character varying,
+    expires_in integer,
+    revoked_at timestamp(6) without time zone,
+    created_at timestamp(6) without time zone NOT NULL,
+    scopes character varying,
+    previous_refresh_token character varying DEFAULT ''::character varying NOT NULL
+);
+
+CREATE SEQUENCE public.oauth_access_tokens_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.oauth_access_tokens_id_seq OWNED BY "oauth_access_tokens".id;
+ALTER TABLE ONLY "oauth_access_tokens" ALTER COLUMN id SET DEFAULT nextval('public.oauth_access_tokens_id_seq'::regclass);
+CREATE INDEX index_oauth_access_tokens_on_application_id ON "oauth_access_tokens" USING btree (application_id);
+CREATE UNIQUE INDEX index_oauth_access_tokens_on_refresh_token ON "oauth_access_tokens" USING btree (refresh_token);
+CREATE INDEX index_oauth_access_tokens_on_resource_owner_id ON "oauth_access_tokens" USING btree (resource_owner_id);
+CREATE UNIQUE INDEX index_oauth_access_tokens_on_token ON "oauth_access_tokens" USING btree (token);
+ALTER TABLE ONLY "oauth_access_tokens"
+    ADD CONSTRAINT fk_oauth_access_tokens_on_oauth_applications_id FOREIGN KEY (application_id) REFERENCES "oauth_applications"(id);
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE IF EXISTS "oauth_applications"
+DROP TABLE IF EXISTS "oauth_access_grants"
+DROP TABLE IF EXISTS "oauth_access_tokens"

migration/db/migrations/20221103144834_add_condo_table.sql

@@ -0,0 +1,39 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+CREATE TABLE "condo_uploads" (
+    id bigint PRIMARY KEY,
+    user_id character varying,
+    file_name character varying,
+    file_size integer,
+    file_id character varying,
+    provider_namespace character varying,
+    provider_name character varying,
+    provider_location character varying,
+    bucket_name character varying,
+    object_key character varying,
+    object_options text,
+    resumable_id character varying,
+    resumable boolean DEFAULT false,
+    created_at timestamp(6) without time zone NOT NULL,
+    updated_at timestamp(6) without time zone NOT NULL,
+    file_path text,
+    part_list character varying,
+    part_data text
+);
+
+CREATE SEQUENCE public.condo_uploads_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE public.condo_uploads_id_seq OWNED BY "condo_uploads".id;
+ALTER TABLE ONLY "condo_uploads" ALTER COLUMN id SET DEFAULT nextval('public.condo_uploads_id_seq'::regclass);
+
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE IF EXISTS "condo_uploads"

migration/db/migrations/20221103181928_add_api_key_table.sql

@@ -0,0 +1,25 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+-- Table for model PlaceOS::Model::ApiKey
+CREATE TABLE IF NOT EXISTS "api_key"(
+     created_at TIMESTAMPTZ NOT NULL,
+   updated_at TIMESTAMPTZ NOT NULL,
+   name TEXT NOT NULL,
+   description TEXT NOT NULL,
+   scopes JSONB NOT NULL,
+   permissions INTEGER,
+   secret TEXT NOT NULL,
+   user_id TEXT,
+   authority_id TEXT,
+   id TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE INDEX IF NOT EXISTS api_key_user_id_index ON "api_key" USING BTREE (user_id);
+CREATE INDEX IF NOT EXISTS api_key_authority_id_index ON "api_key" USING BTREE (authority_id);
+CREATE INDEX IF NOT EXISTS api_key_name_index ON "api_key" USING BTREE (name);
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE IF EXISTS "api_key"

migration/db/migrations/20221103181946_add_asset_table.sql

@@ -0,0 +1,30 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+-- Table for model PlaceOS::Model::Asset
+CREATE TABLE IF NOT EXISTS "asset"(
+     created_at TIMESTAMPTZ NOT NULL,
+   updated_at TIMESTAMPTZ NOT NULL,
+   name TEXT NOT NULL,
+   category TEXT NOT NULL,
+   description TEXT NOT NULL,
+   purchase_date TIMESTAMPTZ NOT NULL,
+   good_until_date TIMESTAMPTZ,
+   identifier TEXT,
+   brand TEXT NOT NULL,
+   purchase_price INTEGER NOT NULL,
+   images TEXT[] NOT NULL,
+   invoice TEXT,
+   quantity INTEGER NOT NULL,
+   in_use INTEGER NOT NULL,
+   other_data JSONB NOT NULL,
+   parent_id TEXT,
+   id TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE INDEX IF NOT EXISTS asset_parent_id_index ON "asset" USING BTREE (parent_id);
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE IF EXISTS "asset"

migration/db/migrations/20221103181954_add_repo_table.sql

@@ -0,0 +1,27 @@
+-- +micrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+-- Table for model PlaceOS::Model::Repository
+CREATE TABLE IF NOT EXISTS "repo"(
+     created_at TIMESTAMPTZ NOT NULL,
+   updated_at TIMESTAMPTZ NOT NULL,
+   name TEXT NOT NULL,
+   description TEXT NOT NULL,
+   folder_name TEXT NOT NULL,
+   uri TEXT NOT NULL,
+   commit_hash TEXT NOT NULL,
+   branch TEXT NOT NULL,
+   deployed_commit_hash TEXT,
+   release BOOLEAN NOT NULL,
+   username TEXT,
+   password TEXT,
+   repo_type INTEGER NOT NULL,
+   id TEXT NOT NULL PRIMARY KEY
+);
+
+CREATE INDEX IF NOT EXISTS repo_folder_name_index ON "repo" USING BTREE (folder_name);
+
+-- +micrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE IF EXISTS "repo"