diff --git a/.ameba.yml b/.ameba.yml new file mode 100644 index 00000000..959982b7 --- /dev/null +++ b/.ameba.yml @@ -0,0 +1,6 @@ +Lint/NotNil: + Enabled: false +Style/PredicateName: + Enabled: false +Style/ParenthesesAroundCondition: + Enabled: false \ No newline at end of file diff --git a/README.md b/README.md index db6c3675..34b6c26e 100644 --- a/README.md +++ b/README.md @@ -16,17 +16,14 @@ We use [RethinkDB](https://rethinkdb.com) to unify our database and event bus, g | Key | Description | Default | | ------------------------- | ---------------------------------------------- | ----------- | | `PLACE_MAX_VERSIONS` | Number of versions to keep of versioned models | 20 | -| `RETHINKDB_HOST` | RethinkDB host | "localhost" | -| `RETHINKDB_PORT` | RethinkDB port | 28015 | -| `RETHINKDB_DB` | Database name | "test" | -| `RETHINKDB_USER` | Database user | "admin" | -| `RETHINKDB_PASSWORD` | Database password | "" | -| `RETHINKDB_TIMEOUT` | Retry interval in seconds | 2 | -| `RETHINKDB_RETRIES` | Times to reattempt failed driver operations | 10 | -| `RETHINKDB_QUERY_RETRIES` | Times to reattempt failed queries | 10 | -| `RETHINKDB_LOCK_EXPIRE` | Expiry on locks in seconds | 30 | -| `RETHINKDB_LOCK_TIMEOUT` | Timeout on retrying a lock in seconds | 5 | - +| `PG_HOST` | Postgresql host | "localhost" | +| `PG_PORT` | Postgresql port | 5432 | +| `PG_DB` | Database name or `PG_DATABASE` | "test" | +| `PG_USER` | Database user | "postgres" | +| `PG_PASSWORD` | Database password | "" | +| `PG_QUERY` | Query string, that can be used to configure pooling | "" | +| `PG_LOCK_TIMEOUT` | Timeout on retrying Advisory lock in seconds | 5 | +| `PG_DATABASE_URL` | Or provide a Database DSN | | ## Contributing See [`CONTRIBUTING.md`](./CONTRIBUTING.md). diff --git a/docker-compose.yml b/docker-compose.yml index 4befade9..120d8a23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,4 @@ version: "3.7" - -# YAML Anchors - -x-rethinkdb-client-env: &rethinkdb-client-env - RETHINKDB_HOST: ${RETHINKDB_HOST:-rethink} - RETHINKDB_PORT: ${RETHINKDB_PORT:-28015} - RETHINKDB_DB: ${RETHINKDB_DB:-place_development} - services: test: image: placeos/service-spec-runner:latest @@ -18,12 +10,37 @@ services: - ${PWD}/shard.yml:/app/shard.yml.input - ${PWD}/coverage:/app/coverage depends_on: - - rethink + - migrator + - postgres environment: GITHUB_ACTION: ${GITHUB_ACTION:-} - <<: *rethinkdb-client-env + PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/model_dev} + + postgres: + hostname: postgres + image: postgres:13-alpine + container_name: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 30s + timeout: 30s + retries: 3 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: model_dev + # volumes: + # - ${PWD}/models_init.sql:/docker-entrypoint-initdb.d/models_init.sql + ports: + - "5432:5432" - rethink: - image: rethinkdb:${RETHINKDB_VERSION:-2.4} - restart: always - hostname: rethink + migrator: + build: + context: ${PWD}/migration + container_name: migrator + depends_on: + postgres: + condition: service_healthy + environment: + GITHUB_ACTION: ${GITHUB_ACTION:-} + PG_DATABASE_URL: ${PG_DATABASE_URL:-postgresql://postgres:password@postgres:5432/model_dev} diff --git a/migration/.dockerignore b/migration/.dockerignore new file mode 100644 index 00000000..00e03f59 --- /dev/null +++ b/migration/.dockerignore @@ -0,0 +1,2 @@ +bin +lib \ No newline at end of file diff --git a/migration/.editorconfig b/migration/.editorconfig new file mode 100644 index 00000000..163eb75c --- /dev/null +++ b/migration/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*.cr] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/migration/.gitignore b/migration/.gitignore new file mode 100644 index 00000000..0bb75ea0 --- /dev/null +++ b/migration/.gitignore @@ -0,0 +1,5 @@ +/docs/ +/lib/ +/bin/ +/.shards/ +*.dwarf diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 00000000..a8e93eb2 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,11 @@ +FROM placeos/crystal:latest + +WORKDIR /app + +COPY . /app + +ENV PATH /app/bin:$PATH + +RUN shards build + +ENTRYPOINT [ "/app/run.sh" ] \ No newline at end of file diff --git a/migration/LICENSE b/migration/LICENSE new file mode 100644 index 00000000..d57fad66 --- /dev/null +++ b/migration/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Ali Naqvi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 00000000..c43d139e --- /dev/null +++ b/migration/README.md @@ -0,0 +1,3 @@ +#PlaceOS Models Migration Script + +This folder contains source code for scaffolding PlaceOS models schema and running migration. diff --git a/migration/db/migrations/20221103144813_add_doorkeeper_table.sql b/migration/db/migrations/20221103144813_add_doorkeeper_table.sql new file mode 100644 index 00000000..8392eff0 --- /dev/null +++ b/migration/db/migrations/20221103144813_add_doorkeeper_table.sql @@ -0,0 +1,92 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model Auth::OAuthApplications +CREATE TABLE IF NOT EXISTS "oauth_applications"( + id bigint PRIMARY KEY, + name character varying NOT NULL, + uid character varying NOT NULL, + secret character varying NOT NULL, + redirect_uri text NOT NULL, + scopes character varying DEFAULT ''::character varying NOT NULL, + confidential boolean DEFAULT true NOT NULL, + owner_id text NOT NULL, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + +CREATE SEQUENCE public.oauth_applications_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.oauth_applications_id_seq OWNED BY "oauth_applications".id; +ALTER TABLE ONLY "oauth_applications" ALTER COLUMN id SET DEFAULT nextval('public.oauth_applications_id_seq'::regclass); +CREATE UNIQUE INDEX index_oauth_applications_on_uid ON "oauth_applications" USING btree (uid); + + +CREATE TABLE IF NOT EXISTS "oauth_access_grants" ( + id bigint PRIMARY KEY, + resource_owner_id TEXT NOT NULL, + application_id bigint NOT NULL, + token character varying NOT NULL, + expires_in integer NOT NULL, + redirect_uri text NOT NULL, + created_at timestamp(6) without time zone NOT NULL, + revoked_at timestamp(6) without time zone, + scopes character varying DEFAULT ''::character varying NOT NULL +); + +CREATE SEQUENCE public.oauth_access_grants_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.oauth_access_grants_id_seq OWNED BY "oauth_access_grants".id; +ALTER TABLE ONLY "oauth_access_grants" ALTER COLUMN id SET DEFAULT nextval('public.oauth_access_grants_id_seq'::regclass); +CREATE INDEX index_oauth_access_grants_on_application_id ON "oauth_access_grants" USING btree (application_id); +CREATE INDEX index_oauth_access_grants_on_resource_owner_id ON "oauth_access_grants" USING btree (resource_owner_id); +CREATE UNIQUE INDEX index_oauth_access_grants_on_token ON "oauth_access_grants" USING btree (token); +ALTER TABLE ONLY "oauth_access_grants" + ADD CONSTRAINT fk_oauth_access_grants_on_oauth_applications_id FOREIGN KEY (application_id) REFERENCES "oauth_applications"(id); + + +CREATE TABLE "oauth_access_tokens" ( + id bigint PRIMARY KEY, + resource_owner_id TEXT, + application_id bigint NOT NULL, + token character varying NOT NULL, + refresh_token character varying, + expires_in integer, + revoked_at timestamp(6) without time zone, + created_at timestamp(6) without time zone NOT NULL, + scopes character varying, + previous_refresh_token character varying DEFAULT ''::character varying NOT NULL +); + +CREATE SEQUENCE public.oauth_access_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.oauth_access_tokens_id_seq OWNED BY "oauth_access_tokens".id; +ALTER TABLE ONLY "oauth_access_tokens" ALTER COLUMN id SET DEFAULT nextval('public.oauth_access_tokens_id_seq'::regclass); +CREATE INDEX index_oauth_access_tokens_on_application_id ON "oauth_access_tokens" USING btree (application_id); +CREATE UNIQUE INDEX index_oauth_access_tokens_on_refresh_token ON "oauth_access_tokens" USING btree (refresh_token); +CREATE INDEX index_oauth_access_tokens_on_resource_owner_id ON "oauth_access_tokens" USING btree (resource_owner_id); +CREATE UNIQUE INDEX index_oauth_access_tokens_on_token ON "oauth_access_tokens" USING btree (token); +ALTER TABLE ONLY "oauth_access_tokens" + ADD CONSTRAINT fk_oauth_access_tokens_on_oauth_applications_id FOREIGN KEY (application_id) REFERENCES "oauth_applications"(id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "oauth_applications" +DROP TABLE IF EXISTS "oauth_access_grants" +DROP TABLE IF EXISTS "oauth_access_tokens" \ No newline at end of file diff --git a/migration/db/migrations/20221103144834_add_condo_table.sql b/migration/db/migrations/20221103144834_add_condo_table.sql new file mode 100644 index 00000000..29e07816 --- /dev/null +++ b/migration/db/migrations/20221103144834_add_condo_table.sql @@ -0,0 +1,39 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +CREATE TABLE "condo_uploads" ( + id bigint PRIMARY KEY, + user_id character varying, + file_name character varying, + file_size integer, + file_id character varying, + provider_namespace character varying, + provider_name character varying, + provider_location character varying, + bucket_name character varying, + object_key character varying, + object_options text, + resumable_id character varying, + resumable boolean DEFAULT false, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + file_path text, + part_list character varying, + part_data text +); + +CREATE SEQUENCE public.condo_uploads_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE public.condo_uploads_id_seq OWNED BY "condo_uploads".id; +ALTER TABLE ONLY "condo_uploads" ALTER COLUMN id SET DEFAULT nextval('public.condo_uploads_id_seq'::regclass); + + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "condo_uploads" \ No newline at end of file diff --git a/migration/db/migrations/20221103181928_add_api_key_table.sql b/migration/db/migrations/20221103181928_add_api_key_table.sql new file mode 100644 index 00000000..00b5209c --- /dev/null +++ b/migration/db/migrations/20221103181928_add_api_key_table.sql @@ -0,0 +1,25 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::ApiKey +CREATE TABLE IF NOT EXISTS "api_key"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + scopes JSONB NOT NULL, + permissions INTEGER, + secret TEXT NOT NULL, + user_id TEXT, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS api_key_user_id_index ON "api_key" USING BTREE (user_id); +CREATE INDEX IF NOT EXISTS api_key_authority_id_index ON "api_key" USING BTREE (authority_id); +CREATE INDEX IF NOT EXISTS api_key_name_index ON "api_key" USING BTREE (name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "api_key" \ No newline at end of file diff --git a/migration/db/migrations/20221103181946_add_asset_table.sql b/migration/db/migrations/20221103181946_add_asset_table.sql new file mode 100644 index 00000000..b8149cc9 --- /dev/null +++ b/migration/db/migrations/20221103181946_add_asset_table.sql @@ -0,0 +1,30 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Asset +CREATE TABLE IF NOT EXISTS "asset"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + category TEXT NOT NULL, + description TEXT NOT NULL, + purchase_date TIMESTAMPTZ NOT NULL, + good_until_date TIMESTAMPTZ, + identifier TEXT, + brand TEXT NOT NULL, + purchase_price INTEGER NOT NULL, + images TEXT[] NOT NULL, + invoice TEXT, + quantity INTEGER NOT NULL, + in_use INTEGER NOT NULL, + other_data JSONB NOT NULL, + parent_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS asset_parent_id_index ON "asset" USING BTREE (parent_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "asset" \ No newline at end of file diff --git a/migration/db/migrations/20221103181954_add_repo_table.sql b/migration/db/migrations/20221103181954_add_repo_table.sql new file mode 100644 index 00000000..203dd8be --- /dev/null +++ b/migration/db/migrations/20221103181954_add_repo_table.sql @@ -0,0 +1,27 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Repository +CREATE TABLE IF NOT EXISTS "repo"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + folder_name TEXT NOT NULL, + uri TEXT NOT NULL, + commit_hash TEXT NOT NULL, + branch TEXT NOT NULL, + deployed_commit_hash TEXT, + release BOOLEAN NOT NULL, + username TEXT, + password TEXT, + repo_type INTEGER NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS repo_folder_name_index ON "repo" USING BTREE (folder_name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "repo" \ No newline at end of file diff --git a/migration/db/migrations/20221103182003_add_driver_table.sql b/migration/db/migrations/20221103182003_add_driver_table.sql new file mode 100644 index 00000000..efc944fc --- /dev/null +++ b/migration/db/migrations/20221103182003_add_driver_table.sql @@ -0,0 +1,28 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Driver +CREATE TABLE IF NOT EXISTS "driver"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + json_schema JSONB NOT NULL, + default_uri TEXT, + default_port INTEGER, + role INTEGER NOT NULL, + file_name TEXT NOT NULL, + commit TEXT NOT NULL, + compilation_output TEXT, + module_name TEXT NOT NULL, + ignore_connected BOOLEAN NOT NULL, + repository_id TEXT NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS driver_repository_id_index ON "driver" USING BTREE (repository_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "driver" \ No newline at end of file diff --git a/migration/db/migrations/20221103182008_add_edge_table.sql b/migration/db/migrations/20221103182008_add_edge_table.sql new file mode 100644 index 00000000..22222c63 --- /dev/null +++ b/migration/db/migrations/20221103182008_add_edge_table.sql @@ -0,0 +1,21 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Edge +CREATE TABLE IF NOT EXISTS "edge"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + api_key_id TEXT NOT NULL, + user_id TEXT NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS edge_api_key_id_index ON "edge" USING BTREE (api_key_id); +CREATE INDEX IF NOT EXISTS edge_name_index ON "edge" USING BTREE (name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "edge" \ No newline at end of file diff --git a/migration/db/migrations/20221103182015_add_mod_table.sql b/migration/db/migrations/20221103182015_add_mod_table.sql new file mode 100644 index 00000000..3c72dccb --- /dev/null +++ b/migration/db/migrations/20221103182015_add_mod_table.sql @@ -0,0 +1,35 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Module +CREATE TABLE IF NOT EXISTS "mod"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + ip TEXT NOT NULL, + port INTEGER NOT NULL, + tls BOOLEAN NOT NULL, + udp BOOLEAN NOT NULL, + makebreak BOOLEAN NOT NULL, + uri TEXT NOT NULL, + name TEXT NOT NULL, + custom_name TEXT, + role INTEGER NOT NULL, + connected BOOLEAN NOT NULL, + running BOOLEAN NOT NULL, + notes TEXT NOT NULL, + ignore_connected BOOLEAN NOT NULL, + ignore_startstop BOOLEAN NOT NULL, + control_system_id TEXT, + driver_id TEXT NOT NULL, + edge_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS mod_control_system_id_index ON "mod" USING BTREE (control_system_id); +CREATE INDEX IF NOT EXISTS mod_driver_id_index ON "mod" USING BTREE (driver_id); +CREATE INDEX IF NOT EXISTS mod_edge_id_index ON "mod" USING BTREE (edge_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "mod" \ No newline at end of file diff --git a/migration/db/migrations/20221103182025_add_zone_table.sql b/migration/db/migrations/20221103182025_add_zone_table.sql new file mode 100644 index 00000000..e7faa2c9 --- /dev/null +++ b/migration/db/migrations/20221103182025_add_zone_table.sql @@ -0,0 +1,31 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Zone +CREATE TABLE IF NOT EXISTS "zone"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + tags TEXT[] NOT NULL, + location TEXT, + display_name TEXT, + code TEXT, + type TEXT, + count INTEGER NOT NULL, + capacity INTEGER NOT NULL, + map_id TEXT, + timezone TEXT, + triggers TEXT[] NOT NULL, + images TEXT[] NOT NULL, + parent_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS zone_parent_id_index ON "zone" USING BTREE (parent_id); +CREATE INDEX IF NOT EXISTS zone_name_index ON "zone" USING BTREE (name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "zone" \ No newline at end of file diff --git a/migration/db/migrations/20221103182030_add_sets_table.sql b/migration/db/migrations/20221103182030_add_sets_table.sql new file mode 100644 index 00000000..494a0fab --- /dev/null +++ b/migration/db/migrations/20221103182030_add_sets_table.sql @@ -0,0 +1,24 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Settings +CREATE TABLE IF NOT EXISTS "sets"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + modified_by_id TEXT, + settings_id TEXT, + encryption_level INTEGER NOT NULL, + settings_string TEXT NOT NULL, + keys TEXT[] NOT NULL, + parent_type INTEGER NOT NULL, + parent_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS sets_settings_id_index ON "sets" USING BTREE (settings_id); +CREATE INDEX IF NOT EXISTS sets_parent_id_index ON "sets" USING BTREE (parent_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "sets" \ No newline at end of file diff --git a/migration/db/migrations/20221103182036_add_sys_table.sql b/migration/db/migrations/20221103182036_add_sys_table.sql new file mode 100644 index 00000000..614d742f --- /dev/null +++ b/migration/db/migrations/20221103182036_add_sys_table.sql @@ -0,0 +1,34 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::ControlSystem +CREATE TABLE IF NOT EXISTS "sys"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + features TEXT[] NOT NULL, + email TEXT, + bookable BOOLEAN NOT NULL, + display_name TEXT, + code TEXT, + type TEXT, + capacity INTEGER NOT NULL, + map_id TEXT, + images TEXT[] NOT NULL, + timezone TEXT, + support_url TEXT NOT NULL, + version INTEGER NOT NULL, + installed_ui_devices INTEGER NOT NULL, + zones TEXT[] NOT NULL, + modules TEXT[] NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS sys_email_index ON "sys" USING BTREE (email); +CREATE INDEX IF NOT EXISTS sys_name_index ON "sys" USING BTREE (name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "sys" \ No newline at end of file diff --git a/migration/db/migrations/20221103182045_add_metadata_table.sql b/migration/db/migrations/20221103182045_add_metadata_table.sql new file mode 100644 index 00000000..5001e529 --- /dev/null +++ b/migration/db/migrations/20221103182045_add_metadata_table.sql @@ -0,0 +1,27 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Metadata +CREATE TABLE IF NOT EXISTS "metadata"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + modified_by_id TEXT, + metadata_id TEXT, + name TEXT NOT NULL, + description TEXT NOT NULL, + details JSONB NOT NULL, + editors TEXT[] NOT NULL, + parent_id TEXT, + schema_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS metadata_metadata_id_index ON "metadata" USING BTREE (metadata_id); +CREATE INDEX IF NOT EXISTS metadata_parent_id_index ON "metadata" USING BTREE (parent_id); +CREATE INDEX IF NOT EXISTS metadata_name_index ON "metadata" USING BTREE (name); +CREATE INDEX IF NOT EXISTS metadata_schema_id_index ON "metadata" USING BTREE (schema_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "metadata" \ No newline at end of file diff --git a/migration/db/migrations/20221103182052_add_user_table.sql b/migration/db/migrations/20221103182052_add_user_table.sql new file mode 100644 index 00000000..aacf0bd2 --- /dev/null +++ b/migration/db/migrations/20221103182052_add_user_table.sql @@ -0,0 +1,48 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::User +CREATE TABLE IF NOT EXISTS "user"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + nickname TEXT NOT NULL, + email TEXT NOT NULL, + phone TEXT NOT NULL, + country TEXT NOT NULL, + image TEXT NOT NULL, + ui_theme TEXT NOT NULL, + misc TEXT NOT NULL, + login_name TEXT, + staff_id TEXT, + first_name TEXT, + last_name TEXT, + building TEXT, + department TEXT, + preferred_language TEXT, + password_digest TEXT, + email_digest TEXT, + card_number TEXT, + deleted BOOLEAN NOT NULL, + groups TEXT[] NOT NULL, + access_token TEXT, + refresh_token TEXT, + expires_at BIGINT, + expires BOOLEAN NOT NULL, + password TEXT, + sys_admin BOOLEAN NOT NULL, + support BOOLEAN NOT NULL, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS user_authority_id_index ON "user" USING BTREE (authority_id); +CREATE INDEX IF NOT EXISTS user_email_digest_index ON "user" USING BTREE (email_digest); +CREATE INDEX IF NOT EXISTS user_login_name_index ON "user" USING BTREE (login_name); +CREATE INDEX IF NOT EXISTS user_staff_id_index ON "user" USING BTREE (staff_id); +CREATE INDEX IF NOT EXISTS user_sys_admin_index ON "user" USING BTREE (sys_admin); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "user" \ No newline at end of file diff --git a/migration/db/migrations/20221103182102_add_ldap_strat_table.sql b/migration/db/migrations/20221103182102_add_ldap_strat_table.sql new file mode 100644 index 00000000..358f7d63 --- /dev/null +++ b/migration/db/migrations/20221103182102_add_ldap_strat_table.sql @@ -0,0 +1,26 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::LdapAuthentication +CREATE TABLE IF NOT EXISTS "ldap_strat"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + port INTEGER NOT NULL, + auth_method TEXT NOT NULL, + uid TEXT NOT NULL, + host TEXT NOT NULL, + base TEXT NOT NULL, + bind_dn TEXT, + password TEXT, + filter TEXT, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS ldap_strat_authority_id_index ON "ldap_strat" USING BTREE (authority_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "ldap_strat" \ No newline at end of file diff --git a/migration/db/migrations/20221103182109_add_oauth_strat_table.sql b/migration/db/migrations/20221103182109_add_oauth_strat_table.sql new file mode 100644 index 00000000..f9a16bc5 --- /dev/null +++ b/migration/db/migrations/20221103182109_add_oauth_strat_table.sql @@ -0,0 +1,30 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::OAuthAuthentication +CREATE TABLE IF NOT EXISTS "oauth_strat"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + client_id TEXT NOT NULL, + client_secret TEXT NOT NULL, + info_mappings JSONB NOT NULL, + authorize_params JSONB NOT NULL, + ensure_matching JSONB NOT NULL, + site TEXT NOT NULL, + authorize_url TEXT NOT NULL, + token_method TEXT NOT NULL, + auth_scheme TEXT NOT NULL, + token_url TEXT NOT NULL, + scope TEXT NOT NULL, + raw_info_url TEXT, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS oauth_strat_authority_id_index ON "oauth_strat" USING BTREE (authority_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "oauth_strat" \ No newline at end of file diff --git a/migration/db/migrations/20221103182121_add_adfs_strat_table.sql b/migration/db/migrations/20221103182121_add_adfs_strat_table.sql new file mode 100644 index 00000000..835bc23b --- /dev/null +++ b/migration/db/migrations/20221103182121_add_adfs_strat_table.sql @@ -0,0 +1,31 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::SamlAuthentication +CREATE TABLE IF NOT EXISTS "adfs_strat"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + issuer TEXT NOT NULL, + idp_sso_target_url_runtime_params JSONB NOT NULL, + name_identifier_format TEXT NOT NULL, + uid_attribute TEXT, + assertion_consumer_service_url TEXT NOT NULL, + idp_sso_target_url TEXT NOT NULL, + idp_cert TEXT, + idp_cert_fingerprint TEXT, + attribute_service_name TEXT, + attribute_statements JSONB NOT NULL, + request_attributes JSONB NOT NULL, + idp_slo_target_url TEXT, + slo_default_relay_state TEXT, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS adfs_strat_authority_id_index ON "adfs_strat" USING BTREE (authority_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "adfs_strat" \ No newline at end of file diff --git a/migration/db/migrations/20221103182131_add_authority_table.sql b/migration/db/migrations/20221103182131_add_authority_table.sql new file mode 100644 index 00000000..8ea21a78 --- /dev/null +++ b/migration/db/migrations/20221103182131_add_authority_table.sql @@ -0,0 +1,23 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Authority +CREATE TABLE IF NOT EXISTS "authority"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + domain TEXT NOT NULL, + login_url TEXT NOT NULL, + logout_url TEXT NOT NULL, + internals JSONB NOT NULL, + config JSONB NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS authority_domain_index ON "authority" USING BTREE (domain); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "authority" \ No newline at end of file diff --git a/migration/db/migrations/20221103182136_add_ass_table.sql b/migration/db/migrations/20221103182136_add_ass_table.sql new file mode 100644 index 00000000..5370e305 --- /dev/null +++ b/migration/db/migrations/20221103182136_add_ass_table.sql @@ -0,0 +1,26 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::AssetInstance +CREATE TABLE IF NOT EXISTS "ass"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + tracking INTEGER NOT NULL, + approval BOOLEAN NOT NULL, + asset_id TEXT, + requester_id TEXT, + zone_id TEXT, + usage_start TIMESTAMPTZ NOT NULL, + usage_end TIMESTAMPTZ NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS ass_asset_id_index ON "ass" USING BTREE (asset_id); +CREATE INDEX IF NOT EXISTS ass_zone_id_index ON "ass" USING BTREE (zone_id); +CREATE INDEX IF NOT EXISTS ass_requester_id_index ON "ass" USING BTREE (requester_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "ass" \ No newline at end of file diff --git a/migration/db/migrations/20221103182144_add_broker_table.sql b/migration/db/migrations/20221103182144_add_broker_table.sql new file mode 100644 index 00000000..5aceb7b4 --- /dev/null +++ b/migration/db/migrations/20221103182144_add_broker_table.sql @@ -0,0 +1,27 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Broker +CREATE TABLE IF NOT EXISTS "broker"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + auth_type INTEGER NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + host TEXT NOT NULL, + port INTEGER NOT NULL, + tls BOOLEAN NOT NULL, + username TEXT, + password TEXT, + certificate TEXT, + secret TEXT NOT NULL, + filters TEXT[] NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS broker_name_index ON "broker" USING BTREE (name); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP TABLE IF EXISTS "broker" \ No newline at end of file diff --git a/migration/db/migrations/20221103182204_add_json_schema_table.sql b/migration/db/migrations/20221103182204_add_json_schema_table.sql new file mode 100644 index 00000000..9008f330 --- /dev/null +++ b/migration/db/migrations/20221103182204_add_json_schema_table.sql @@ -0,0 +1,17 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::JsonSchema +CREATE TABLE IF NOT EXISTS "json_schema"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + schema JSONB NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE IF EXISTS "json_schema" \ No newline at end of file diff --git a/migration/db/migrations/20221103182211_add_trigger_table.sql b/migration/db/migrations/20221103182211_add_trigger_table.sql new file mode 100644 index 00000000..444686d0 --- /dev/null +++ b/migration/db/migrations/20221103182211_add_trigger_table.sql @@ -0,0 +1,24 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Trigger +CREATE TABLE IF NOT EXISTS "trigger"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + name TEXT NOT NULL, + description TEXT NOT NULL, + actions JSONB NOT NULL, + conditions JSONB NOT NULL, + debounce_period INTEGER NOT NULL, + important BOOLEAN NOT NULL, + enable_webhook BOOLEAN NOT NULL, + supported_methods TEXT[] NOT NULL, + control_system_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS trigger_control_system_id_index ON "trigger" USING BTREE (control_system_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE IF EXISTS "trigger" \ No newline at end of file diff --git a/migration/db/migrations/20221103182218_add_trig_table.sql b/migration/db/migrations/20221103182218_add_trig_table.sql new file mode 100644 index 00000000..96123bc3 --- /dev/null +++ b/migration/db/migrations/20221103182218_add_trig_table.sql @@ -0,0 +1,25 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied +-- Table for model PlaceOS::Model::TriggerInstance +CREATE TABLE IF NOT EXISTS "trig"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + enabled BOOLEAN NOT NULL, + triggered BOOLEAN NOT NULL, + important BOOLEAN NOT NULL, + exec_enabled BOOLEAN NOT NULL, + webhook_secret TEXT NOT NULL, + trigger_count INTEGER NOT NULL, + control_system_id TEXT, + trigger_id TEXT, + zone_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS trig_control_system_id_index ON "trig" USING BTREE (control_system_id); +CREATE INDEX IF NOT EXISTS trig_trigger_id_index ON "trig" USING BTREE (trigger_id); +CREATE INDEX IF NOT EXISTS trig_zone_id_index ON "trig" USING BTREE (zone_id); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE IF EXISTS "trig" \ No newline at end of file diff --git a/migration/db/migrations/20221103182226_add_authentication_table.sql b/migration/db/migrations/20221103182226_add_authentication_table.sql new file mode 100644 index 00000000..a4730783 --- /dev/null +++ b/migration/db/migrations/20221103182226_add_authentication_table.sql @@ -0,0 +1,20 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied +-- Table for model PlaceOS::Model::UserAuthLookup +CREATE TABLE IF NOT EXISTS "authentication"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + uid TEXT NOT NULL, + provider TEXT NOT NULL, + user_id TEXT, + authority_id TEXT, + id TEXT NOT NULL PRIMARY KEY +); + +CREATE INDEX IF NOT EXISTS authentication_user_id_index ON "authentication" USING BTREE (user_id); +CREATE INDEX IF NOT EXISTS authentication_authority_id_index ON "authentication" USING BTREE (authority_id); + + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE IF EXISTS "authentication" \ No newline at end of file diff --git a/migration/db/migrations/20221103182518_add_stats_table.sql b/migration/db/migrations/20221103182518_add_stats_table.sql new file mode 100644 index 00000000..3e284d48 --- /dev/null +++ b/migration/db/migrations/20221103182518_add_stats_table.sql @@ -0,0 +1,19 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- Table for model PlaceOS::Model::Statistics +CREATE TABLE IF NOT EXISTS "stats"( + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + modules_disconnected INTEGER NOT NULL, + triggers_active INTEGER NOT NULL, + websocket_connections INTEGER NOT NULL, + fixed_connections INTEGER NOT NULL, + core_nodes_online INTEGER NOT NULL, + ttl BIGINT NOT NULL, + id TEXT NOT NULL PRIMARY KEY +); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +DROP TABLE IF EXISTS "stats" \ No newline at end of file diff --git a/migration/db/migrations/20230224115858_add_sys_booking_feat.sql b/migration/db/migrations/20230224115858_add_sys_booking_feat.sql new file mode 100644 index 00000000..f000b56f --- /dev/null +++ b/migration/db/migrations/20230224115858_add_sys_booking_feat.sql @@ -0,0 +1,9 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +ALTER TABLE "sys" ADD COLUMN public BOOLEAN NOT NULL DEFAULT false; + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +ALTER TABLE "sys" DROP COLUMN public; diff --git a/migration/db/migrations/20230314100503_alter_user_table.sql b/migration/db/migrations/20230314100503_alter_user_table.sql new file mode 100644 index 00000000..5477adc1 --- /dev/null +++ b/migration/db/migrations/20230314100503_alter_user_table.sql @@ -0,0 +1,18 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +alter table "user" alter COLUMN nickname drop NOT NULL; +alter table "user" alter COLUMN phone drop NOT NULL; +alter table "user" alter COLUMN country drop NOT NULL; +alter table "user" alter COLUMN image drop NOT NULL; +alter table "user" alter COLUMN ui_theme drop NOT NULL; +alter table "user" alter COLUMN misc drop NOT NULL; + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back +alter table "user" alter COLUMN nickname set NOT NULL; +alter table "user" alter COLUMN phone set NOT NULL; +alter table "user" alter COLUMN country set NOT NULL; +alter table "user" alter COLUMN image set NOT NULL; +alter table "user" alter COLUMN ui_theme set NOT NULL; +alter table "user" alter COLUMN misc set NOT NULL; \ No newline at end of file diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 00000000..5b76a708 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eu + +bin/migration up \ No newline at end of file diff --git a/migration/shard.yml b/migration/shard.yml new file mode 100644 index 00000000..ae3873b3 --- /dev/null +++ b/migration/shard.yml @@ -0,0 +1,16 @@ +name: placeos-models-migration +version: 0.9.9 +crystal: ~> 1.4 +license: MIT + + +targets: + migration: + main: src/migration.cr + +dependencies: + micrate: + github: amberframework/micrate + version: "0.12.0" + pg: + github: will/crystal-pg diff --git a/migration/src/migration.cr b/migration/src/migration.cr new file mode 100644 index 00000000..d59fe708 --- /dev/null +++ b/migration/src/migration.cr @@ -0,0 +1,5 @@ +require "micrate" +require "pg" + +Micrate::DB.connection_url = ENV["PG_DATABASE_URL"] +Micrate::Cli.run diff --git a/shard.override.yml b/shard.override.yml index 8202e4e2..9d77f637 100644 --- a/shard.override.yml +++ b/shard.override.yml @@ -1,8 +1,8 @@ dependencies: neuroplastic: github: place-labs/neuroplastic + commit: 29b6a363e101ebdc88e28755f872a200a4c800e8 retriable: github: Sija/retriable.cr - rethinkdb-orm: - github: spider-gazelle/rethinkdb-orm - version: ">= 6.0" + pg-orm: + github: spider-gazelle/pg-orm \ No newline at end of file diff --git a/shard.yml b/shard.yml index b59e1213..42fd4f71 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: placeos-models -version: 8.13.4 +version: 8.13.2 crystal: ~> 1.4 dependencies: @@ -47,10 +47,9 @@ dependencies: github: place-labs/log-backend version: ">= 0.11.0" - # ORM for RethinkDB built on active-model - rethinkdb-orm: - github: spider-gazelle/rethinkdb-orm - version: ~> 6.0 + # ORM for Postgres built on active-model + pg-orm: + github: spider-gazelle/pg-orm development_dependencies: ameba: diff --git a/spec/api_key_spec.cr b/spec/api_key_spec.cr index ae1092e5..a6320bdf 100644 --- a/spec/api_key_spec.cr +++ b/spec/api_key_spec.cr @@ -1,6 +1,9 @@ require "./helper" module PlaceOS::Model + Spec.before_each do + Authority.clear + end describe ApiKey do test_round_trip(ApiKey) @@ -73,7 +76,7 @@ module PlaceOS::Model found = ApiKey.find_key! api_key found.id.should eq(key.id) - expect_raises(RethinkORM::Error::DocumentNotFound) do + expect_raises(PgORM::Error::RecordNotFound) do fake_id = "#{key.id}.notamatch" ApiKey.find_key! fake_id end @@ -123,7 +126,7 @@ module PlaceOS::Model ApiKey.find!(id).id.should eq key.id key.user.not_nil!.destroy - ApiKey.find(id).should be_nil + ApiKey.find?(id).should be_nil end end end diff --git a/spec/asset_instance_spec.cr b/spec/asset_instance_spec.cr index 25f095c5..c53f01c9 100644 --- a/spec/asset_instance_spec.cr +++ b/spec/asset_instance_spec.cr @@ -13,7 +13,7 @@ module PlaceOS::Model end it "prevents an AssetInstance from ending before it starts" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do inst = Generator.asset_instance.save! inst.usage_end = Time.local - 1.hour inst.save! diff --git a/spec/asset_spec.cr b/spec/asset_spec.cr index 9ab0d60d..8a718466 100644 --- a/spec/asset_spec.cr +++ b/spec/asset_spec.cr @@ -14,7 +14,7 @@ module PlaceOS::Model describe "validations" do it "in_use <= quantity" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do asset = Generator.asset asset.quantity = 20 asset.save! diff --git a/spec/authority_spec.cr b/spec/authority_spec.cr index d7cb51ae..eeaf8585 100644 --- a/spec/authority_spec.cr +++ b/spec/authority_spec.cr @@ -34,28 +34,28 @@ module PlaceOS::Model auth = Generator.authority.save! strat = Generator.adfs_strat(authority: auth).save! auth.destroy - SamlAuthentication.find(strat.id.as(String)).should be_nil + SamlAuthentication.find?(strat.id.as(String)).should be_nil end it "removes dependent ldap authentications" do auth = Generator.authority.save! strat = Generator.ldap_strat(authority: auth).save! auth.destroy - LdapAuthentication.find(strat.id.as(String)).should be_nil + LdapAuthentication.find?(strat.id.as(String)).should be_nil end it "removes dependent oauth authentications" do auth = Generator.authority.save! strat = Generator.oauth_strat(authority: auth).save! auth.destroy - OAuthAuthentication.find(strat.id.as(String)).should be_nil + OAuthAuthentication.find?(strat.id.as(String)).should be_nil end it "removes dependent users" do auth = Generator.authority.save! user = Generator.user(authority: auth, admin: false).save! auth.destroy - User.find(user.id.as(String)).should be_nil + User.find?(user.id.as(String)).should be_nil end it "fails if it will destroy the only remaining admin user" do diff --git a/spec/control_system_spec.cr b/spec/control_system_spec.cr index 58599314..c7408548 100644 --- a/spec/control_system_spec.cr +++ b/spec/control_system_spec.cr @@ -8,7 +8,7 @@ module PlaceOS::Model cs = Generator.control_system begin cs.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -20,7 +20,7 @@ module PlaceOS::Model end it "no duplicate control system names" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do name = RANDOM.base64(10) cs1 = ControlSystem.new( name: name, @@ -44,7 +44,7 @@ module PlaceOS::Model control_system = ControlSystem.find!(control_system.id.as(String)) control_system.destroy - Module.find(mod.id.as(String)).should be_nil + Module.find?(mod.id.as(String)).should be_nil driver.destroy end @@ -201,7 +201,7 @@ module PlaceOS::Model end zone.save! zone_id = zone.id - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end diff --git a/spec/doorkeeper_app_spec.cr b/spec/doorkeeper_app_spec.cr index 80ad3a1a..39926129 100644 --- a/spec/doorkeeper_app_spec.cr +++ b/spec/doorkeeper_app_spec.cr @@ -10,7 +10,7 @@ module PlaceOS::Model begin app.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -20,7 +20,7 @@ module PlaceOS::Model end it "no duplicate app uris" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do uri = "appuri://test.redirect.com.au/#{RANDOM.hex(3)}" app1 = DoorkeeperApplication.new app1.name = RANDOM.hex(10) @@ -30,13 +30,13 @@ module PlaceOS::Model app2 = DoorkeeperApplication.new app2.name = RANDOM.hex(10) app2.redirect_uri = uri - app2.owner_id = RANDOM.hex(10) + app2.owner_id = app1.owner_id app2.save! end end it "no duplicate app names" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do name = RANDOM.hex(3) app1 = DoorkeeperApplication.new app1.name = name @@ -46,7 +46,7 @@ module PlaceOS::Model app2 = DoorkeeperApplication.new app2.name = name app2.redirect_uri = "appuri://test.redirect.com.au/#{RANDOM.hex(3)}" - app2.owner_id = RANDOM.hex(10) + app2.owner_id = app1.owner_id app2.save! end end @@ -59,7 +59,7 @@ module PlaceOS::Model begin app.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -69,6 +69,7 @@ module PlaceOS::Model end it "saves an app with a specified UID" do + DoorkeeperApplication.clear app = DoorkeeperApplication.new app.name = RANDOM.hex(10) app.redirect_uri = "http://test.redirect.com.au/#{RANDOM.hex(3)}" @@ -77,7 +78,7 @@ module PlaceOS::Model begin app.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end diff --git a/spec/driver_spec.cr b/spec/driver_spec.cr index 08983f4b..fd236be6 100644 --- a/spec/driver_spec.cr +++ b/spec/driver_spec.cr @@ -47,7 +47,7 @@ module PlaceOS::Model Module.by_driver_id(driver.id.as(String)).first.id.should eq mod.id driver.destroy - Module.find(mod.id.as(String)).should be_nil + Module.find?(mod.id.as(String)).should be_nil end it "#update_modules updates dependent modules' driver metadata" do diff --git a/spec/helper.cr b/spec/helper.cr index d12d1fb2..ca8058f2 100644 --- a/spec/helper.cr +++ b/spec/helper.cr @@ -6,28 +6,19 @@ require "placeos-log-backend" require "spec" require "random" -require "rethinkdb-orm" +require "pg-orm" require "timecop" # Generators for Engine models require "./generator" -# Configure DB -db_name = "test" - -Spec.before_suite do - RethinkORM.configure do |settings| - settings.db = db_name - end -end +PgORM::Database.parse(ENV["PG_DATABASE_URL"]) # Clear test tables on exit Spec.after_suite do - RethinkORM::Connection.raw do |q| - q.db(db_name).table_list.for_each do |t| - q.db(db_name).table(t).delete - end - end + {% for model in PlaceOS::Model::ModelBase.subclasses %} + {{model.id}}.clear + {% end %} end # Spec Macros @@ -48,7 +39,7 @@ end ################################################################# # Pretty prints document errors -def inspect_error(error : RethinkORM::Error::DocumentInvalid) +def inspect_error(error : PgORM::Error::RecordInvalid) message = error.model.errors.join('\n') do |e| "#{e.field} #{e.message}" end diff --git a/spec/metadata_spec.cr b/spec/metadata_spec.cr index 5ce7495e..39278f83 100644 --- a/spec/metadata_spec.cr +++ b/spec/metadata_spec.cr @@ -116,7 +116,7 @@ module PlaceOS::Model original, duplicate = Array(Metadata).new(2) { Generator.metadata(name: name, parent: parent_id) } original.save! - expect_raises(RethinkORM::Error::DocumentInvalid, /`name` must be unique beneath 'parent_id'/) do + expect_raises(PgORM::Error::RecordInvalid, /`name` must be unique beneath 'parent_id'/) do duplicate.save! end end @@ -125,7 +125,7 @@ module PlaceOS::Model parent_id = "zone-doesnotexist" metadata = Generator.metadata(parent: parent_id) - expect_raises(RethinkORM::Error::DocumentInvalid, /`parent_id` must reference an existing model/) do + expect_raises(PgORM::Error::RecordInvalid, /`parent_id` must reference an existing model/) do metadata.save! end end diff --git a/spec/module_spec.cr b/spec/module_spec.cr index c5043930..5e25b8ac 100644 --- a/spec/module_spec.cr +++ b/spec/module_spec.cr @@ -11,7 +11,7 @@ module PlaceOS::Model mod = Generator.module(driver: driver) begin mod.save! - rescue error : RethinkORM::Error::DocumentInvalid + rescue error : PgORM::Error::RecordInvalid inspect_error(error) end mod.persisted?.should be_true @@ -125,7 +125,7 @@ module PlaceOS::Model control_system = Generator.control_system.save! begin zone = Generator.zone.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -208,7 +208,7 @@ module PlaceOS::Model driver_settings_string = %(value: 0\nscreen: 0\nfrangos: 0\nchop: 0) begin Generator.settings(driver: driver, settings_string: driver_settings_string).save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -279,7 +279,7 @@ module PlaceOS::Model # Reset the parent association reference through `reload!` {driver, zone, control_system, mod.reload!}.each &.destroy {control_system_settings, driver_settings, module_settings, zone_settings}.each do |setting| - Settings.find(setting.id.as(String)).should be_nil + Settings.find?(setting.id.as(String)).should be_nil end end @@ -318,7 +318,7 @@ module PlaceOS::Model # Reset the parent association reference through `reload!` {driver, zone, control_system, mod.reload!}.each &.destroy {control_system_settings, driver_settings, module_settings, zone_settings}.each do |setting| - Settings.find(setting.id.as(String)).should be_nil + Settings.find?(setting.id.as(String)).should be_nil end end end diff --git a/spec/statistics_spec.cr b/spec/statistics_spec.cr index 0452bb8b..55b210b9 100644 --- a/spec/statistics_spec.cr +++ b/spec/statistics_spec.cr @@ -8,7 +8,7 @@ module PlaceOS::Model begin stats.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end diff --git a/spec/user_spec.cr b/spec/user_spec.cr index 8d3c2112..ecfa634e 100644 --- a/spec/user_spec.cr +++ b/spec/user_spec.cr @@ -144,8 +144,8 @@ module PlaceOS::Model end describe "#assign_admin_attributes_from_json" do - {% for field in PlaceOS::Model::User::AdminAttributes.instance_vars %} - it "assigns {{ field.name }} attribute if present" do + {% for field in %w(sys_admin support login_name staff_id card_number groups) %} + it "assigns {{ field.id }} attribute if present" do support, updated_support = false, true sys_admin, updated_sys_admin = false, true login_name, updated_login_name = "fake", "even faker" @@ -154,16 +154,16 @@ module PlaceOS::Model groups, updated_groups = ["public"], ["private"] user = Model::User.new( support: support, - admin: admin, + sys_admin: sys_admin, login_name: login_name, staff_id: staff_id, card_number: card_number, groups: groups, ) - - user.assign_admin_attributes_from_json({ email: "shouldn't change", {{field.name}}: {{field.name.id}}_updated }.to_json) + user.clear_changes_information + user.assign_admin_attributes_from_json({ email: "shouldn't change", {{field.id}}: updated_{{field.id}} }.to_json) user.email_changed?.should be_false - user.{{field.id}}.should eq {{field.id}}_updated + user.{{field.id}}.should eq updated_{{field.id}} end {% end %} end diff --git a/spec/utilities/settings_helper_spec.cr b/spec/utilities/settings_helper_spec.cr index 8ce08d2c..c2018a60 100644 --- a/spec/utilities/settings_helper_spec.cr +++ b/spec/utilities/settings_helper_spec.cr @@ -23,7 +23,7 @@ module PlaceOS::Model model.destroy # Testing the destruction methods - Settings.find(settings.id.as(String)).should be_nil + Settings.find?(settings.id.as(String)).should be_nil end end end diff --git a/spec/zone_spec.cr b/spec/zone_spec.cr index 35f6a716..2ff088ab 100644 --- a/spec/zone_spec.cr +++ b/spec/zone_spec.cr @@ -9,7 +9,7 @@ module PlaceOS::Model begin zone.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -21,7 +21,7 @@ module PlaceOS::Model end it "no duplicate zone names" do - expect_raises(RethinkORM::Error::DocumentInvalid) do + expect_raises(PgORM::Error::RecordInvalid) do name = RANDOM.base64(10) zone1 = Zone.new( name: name, @@ -48,7 +48,7 @@ module PlaceOS::Model begin zone.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -62,7 +62,7 @@ module PlaceOS::Model zone2.parent_id = id begin zone2.save! - rescue e : RethinkORM::Error::DocumentInvalid + rescue e : PgORM::Error::RecordInvalid inspect_error(e) raise e end @@ -76,7 +76,7 @@ module PlaceOS::Model # show that deleting the parent deletes the children Zone.find!(id2.as(String)).id.should eq id2 zone.destroy - Zone.find(id2.as(String)).should be_nil + Zone.find?(id2.as(String)).should be_nil end it "should create triggers when added and removed from a zone" do @@ -113,7 +113,6 @@ module PlaceOS::Model zone = Zone.find!(zone.id.as(String)) zone.trigger_instances.to_a.size.should eq 0 - {cs, zone, trigger}.each &.destroy end diff --git a/src/placeos-models.cr b/src/placeos-models.cr index 0425b2d3..3201dc6f 100644 --- a/src/placeos-models.cr +++ b/src/placeos-models.cr @@ -1,17 +1,14 @@ +require "pg-orm" require "./ext/*" require "log" -require "rethinkdb-orm" - require "./placeos-models/base/*" module PlaceOS::Model Log = ::Log.for(self) - # Expose RethinkDB connection - # Use for configuration, raw queries - class Connection < RethinkORM::Connection - end + # class Connection < RethinkORM::Connection + # end end require "./placeos-models/*" diff --git a/src/placeos-models/api_key.cr b/src/placeos-models/api_key.cr index 867389a0..ca56a4a0 100644 --- a/src/placeos-models/api_key.cr +++ b/src/placeos-models/api_key.cr @@ -6,14 +6,14 @@ require "./user_jwt" module PlaceOS::Model class ApiKey < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :api_key attribute name : String, es_subfield: "keyword" attribute description : String = "" - attribute scopes : Array(UserJWT::Scope) = [UserJWT::Scope::PUBLIC], es_type: "keyword" + attribute scopes : Array(UserJWT::Scope) = [UserJWT::Scope::PUBLIC], converter: PlaceOS::Model::DBArrConverter(PlaceOS::Model::UserJWT::Scope), es_type: "keyword" # when nil it defaults to the users permissions attribute permissions : UserJWT::Permissions? = nil, es_type: "keyword" @@ -23,8 +23,6 @@ module PlaceOS::Model belongs_to User belongs_to Authority - secondary_index :authority_id - macro finished def user=(user) super(user) @@ -62,7 +60,7 @@ module PlaceOS::Model before_create :hash! protected def safe_id - self._new_flag = true + self.new_record = true @id ||= Random.new.hex(16) end @@ -92,7 +90,7 @@ module PlaceOS::Model # Same error as being unable to find the model if model.secret != OpenSSL::HMAC.hexdigest(:sha512, secret, id) - raise RethinkORM::Error::DocumentNotFound.new("Key not present: #{id}") + raise PgORM::Error::RecordNotFound.new("Key not present: #{id}") end model @@ -134,7 +132,7 @@ module PlaceOS::Model user_id = user.id.as(String) saas_scope = UserJWT::Scope::SAAS.to_s public_scope = UserJWT::Scope::PUBLIC.to_s - existing_key = Model::ApiKey.where(authority_id: authority_id, user_id: user_id, scopes: [saas_scope, public_scope]).first? + existing_key = Model::ApiKey.where(authority_id: authority_id, user_id: user_id, scopes: [saas_scope, public_scope].to_json).first? if existing_key.nil? key = Model::ApiKey.new( name: "Portal SaaS Key", diff --git a/src/placeos-models/asset.cr b/src/placeos-models/asset.cr index c26e2f3e..1ca2e2f6 100644 --- a/src/placeos-models/asset.cr +++ b/src/placeos-models/asset.cr @@ -2,7 +2,7 @@ require "./base/model" module PlaceOS::Model class Asset < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :asset diff --git a/src/placeos-models/asset_instance.cr b/src/placeos-models/asset_instance.cr index e00732be..eb0c9f89 100644 --- a/src/placeos-models/asset_instance.cr +++ b/src/placeos-models/asset_instance.cr @@ -3,7 +3,7 @@ require "./user" module PlaceOS::Model class AssetInstance < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps enum Tracking InStorage @@ -15,7 +15,7 @@ module PlaceOS::Model table :ass attribute name : String, es_subfield: "keyword" - attribute tracking : Tracking = Tracking::InStorage + attribute tracking : Tracking = Tracking::InStorage, converter: PlaceOS::Model::EnumConverter(PlaceOS::Model::AssetInstance::Tracking) attribute approval : Bool = false attribute asset_id : String diff --git a/src/placeos-models/authority.cr b/src/placeos-models/authority.cr index b07a7279..76169cc6 100644 --- a/src/placeos-models/authority.cr +++ b/src/placeos-models/authority.cr @@ -9,7 +9,7 @@ require "./user" module PlaceOS::Model class Authority < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :authority @@ -59,7 +59,7 @@ module PlaceOS::Model validates :domain, presence: true validates :name, presence: true - ensure_unique :domain, create_index: true + ensure_unique :domain # Queries ########################################################################### @@ -68,7 +68,7 @@ module PlaceOS::Model # def self.find_by_domain(domain : String) : Authority? host = URI.parse(domain).host || domain - Authority.find_all([host], index: :domain).first? + Authority.where(domain: host).first? end end end diff --git a/src/placeos-models/base/associations.cr b/src/placeos-models/base/associations.cr new file mode 100644 index 00000000..bfaf7619 --- /dev/null +++ b/src/placeos-models/base/associations.cr @@ -0,0 +1,144 @@ +module PlaceOS::Model + module Associations + # Defines getter and setter for parent relationship + macro belongs_to(parent_class, dependent = :none, association_name = nil, foreign_key = nil, presence = false) + {% parent_name = association_name || parent_class.id.stringify.underscore.downcase.gsub(/::/, "_") %} + {% foreign_key = (foreign_key || "#{parent_name.id}_id").id %} + {% association_method = parent_name.id.symbolize %} + {% assoc_var = "__#{parent_name.id}".id %} + + # Prevent association from being serialised + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + @{{ assoc_var }} : {{ parent_class }}? + + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + property {{ assoc_var }} : {{ parent_class }}? + + attribute {{ foreign_key.id }} : String {% unless presence %} | Nil {% end %}, parent: {{ parent_class.id.stringify }}, es_type: "keyword" + + destroy_callback({{ association_method }}, {{dependent}}) + + # Retrieves the parent relationship + def {{ parent_name.id }} : {{ parent_class }}? + parent = @{{ assoc_var }} + key = self.{{ foreign_key }} + + return parent if parent + + self.{{ assoc_var }} = key ? {{ parent_class }}.find?(key) : nil + end + + def {{ parent_name.id }}! : {{ parent_class }} + parent = @{{ assoc_var }} + key = self.{{ foreign_key }} + + return parent if parent + raise PgORM::Error.new("No {{ foreign_key }} set") unless key + + self.{{ assoc_var }} = {{ parent_class }}.find!(key) + end + + # Sets the parent relationship + def {{ parent_name.id }}=(parent : {{ parent_class }}) + self.{{ assoc_var }} = parent + self.{{ foreign_key.id }} = parent.id.as(String) + end + + def reset_associations + self.{{ assoc_var }} = nil + end + + # Look up instances of this model dependent on the foreign key + def self.by_{{ foreign_key.id }}(id) + self.where({{ foreign_key }}: id) + end + end + + macro has_one(child_class, dependent = :none, association_name = nil, presence = false) + {% child = association_name || child_class.id.underscore.downcase.gsub(/::/, "_") %} + {% assoc_var = "__#{child.id}".id %} + {% foreign_key = child + "_id" %} + {% association_method = child.id.symbolize %} + + # Prevent association from being serialised + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + @{{ assoc_var }} : {{ child_class }}? + + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + property {{ assoc_var }} : {{ child_class }}? + + attribute {{ foreign_key.id }} : String {% unless presence %} | Nil {% end %} + + destroy_callback({{ association_method }}, {{dependent}}) + + # Get cached child or attempt to load an associated {{child.id}} + def {{ child.id }} : {{ child_class }}? + key = self.{{ foreign_key.id }} + child = @{{ assoc_var }} + return child unless child.nil? + + self.{{ assoc_var }} = key && !key.empty? ? {{ child_class }}.find?(key) + : nil + end + + def {{ child.id }}! : {{ child_class }} + key = self.{{ foreign_key.id }} + child = @{{ assoc_var }} + return child unless child.nil? + raise PgORM::Error.new("No {{ foreign_key.id }} set") unless key + + self.{{ assoc_var }} = {{ child_class }}.find!(key) + end + + def {{ child.id }}=(child) + self.{{ assoc_var }} = child + self.{{ foreign_key.id }} = child.id.not_nil! + end + + def reset_associations + self.{{ assoc_var }} = nil + end + end + + # Must be used in conjunction with the belongs_to macro + macro has_many(child_class, collection_name = nil, dependent = :none, foreign_key = nil) + {% child_collection = (collection_name ? collection_name : child_class + 's').underscore.downcase %} + {% association_method = child_collection.id.symbolize %} + + destroy_callback({{association_method}}, {{ dependent }}) + + @[JSON::Field(ignore: true)] + @[YAML::Field(ignore: true)] + @{{child_collection.id}} : ::PgORM::Relation({{child_class}})? + + def {{ child_collection.id }} + @{{child_collection.id}} ||= ::PgORM::Relation({{child_class}}).new(self, {{foreign_key.id.symbolize}}) + end + end + + # Generate destroy callbacks for dependent associations + private macro destroy_callback(method, dependent) + {% if dependent.id == :destroy || dependent.id == :delete %} + + def destroy_{{ method.id }} + return unless (association = {{ method.id }}) + if association.is_a?(PgORM::Relation) + association.each { |model| model.destroy } + else + association.destroy + end + end + + before_destroy :destroy_{{ method.id }} + {% end %} + end + + def reset_associations + # noop + end + end +end diff --git a/src/placeos-models/base/model.cr b/src/placeos-models/base/model.cr index 395c26ad..2f082c75 100644 --- a/src/placeos-models/base/model.cr +++ b/src/placeos-models/base/model.cr @@ -2,7 +2,7 @@ require "active-model" require "json" require "log" require "neuroplastic" -require "rethinkdb-orm" +require "pg-orm" require "openapi-generator" require "openapi-generator/serializable" @@ -10,17 +10,28 @@ require "openapi-generator/serializable/adapters/active-model" require "../utilities/encryption" require "../utilities/validation" +require "../utilities/id_generator" +require "./associations" +require "./timestamps" module PlaceOS::Model # Base class for all Engine models - abstract class ModelBase < RethinkORM::Base + abstract class ModelBase < PgORM::Base include Neuroplastic macro inherited + macro finished + default_primary_key id : String?, autogenerated: true + end + Log = ::Log.for(self) include OpenAPI::Generator::Serializable::Adapters::ActiveModel extend OpenAPI::Generator::Serializable end + + before_create { self.id = Utilities::IdGenerator.next(self) unless self.id? } + before_save { self.id = Utilities::IdGenerator.next(self) unless self.id? } + include Model::Associations end # Validation for embedded objects in Engine models @@ -49,4 +60,66 @@ module PlaceOS::Model end end end + + # :nodoc: + module DBArrConverter(T) + def self.from_rs(rs : ::DB::ResultSet) + vals = JSON::Any.new(rs.read(JSON::PullParser)).to_json + Array(T).from_json(vals) + end + + def self.from_json(pull : JSON::PullParser) + arr = Array(T).new + pull.read_array do + if T <= UserJWT::Scope + arr << T.new(pull) + else + arr << T.from_json(pull.read_string) + end + end + arr + end + + def self.to_json(value : Array(T) | Nil) + String.build do |sb| + value.to_json(sb) + end + end + + def self.to_json(value : Array(T) | Nil, builder) + value.to_json(builder) + end + end + + # :nodoc: + module EnumConverter(T) + def self.from_rs(rs : ::DB::ResultSet) + val = rs.read(Int32) + T.from_value(val) + end + + def self.from_json(pull : JSON::PullParser) : T + str = pull.read_raw + if (val = str.to_i?) + T.from_value?(val) || pull.raise "Unknown enum #{T} value: #{str}" + else + T.parse?(str.strip('"')) || pull.raise "Uknown enum #{T} value: #{str}" + end + end + + def self.to_json(val : T | Nil) + val.to_s + end + + def self.to_json(val : T | Nil, builder) + val.try &.to_json(builder) + end + end +end + +# :nodoc: +module Time::EpochConverter + def self.from_rs(rs : DB::ResultSet) + rs.read(Time?) + end end diff --git a/src/placeos-models/base/timestamps.cr b/src/placeos-models/base/timestamps.cr new file mode 100644 index 00000000..4957c78a --- /dev/null +++ b/src/placeos-models/base/timestamps.cr @@ -0,0 +1,75 @@ +require "time" + +# Creates created_at and updated_at attributes. +# - `updated_at` is set through the `before_update` callback +# - `created_at` is set through the `before_update` callback +# +module PlaceOS::Model::Timestamps + macro included + attribute created_at : Time = ->{ Time.utc }, converter: PlaceOS::Model::Timestamps::EpochConverter, type: "integer", format: "Int64" + attribute updated_at : Time = ->{ Time.utc }, converter: PlaceOS::Model::Timestamps::EpochConverter, type: "integer", format: "Int64" + + before_create do + self.created_at = self.updated_at = Time.utc + end + + before_update do + self.updated_at = Time.utc + end + end +end + +# :nodoc: +module PlaceOS::Model::Timestamps::EpochConverter + def self.from_rs(rs : DB::ResultSet) : Time + rs.read(Time) + end + + def self.from_json(value : JSON::PullParser) : Time + str = value.read_raw + if (val = str.to_i?) + Time.unix(val) + else + Time.from_json(str) + end + end + + def self.to_json(value : Time, json : JSON::Builder) : Nil + json.number(value.to_unix) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.kind}" + end + + Time.unix(node.value.to_i) + end +end + +module PlaceOS::Model::Timestamps::EpochMillisConverter + def self.from_rs(rs : DB::ResultSet) : Time + rs.read(Time) + end + + def self.from_json(value : JSON::PullParser) : Time + str = value.read_raw + if (val = str.to_i?) + Time.unix_ms(val) + else + Time.from_json(str) + end + end + + def self.to_json(value : Time, json : JSON::Builder) : Nil + json.number(value.to_unix_ms) + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time + unless node.is_a?(YAML::Nodes::Scalar) + node.raise "Expected scalar, not #{node.kind}" + end + + Time.unix_ms(node.value.to_i64) + end +end diff --git a/src/placeos-models/broker.cr b/src/placeos-models/broker.cr index 45487088..1a71140c 100644 --- a/src/placeos-models/broker.cr +++ b/src/placeos-models/broker.cr @@ -1,12 +1,11 @@ require "openssl" require "random" -require "rethinkdb-orm" require "./base/model" module PlaceOS::Model class Broker < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :broker diff --git a/src/placeos-models/control_system.cr b/src/placeos-models/control_system.cr index ec0f7fcc..92bd3fc9 100644 --- a/src/placeos-models/control_system.cr +++ b/src/placeos-models/control_system.cr @@ -1,4 +1,3 @@ -require "rethinkdb-orm" require "time" require "uri" require "future" @@ -13,7 +12,7 @@ require "./utilities/metadata_helper" module PlaceOS::Model class ControlSystem < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::SettingsHelper include Utilities::MetadataHelper @@ -25,7 +24,7 @@ module PlaceOS::Model # Room search meta-data # Building + Level are both filtered using zones attribute features : Set(String) = ->{ Set(String).new } - attribute email : Email? + attribute email : Email?, converter: PlaceOS::Model::EmailConverter attribute bookable : Bool = false attribute public : Bool = false attribute display_name : String? @@ -79,9 +78,6 @@ module PlaceOS::Model foreign_key: "control_system_id" ) - # Provide a email lookup helpers - secondary_index :email - # Validation ############################################################################################### @@ -103,11 +99,7 @@ module PlaceOS::Model ############################################################################################### def self.by_zone_id(id) - ControlSystem.raw_query do |q| - q.table(ControlSystem.table_name).filter do |doc| - doc["zones"].contains(id) - end - end + ControlSystem.where("$1 = Any(zones)", id) end @[Deprecated("Use `by_zone_id`")] @@ -116,11 +108,7 @@ module PlaceOS::Model end def self.by_module_id(id) - ControlSystem.raw_query do |q| - q.table(ControlSystem.table_name).filter do |doc| - doc["modules"].contains(id) - end - end + ControlSystem.where("$1 = Any(modules)", id) end @[Deprecated("Use `by_module_id`")] @@ -214,15 +202,19 @@ module PlaceOS::Model return if self.modules.empty? # Locate modules that have no other associated ControlSystems - lonesome_modules = Module.raw_query do |r| - r.table(Module.table_name).get_all(self.modules).filter do |mod| - # Find the control systems that have the module - r.table(ControlSystem.table_name).filter do |sys| - sys["modules"].contains(mod["id"]) - end.count.eq(1) - end + args = [] of String + query = "" + + self.modules.each_with_index do |v, i| + args << v + query += ", " unless i == 0 + query += "$#{i + 1}" end + lonesome_modules = Module.find_all_by_sql(<<-SQL, args: args) + select m.* from "#{Module.table_name}" m, "#{ControlSystem.table_name}" s where m.id in (#{query}) and m.id = ANY(s.modules) + SQL + # Asynchronously remove the modules lonesome_modules.map do |m| future { m.destroy } @@ -291,60 +283,52 @@ module PlaceOS::Model def add_module(module_id : String) if !self.modules.includes?(module_id) && ControlSystem.add_module(id.as(String), module_id) self.modules << module_id - self.version = ControlSystem.table_query(&.get(id.as(String))["version"]).as_i + self.version = ControlSystem.find(id).version end end def self.add_module(control_system_id : String, module_id : String) - response = Model::ControlSystem.table_query do |q| - q - .get(control_system_id) - .update { |sys| - { - "modules" => sys["modules"].set_insert(module_id), - "version" => sys["version"] + 1, - } - } + response = PgORM::Database.connection do |db| + db.exec(<<-SQL, control_system_id, [module_id]) + update #{ControlSystem.table_name} set modules = modules || $2, version = version + 1 where id = $1 + SQL end - {"replaced", "updated"}.any? { |k| response[k].try(&.as_i) || 0 > 0 } + response.rows_affected > 0 end # Removes the module from the system and deletes it if not used elsewhere # def remove_module(module_id : String) - mod = Module.find(module_id) + mod = Module.find?(module_id) if self.modules.includes?(module_id) && ControlSystem.remove_module(id.as(String), module_id) + self.modules_will_change! self.modules.delete(module_id) unless mod.nil? # Remove the module from the control system's features + self.features_will_change! self.features.delete(mod.resolved_name) self.features.delete(mod.name) end - self.version = ControlSystem.table_query(&.get(id.as(String))["version"]).as_i + self.version = ControlSystem.find(id).version end end def self.remove_module(control_system_id : String, module_id : String) - response = ControlSystem.table_query do |q| - q - .get(control_system_id) - .update { |sys| - { - "modules" => sys["modules"].set_difference([module_id]), - "version" => sys["version"] + 1, - } - } + response = PgORM::Database.connection do |db| + db.exec(<<-SQL, control_system_id, [module_id]) + update #{ControlSystem.table_name} set modules=(select array(select unnest(modules) except select unnest($2::text[]))), version = version + 1 where id = $1 + SQL end - return false unless {"replaced", "updated"}.any? { |k| response[k].try(&.as_i) || 0 > 0 } + return false unless response.rows_affected > 0 # Keep if any other ControlSystem is using the module still_in_use = ControlSystem.by_module_id(module_id).any? do |sys| sys.id != control_system_id end - Module.find(module_id).try(&.destroy) if !still_in_use + Module.find?(module_id).try(&.destroy) unless still_in_use Log.debug { { message: "module removed from system #{still_in_use ? "still in use" : "deleted as not in any other systems"}", diff --git a/src/placeos-models/converter/json_string.cr b/src/placeos-models/converter/json_string.cr index 7ac4a22e..6c70dd2e 100644 --- a/src/placeos-models/converter/json_string.cr +++ b/src/placeos-models/converter/json_string.cr @@ -2,14 +2,26 @@ require "json" require "yaml" # :nodoc: -# Used to prevent overwrite the object merges of RethinkDB module JSON::Any::StringConverter def self.from_json(value : JSON::PullParser) : JSON::Any - JSON.parse(value.read_string) + v = value.read_raw + if v.is_a?(String) + if v.strip('"') == "{}" + JSON::Any.new({} of String => JSON::Any) + else + JSON.parse(v.to_s) + end + else + JSON::Any.new(v) + end end def self.to_json(value : JSON::Any, json : JSON::Builder) - json.string(value.to_json) + if h = value.as_h? + JSON::Any.new(h.to_json).to_json(json) + else + value.to_json(json) + end end def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : JSON::Any @@ -20,4 +32,18 @@ module JSON::Any::StringConverter def self.to_yaml(value : JSON::Any, yaml : YAML::Nodes::Builder) yaml.scalar(value.to_json) end + + def self.from_rs(rs : ::DB::ResultSet) + JSON::Any.new(rs.read(JSON::PullParser)) + end + + def self.to_json(value : JSON::Any?) + value.try &.to_json + end +end + +module Enum::ValueConverter(T) + def self.from_rs(rs : ::DB::ResultSet) + T.from_value(rs.read(Int32)) + end end diff --git a/src/placeos-models/converter/time_location.cr b/src/placeos-models/converter/time_location.cr index 1021e204..c2bb6da5 100644 --- a/src/placeos-models/converter/time_location.cr +++ b/src/placeos-models/converter/time_location.cr @@ -1,5 +1,12 @@ require "json" require "time" +require "db" + +class Time::Location + def to_json(json : JSON::Builder) : Nil + json.string(self.to_s) + end +end module Time::Location::Converter def self.from_json(value : JSON::PullParser) : Time::Location @@ -10,6 +17,10 @@ module Time::Location::Converter json.string(value.to_s) end + def self.to_json(value : Time::Location?) + value.try &.to_s + end + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Location unless node.is_a?(YAML::Nodes::Scalar) node.raise "Expected scalar, not #{node.class}" @@ -20,4 +31,9 @@ module Time::Location::Converter def self.to_yaml(value : Time::Location, yaml : YAML::Nodes::Builder) yaml.scalar(value.to_s) end + + def self.from_rs(rs : ::DB::ResultSet) + val = rs.read(String?) + val ? Time::Location.load(val.not_nil!) : nil + end end diff --git a/src/placeos-models/doorkeeper_application.cr b/src/placeos-models/doorkeeper_application.cr index 4518033c..aa40e72e 100644 --- a/src/placeos-models/doorkeeper_application.cr +++ b/src/placeos-models/doorkeeper_application.cr @@ -6,33 +6,35 @@ require "./base/model" require "digest/md5" module PlaceOS::Model - class DoorkeeperApplication < ModelBase - include RethinkORM::Timestamps - - table :doorkeeper_app - + class DoorkeeperApplication < PgORM::Base + include Neuroplastic + include OpenAPI::Generator::Serializable::Adapters::ActiveModel + extend OpenAPI::Generator::Serializable + include Model::Associations + include PlaceOS::Model::Timestamps + + table :oauth_applications + default_primary_key id : Int64?, autogenerated: true attribute name : String, es_subfield: "keyword" attribute secret : String attribute scopes : String = "public" attribute owner_id : String, es_type: "keyword" attribute redirect_uri : String - attribute skip_authorization : Bool = true attribute confidential : Bool = false - attribute revoked_at : Time?, converter: Time::EpochConverter, type: "integer", format: "Int64" attribute uid : String, mass_assignment: false # Validation ############################################################################################### - ensure_unique :uid, create_index: true + ensure_unique :uid, scope: [:owner_id] - ensure_unique :redirect_uri do |redirect_uri| - redirect_uri.strip + ensure_unique :redirect_uri, scope: [:owner_id, :redirect_uri] do |owner_id, redirect_uri| + {owner_id, redirect_uri.strip} end - ensure_unique :name do |name| - name.strip + ensure_unique :name, scope: [:owner_id, :name] do |owner_id, name| + {owner_id, name.strip} end validates :name, presence: true @@ -59,8 +61,7 @@ module PlaceOS::Model current_id = @id if current_id.nil? # Ensure document is treated as unpersisted - self._new_flag = true - @id = self.uid + self.new_record = true end end end diff --git a/src/placeos-models/driver.cr b/src/placeos-models/driver.cr index e954a588..a33ff78e 100644 --- a/src/placeos-models/driver.cr +++ b/src/placeos-models/driver.cr @@ -7,7 +7,7 @@ require "./settings" module PlaceOS::Model class Driver < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::SettingsHelper table :driver @@ -51,7 +51,6 @@ module PlaceOS::Model # Association ############################################################################################### - belongs_to Repository, foreign_key: "repository_id", presence: true # Encrypted yaml settings diff --git a/src/placeos-models/edge.cr b/src/placeos-models/edge.cr index 7f1df668..f52e072c 100644 --- a/src/placeos-models/edge.cr +++ b/src/placeos-models/edge.cr @@ -4,7 +4,7 @@ require "./api_key" module PlaceOS::Model class Edge < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :edge @@ -51,7 +51,7 @@ module PlaceOS::Model def save!(**options) super(**options) - rescue error : RethinkORM::Error + rescue error : PgORM::Error # Ensure api_key is cleaned up self.api_key.try(&.destroy) raise error @@ -81,9 +81,8 @@ module PlaceOS::Model has_one( child_class: ApiKey, dependent: :destroy, - create_index: true, association_name: "api_key", - presence: true, + presence: false, ) # Callbacks @@ -95,8 +94,8 @@ module PlaceOS::Model # Generate ID before document is created protected def set_id if @id.nil? - self._new_flag = true - @id = RethinkORM::IdGenerator.next(self) + self.new_record = true + @id = Utilities::IdGenerator.next(self) end end diff --git a/src/placeos-models/email.cr b/src/placeos-models/email.cr index fb389fee..1ab80d21 100644 --- a/src/placeos-models/email.cr +++ b/src/placeos-models/email.cr @@ -34,4 +34,32 @@ module PlaceOS::Model {type: "string", format: "email"} end end + + # :nodoc: + module EmailConverter + def self.from_rs(rs : ::DB::ResultSet) + Email.new(rs.read(String?) || "") + end + + def self.from_json(value : JSON::PullParser) + Email.from_json(value) + end + + def self.to_json(value : Email, json : JSON::Builder) + json.string(value.to_s) + end + + def self.to_json(value : Email?) + value.try &.to_s + end + + def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Email + node.raise "Expected scalar, not #{node.class}" unless node.is_a?(YAML::Nodes::Scalar) + Email.new(node.value.to_s) + end + + def self.to_yaml(value : JSON::Any, yaml : YAML::Nodes::Builder) + yaml.scalar(value.to_s) + end + end end diff --git a/src/placeos-models/json_schema.cr b/src/placeos-models/json_schema.cr index 39a2f81d..039cb915 100644 --- a/src/placeos-models/json_schema.cr +++ b/src/placeos-models/json_schema.cr @@ -3,7 +3,7 @@ require "./converter/json_string" module PlaceOS::Model class JsonSchema < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :json_schema diff --git a/src/placeos-models/ldap_authentication.cr b/src/placeos-models/ldap_authentication.cr index 624facab..c20362cc 100644 --- a/src/placeos-models/ldap_authentication.cr +++ b/src/placeos-models/ldap_authentication.cr @@ -5,7 +5,7 @@ require "./base/model" module PlaceOS::Model # see: https://github.com/omniauth/omniauth-ldap class LdapAuthentication < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :ldap_strat diff --git a/src/placeos-models/metadata.cr b/src/placeos-models/metadata.cr index 1471521b..a07bed72 100644 --- a/src/placeos-models/metadata.cr +++ b/src/placeos-models/metadata.cr @@ -2,8 +2,8 @@ require "json" require "json-merge-patch/ext" require "openapi-generator/serializable" require "pars" -require "rethinkdb" -require "rethinkdb-orm" + +require "pg-orm" require "time" require "./utilities/last_modified" @@ -15,7 +15,7 @@ require "./zone" module PlaceOS::Model class Metadata < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::LastModified include Utilities::Versions @@ -32,10 +32,8 @@ module PlaceOS::Model # Association ############################################################################################### - secondary_index :parent_id - secondary_index :name - # Models that `Metadata` is attached to + belongs_to Zone, foreign_key: "parent_id", association_name: "zone" belongs_to ControlSystem, foreign_key: "parent_id", association_name: "control_system" belongs_to User, foreign_key: "parent_id", association_name: "user" @@ -70,14 +68,18 @@ module PlaceOS::Model def self.validate_parent_exists(metadata : Metadata) # Skip validation if `Metadata` has been created - return unless metadata.id.nil? + return unless metadata.new_record? # `parent_id` presence is already enforced return if metadata.parent_id.nil? table_name = metadata.parent_id.as(String).partition('-').first - if RethinkORM::Connection.raw(&.table(table_name).get(metadata.parent_id)).raw.nil? - metadata.validation_error(:parent_id, "must reference an existing model") + + found = PgORM::Database.connection do |db| + db.scalar(<<-SQL, metadata.parent_id).as(Int64) > 0 + select count(*) from "#{table_name}" where id = $1 + SQL end + metadata.validation_error(:parent_id, "must reference an existing model") unless found end def self.validate_unique_name(metadata : Metadata) @@ -109,8 +111,8 @@ module PlaceOS::Model end master_metadata_query do |q| - q = q.get_all(parent_id, index: :parent_id) - q = q.filter({name: name}) if name && !name.empty? + q = q.where(parent_id: parent_id) + q = q.where(name: name) if name && !name.empty? q end end diff --git a/src/placeos-models/module.cr b/src/placeos-models/module.cr index f3aaa803..a84e5ff5 100644 --- a/src/placeos-models/module.cr +++ b/src/placeos-models/module.cr @@ -1,4 +1,3 @@ -require "rethinkdb-orm" require "future" require "uri" @@ -10,7 +9,7 @@ require "./utilities/settings_helper" module PlaceOS::Model class Module < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::SettingsHelper table :mod @@ -152,45 +151,25 @@ module PlaceOS::Model # Find `Module`s allocated to an `Edge` # def self.on_edge(edge_id : String) - Module.find_all([edge_id], index: :edge_id) + Module.where(edge_id: [edge_id]) end # Fetch `Module`s who have a direct parent `ControlSystem` # def self.logic_for(control_system_id : String) - Module.find_all([control_system_id], index: :control_system_id) + Module.where(control_system_id: [control_system_id]) end def self.in_control_system(control_system_id : String) - Module.raw_query do |q| - q - .table(PlaceOS::Model::ControlSystem.table_name) - # Find the control system - .get(control_system_id)["modules"] - # Find the module ids for the control systems - .map { |id| q.table(PlaceOS::Model::Module.table_name).get(id) } - # Return all modules located - .filter(&.has_fields("id")) - # Unique module ids - .distinct - end + Module.find_all_by_sql(<<-SQL, args: [control_system_id]) + select distinct * from "#{Module.table_name}" where id in (select unnest(modules) from "#{ControlSystem.table_name}" where id = $1) + SQL end def self.in_zone(zone_id : String) - Module.raw_query do |q| - q - .table(PlaceOS::Model::ControlSystem.table_name) - # Find control systems that have the zone - .filter(&.["zones"].contains(zone_id)) - # Find the module ids for the control systems - .concat_map { |sys| - sys["modules"].map { |id| q.table(PlaceOS::Model::Module.table_name).get(id) } - } - # Return all modules located - .filter(&.has_fields("id")) - # Unique module ids - .distinct - end + Module.find_all_by_sql(<<-SQL, args: [zone_id]) + select distinct * from "#{Module.table_name}" where id in (select unnest(modules) from "#{ControlSystem.table_name}" where $1 = ANY(zones)) + SQL end # Collect Settings ordered by hierarchy @@ -248,6 +227,7 @@ module PlaceOS::Model protected def add_logic_module return unless (cs = self.control_system) + cs.modules_will_change! cs.modules = cs.modules << self.id.as(String) cs.version = cs.version + 1 cs.save! @@ -258,15 +238,11 @@ module PlaceOS::Model protected def remove_module mod_id = self.id.as(String) - ControlSystem - .raw_query(&.table(ControlSystem.table_name).filter(&.["modules"].contains(mod_id))) + ControlSystem.where("$1 = ANY(modules)", mod_id) .map do |sys| sys.remove_module(mod_id) - # The `ControlSystem` will regenerate `features` - future { - sys.save! - } - end.each &.get + sys.save! + end end # Set the name/role from the associated Driver @@ -283,8 +259,8 @@ module PlaceOS::Model protected def set_edge_hint if on_edge? - self._new_flag = true - @id = RethinkORM::IdGenerator.next(self) + EDGE_HINT + self.new_record = true + @id = Utilities::IdGenerator.next(self) + EDGE_HINT end end diff --git a/src/placeos-models/oauth_authentication.cr b/src/placeos-models/oauth_authentication.cr index 073d0dcc..55ff65b1 100644 --- a/src/placeos-models/oauth_authentication.cr +++ b/src/placeos-models/oauth_authentication.cr @@ -6,7 +6,7 @@ require "./base/model" module PlaceOS::Model # See: https://github.com/omniauth/omniauth-oauth2 class OAuthAuthentication < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :oauth_strat diff --git a/src/placeos-models/repository.cr b/src/placeos-models/repository.cr index 72fd7321..6f141bef 100644 --- a/src/placeos-models/repository.cr +++ b/src/placeos-models/repository.cr @@ -1,5 +1,3 @@ -require "rethinkdb" -require "rethinkdb-orm" require "time" require "./base/model" @@ -9,7 +7,7 @@ module PlaceOS::Model # Pins engine's driver sources to a specific repository state. # Enables external driver management from a VCS. class Repository < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :repo @@ -41,7 +39,7 @@ module PlaceOS::Model end end - attribute repo_type : Type = Type::Driver, es_type: "text" + attribute repo_type : Type = Type::Driver, converter: PlaceOS::Model::EnumConverter(PlaceOS::Model::Repository::Type), es_type: "text" # Association ############################################################################################### @@ -78,8 +76,8 @@ module PlaceOS::Model # Generate ID before document is created protected def set_id - self._new_flag = true - @id = RethinkORM::IdGenerator.next(self) + self.new_record = true + @id = Utilities::IdGenerator.next(self) end # Encrypt sensitive fields diff --git a/src/placeos-models/saml_authentication.cr b/src/placeos-models/saml_authentication.cr index 6ebeacc0..5d7d5f89 100644 --- a/src/placeos-models/saml_authentication.cr +++ b/src/placeos-models/saml_authentication.cr @@ -5,7 +5,7 @@ require "./base/model" module PlaceOS::Model # See: https://github.com/omniauth/omniauth-saml class SamlAuthentication < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :adfs_strat @@ -54,7 +54,7 @@ module PlaceOS::Model {name: "name", name_format: "urn:oasis:names:tc:SAML:2.0:attrname-format:basic", friendly_name: "Full name"}, {name: "first_name", name_format: "urn:oasis:names:tc:SAML:2.0:attrname-format:basic", friendly_name: "Given name"}, {name: "last_name", name_format: "urn:oasis:names:tc:SAML:2.0:attrname-format:basic", friendly_name: "Family name"}, - ] + ], converter: PlaceOS::Model::DBArrConverter(NamedTuple(name: String, name_format: String, friendly_name: String)) # The URL to which the single logout request and response should be sent attribute idp_slo_target_url : String? diff --git a/src/placeos-models/settings.cr b/src/placeos-models/settings.cr index 10a4b513..59431157 100644 --- a/src/placeos-models/settings.cr +++ b/src/placeos-models/settings.cr @@ -1,4 +1,3 @@ -require "rethinkdb-orm" require "time" require "./base/model" @@ -14,7 +13,7 @@ require "./zone" module PlaceOS::Model class Settings < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::LastModified include Utilities::Versions @@ -46,7 +45,7 @@ module PlaceOS::Model end end - attribute parent_type : ParentType, es_type: "keyword" + attribute parent_type : ParentType, converter: PlaceOS::Model::EnumConverter(PlaceOS::Model::Settings::ParentType), es_type: "keyword" # Association ############################################################################################### @@ -54,9 +53,6 @@ module PlaceOS::Model attribute parent_id : String?, es_type: "keyword" attribute settings_id : String? = nil - secondary_index :parent_id - secondary_index :settings_id - belongs_to ControlSystem, foreign_key: "parent_id" belongs_to Driver, foreign_key: "parent_id" belongs_to Module, foreign_key: "parent_id", association_name: "mod" @@ -151,7 +147,7 @@ module PlaceOS::Model # def self.for_parent(parent_ids : String | Array(String), &) : Array(self) master_settings_query do |q| - yield (q.get_all(parent_ids, index: :parent_id)) + q.where({parent_id: parent_ids}) end.sort_by! do |setting| # Reversed -1 * setting.encryption_level.value @@ -166,10 +162,7 @@ module PlaceOS::Model # Query all settings under `parent_id` # def self.query(ids : String | Array(String)) - ids = ids.is_a?(Array) ? ids : [ids] - Settings.raw_query do |q| - yield q.table(Settings.table_name).get_all(ids, index: :parent_id) - end + Settings.find_all(ids.is_a?(Array) ? ids : [ids]) end # Locate the modules that will be affected by the change of this setting diff --git a/src/placeos-models/statistics.cr b/src/placeos-models/statistics.cr index a3834ffd..96507fc1 100644 --- a/src/placeos-models/statistics.cr +++ b/src/placeos-models/statistics.cr @@ -1,10 +1,9 @@ -require "rethinkdb-orm" require "time" require "./base/model" module PlaceOS::Model class Statistics < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :stats diff --git a/src/placeos-models/trigger.cr b/src/placeos-models/trigger.cr index 6e3aaafe..0e5e9cc3 100644 --- a/src/placeos-models/trigger.cr +++ b/src/placeos-models/trigger.cr @@ -1,5 +1,4 @@ require "json" -require "rethinkdb-orm" require "time" require "./base/model" @@ -7,7 +6,7 @@ require "./trigger/*" module PlaceOS::Model class Trigger < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :trigger diff --git a/src/placeos-models/trigger_instance.cr b/src/placeos-models/trigger_instance.cr index 12df0021..8230dcca 100644 --- a/src/placeos-models/trigger_instance.cr +++ b/src/placeos-models/trigger_instance.cr @@ -1,12 +1,11 @@ require "random" -require "rethinkdb-orm" require "time" require "./base/model" module PlaceOS::Model class TriggerInstance < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :trig @@ -88,11 +87,8 @@ module PlaceOS::Model # Increment the `trigger_count` of a `TriggerInstance` in place. # def self.increment_trigger_count(id : String) - TriggerInstance.table_query do |q| - q.get(id).update do |doc| - doc.merge({"trigger_count" => doc["trigger_count"].add(1)}) - end - end + inst = TriggerInstance.find(id) + inst.update(trigger_count: inst.trigger_count + 1) end # Proxied `Trigger` attributes diff --git a/src/placeos-models/user.cr b/src/placeos-models/user.cr index 12fec13b..02b74e0f 100644 --- a/src/placeos-models/user.cr +++ b/src/placeos-models/user.cr @@ -1,9 +1,6 @@ require "CrystalEmail" require "crypto/bcrypt/password" require "digest/md5" -require "rethinkdb-orm" -require "rethinkdb-orm/lock" - require "./base/model" require "./api_key" require "./metadata" @@ -12,14 +9,14 @@ require "./utilities/metadata_helper" module PlaceOS::Model class User < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::MetadataHelper table :user attribute name : String, es_subfield: "keyword" attribute nickname : String = "" - attribute email : Email = Email.new(""), es_type: "text" + attribute email : Email = Email.new(""), converter: PlaceOS::Model::EmailConverter, es_type: "text" attribute phone : String = "" attribute country : String = "" attribute image : String = "" @@ -54,8 +51,6 @@ module PlaceOS::Model # Association ################################################################################################ - secondary_index :authority_id - belongs_to Authority has_many( @@ -108,8 +103,8 @@ module PlaceOS::Model {authority_id, email_digest} end - ensure_unique :login_name - ensure_unique :staff_id + ensure_unique :login_name, scope: [:authority_id] + ensure_unique :staff_id, scope: [:authority_id] # Callbacks ############################################################################################### @@ -119,8 +114,8 @@ module PlaceOS::Model before_save :build_name before_save :write_email_fields - private getter admin_destroy_lock : RethinkORM::Lock do - RethinkORM::Lock.new("admin_destroy_lock") + private getter admin_destroy_lock : PgORM::PgAdvisoryLock do + PgORM::PgAdvisoryLock.new("admin_destroy_lock") end # :inherit: @@ -134,7 +129,7 @@ module PlaceOS::Model protected def ensure_admin_remains return unless self.sys_admin - if User.count(sys_admin: true) == 1 + if User.where({sys_admin: true}).count == 1 raise Model::Error.new("At least one admin must remain") end end @@ -144,23 +139,15 @@ module PlaceOS::Model user_id = self.id begin - ::RethinkORM::Connection.raw do |r| - r.table("doorkeeper_grant").filter { |grant| - grant["resource_owner_id"].eq(user_id) - }.delete - end + PgORM::Database.exec_sql("delete from \"doorkeeper_grant\" where resource_owner_id = $1", user_id) rescue error Log.warn(exception: error) { "failed to remove User<#{user_id}> auth grants" } end begin - ::RethinkORM::Connection.raw do |r| - r.table("doorkeeper_token").filter { |token| - token["resource_owner_id"].eq(user_id) - }.delete - end + PgORM::Database.exec_sql("delete from \"doorkeeper_token\" where resource_owner_id = $1", user_id) rescue error - Log.warn(exception: error) { "failed to remove User<#{user_id}> auth tokens" } + Log.warn(exception: error) { "failed to remove User<#{user_id}> auth token" } end end @@ -177,11 +164,9 @@ module PlaceOS::Model ############################################################################################### def by_authority_id(auth_id : String) - User.find_all([auth_id], index: :authority_id) + User.where(auth_id: auth_id) end - secondary_index :email_digest - def self.find_by_email(authority_id : String, email : PlaceOS::Model::Email | String) find_by_emails(authority_id, [email]).first? end @@ -194,45 +179,27 @@ module PlaceOS::Model email.digest end - User.collection_query do |table| - table - .get_all(digests, index: :email_digest) - .filter({authority_id: authority_id}) - end + User.where(email_digest: digests, authority_id: authority_id) end - secondary_index :login_name - def self.find_by_login_name(login_name : String) - User.find_all([login_name], index: :login_name).first? + User.where(login_name: login_name).first? end def self.find_by_login_name(authority_id : String, login_name : String) - User.collection_query do |table| - table - .get_all(login_name, index: :login_name) - .filter({authority_id: authority_id}) - end.first? + User.where({login_name: login_name, authority_id: authority_id}).first? end - secondary_index :staff_id - def self.find_by_staff_id(staff_id : String) - User.find_all([staff_id], index: :staff_id).first? + User.where(staff_id: staff_id).first? end def self.find_by_staff_id(authority_id : String, staff_id : String) - User.collection_query do |table| - table - .get_all(staff_id, index: :staff_id) - .filter({authority_id: authority_id}) - end.first? + User.where(staff_id: staff_id, authority_id: authority_id).first? end - secondary_index :sys_admin - def self.find_sys_admins - User.find_all([true], index: :sys_admin) + User.where(sys_admin: true) end # Access Control diff --git a/src/placeos-models/user_auth_lookup.cr b/src/placeos-models/user_auth_lookup.cr index d3e67c9f..3d61467b 100644 --- a/src/placeos-models/user_auth_lookup.cr +++ b/src/placeos-models/user_auth_lookup.cr @@ -1,11 +1,10 @@ -require "rethinkdb-orm" require "./base/model" require "./authority" require "./user" module PlaceOS::Model class UserAuthLookup < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps table :authentication @@ -14,6 +13,8 @@ module PlaceOS::Model # Association ############################################################################################### + attribute user_id : String? + attribute authority_id : String? belongs_to User belongs_to Authority @@ -24,7 +25,7 @@ module PlaceOS::Model before_create :generate_id protected def generate_id - self._new_flag = true + self.new_record = true self.id = "auth-#{self.authority_id}-#{self.provider}-#{self.uid}" end end diff --git a/src/placeos-models/utilities/id_generator.cr b/src/placeos-models/utilities/id_generator.cr new file mode 100644 index 00000000..506c5653 --- /dev/null +++ b/src/placeos-models/utilities/id_generator.cr @@ -0,0 +1,81 @@ +require "random" +require "time" + +# Generates time-sortable, collision resistant primary keys. +# +# Provides suitable performance with local, high-frequency batch insertions and +# distributed operation will low collision probability. Generated ID's are in +# the form `-` where prefix defaults to the table name and +# postfix is a lexicographically sortable 10 character unique identifier. +class PlaceOS::Model::Utilities::IdGenerator + ENCODING = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '-', '_', '~', + } + + # Coverage to 2050 in 30 bits + TIME_OFFSET = Time.utc(2014, 1, 1).to_unix + + # 30-bits of entropy + RAND_LEN = 30 + RAND_GEN = seq max: 2_u32**RAND_LEN + + # Provides a pseudo-random sequence of non-repeating positive integers. + # + # Internally this implements a maximal linear feedback shift register via + # Xorshift. For non-zero seeds this provides a cycle length of `2**32 - 1`. + private class LSFR + include Iterator(UInt32) + + def initialize(@seed : UInt32) + @state = @seed + end + + def next + @state ^= @state << 13 + @state ^= @state >> 17 + @state ^= @state << 5 + if @state == @seed + stop + else + @state + end + end + end + + # Provides a channel with an infinite stream of psuedo-random values up to + # *max* size and a guaranteed cycle of at least *max* samples. + private def self.seq(max = UInt32::MAX, r = Random::DEFAULT) + ch = Channel(UInt32).new + spawn do + loop do + mask = Random::Secure.rand max + lfsr = LSFR.new(r.rand UInt32) + lfsr.select(&.< max).each do |count| + ch.send count ^ mask + end + end + end + ch + end + + def self.next(model) + "#{model.table_name}-#{postfix}" + end + + def self.next(table_name : String) + "#{table_name}-#{postfix}" + end + + def self.postfix + String.build do |io| + time = Time.utc.to_unix - TIME_OFFSET + rand = RAND_GEN.receive + (time << RAND_LEN | rand).digits(ENCODING.size).reverse_each do |ord| + io << ENCODING[ord] + end + end + end +end diff --git a/src/placeos-models/utilities/last_modified.cr b/src/placeos-models/utilities/last_modified.cr index 75e98ff2..0617ea4c 100644 --- a/src/placeos-models/utilities/last_modified.cr +++ b/src/placeos-models/utilities/last_modified.cr @@ -1,5 +1,3 @@ -require "rethinkdb-orm" - require "../user" # Adds modification data to a `PlaceOS::Model` diff --git a/src/placeos-models/utilities/metadata_helper.cr b/src/placeos-models/utilities/metadata_helper.cr index d59ed75c..a6cdde35 100644 --- a/src/placeos-models/utilities/metadata_helper.cr +++ b/src/placeos-models/utilities/metadata_helper.cr @@ -1,5 +1,3 @@ -require "rethinkdb-orm" - require "../metadata" module PlaceOS::Model::Utilities diff --git a/src/placeos-models/utilities/settings_helper.cr b/src/placeos-models/utilities/settings_helper.cr index d9d581c8..41c03514 100644 --- a/src/placeos-models/utilities/settings_helper.cr +++ b/src/placeos-models/utilities/settings_helper.cr @@ -1,5 +1,3 @@ -require "rethinkdb-orm" - require "../settings" module PlaceOS::Model::Utilities diff --git a/src/placeos-models/utilities/versions.cr b/src/placeos-models/utilities/versions.cr index 85e2074a..7f63be89 100644 --- a/src/placeos-models/utilities/versions.cr +++ b/src/placeos-models/utilities/versions.cr @@ -1,16 +1,3 @@ -require "rethinkdb-orm" - -# TODO: Add `RethinkDB::StreamTerm#slice` to `crystal-rethinkdb` -class RethinkDB::StreamTerm - def slice(start) - StreamTerm.new(TermType::SLICE, [self, start]) - end - - def slice(start, size) - StreamTerm.new(TermType::SLICE, [self, start, size]) - end -end - # Adds version history to a `PlaceOS::Model` module PlaceOS::Model::Utilities::Versions # Number of version models to retain @@ -25,9 +12,8 @@ module PlaceOS::Model::Utilities::Versions # Associate with main version attribute {{ parent_id }} : String? - secondary_index {{ parent_id.symbolize }} + #secondary_index {{ parent_id.symbolize }} - # {{ @type }} self-referential entity relationship acts as a 2-level tree has_many( child_class: {{ @type }}, collection_name: {{ klass_name.stringify }}, @@ -57,6 +43,7 @@ module PlaceOS::Model::Utilities::Versions version = self.dup version.id = nil + version.new_record = true version.{{ parent_id }} = self.id create_version(version).save! @@ -66,12 +53,11 @@ module PlaceOS::Model::Utilities::Versions private def cleanup_history return if is_version? - - {{ @type }}.table_query do |q| - associated_version_query(q, &.itself) - .slice(MAX_VERSIONS) - .delete - end + query = {{@type}}.all + ids = associated_version_query(query, &.itself) + .offset(MAX_VERSIONS) + .pluck(:id) + {{@type}}.where(id: ids).delete_all unless ids.empty? end # Queries @@ -81,11 +67,10 @@ module PlaceOS::Model::Utilities::Versions # # Versions are in descending order of creation def history(offset : Int32 = 0, limit : Int32 = 10, &) - {{ @type }}.raw_query do |r| - associated_version_query(r.table({{ @type }}.table_name)) do |query_builder| - (yield query_builder).slice(offset, offset + limit) - end - end + query = {{@type}}.all + associated_version_query(query) do |query_builder| + (yield query_builder).offset(offset).limit(limit) + end.to_a end # :ditto: @@ -93,53 +78,51 @@ module PlaceOS::Model::Utilities::Versions history(offset, limit, &.itself) end - # Return the number of versions for the main document. + # Return the number of versions for the main record. # - # If the document is a version, this is always 0. + # If the record is a version, this is always 0. def history_count return 0 if is_version? - {{ @type }}.table_query do |q| - associated_version_query(q, &.itself).count - end.as_i + query = {{@type}}.all + associated_version_query(query, &.itself).unscope(:order).count end private def associated_version_query(query_builder) query_builder = query_builder - .get_all([id.as(String)], index: {{ parent_id.symbolize }}) + .where({{parent_id}}: id) query_builder = yield query_builder - query_builder.order_by(r.desc(:created_at)) + query_builder.order(created_at: :desc) end - # Query on main {{ klass_name }} documents + # Query on main {{ klass_name }} records # - # Gets documents where the {{ parent_id }} does not exist, i.e. is the main + # Gets records where the {{ parent_id }} does not exist, i.e. is the main def self.master_{{ klass_name }}_query(offset : Int32 = 0, limit : Int32 = 100) - raw_query do |q| - (yield q.table(table_name)) - .filter(&.has_fields({{ parent_id.symbolize }}).not) - .slice(offset, offset + limit) - end.to_a + query = {{@type}}.all + query = yield query + query = query.where({{parent_id}}: nil) + .limit(limit) + .offset(offset) + .to_a end - # Query on main {{ klass_name }} documents + # Query on main {{ klass_name }} records # - # Gets documents where the {{ parent_id }} does not exist, i.e. is the main + # Gets records where the {{ parent_id }} does not exist, i.e. is the main def self.master_{{ klass_name }}_raw_query(count : Bool = false) - ::RethinkORM::Connection.raw do |q| - (yield q.table(table_name)) - .filter(&.has_fields({{ parent_id.symbolize }}).not) - end + query = {{@type}}.all + query = yield query + query = query.where({{parent_id}}: nil) end - # Count of documents returned by query on main {{ klass_name }} documents + # Count of records returned by query on main {{ klass_name }} records # - # Gets documents where the {{ parent_id }} does not exist, i.e. is the main + # Gets records where the {{ parent_id }} does not exist, i.e. is the main def self.master_{{ klass_name }}_query_count - ::RethinkORM::Connection.raw do |q| - (yield q.table(table_name)) - .filter(&.has_fields({{ parent_id.symbolize }}).not) - .count - end.as_i + query = {{@type}}.all + query = yield query + query = query.where({{parent_id}}: nil) + .count end end end diff --git a/src/placeos-models/zone.cr b/src/placeos-models/zone.cr index 5e4e60ed..185a2ed6 100644 --- a/src/placeos-models/zone.cr +++ b/src/placeos-models/zone.cr @@ -1,4 +1,3 @@ -require "rethinkdb-orm" require "time" require "./base/model" @@ -9,7 +8,7 @@ require "./converter/time_location" module PlaceOS::Model class Zone < ModelBase - include RethinkORM::Timestamps + include PlaceOS::Model::Timestamps include Utilities::SettingsHelper include Utilities::MetadataHelper @@ -45,7 +44,6 @@ module PlaceOS::Model # Association ############################################################################################### - belongs_to Zone, foreign_key: "parent_id", association_name: "parent" has_many( @@ -158,7 +156,7 @@ module PlaceOS::Model end def self.with_tag(tag : String) - Zone.collection_query &.filter(&.["tags"].contains(tag)) + Zone.where("$1 = Any(tags)", tag) end # TODO: Implement multiple element `contains` in crystal-rethinkdb