Merge pull request #1288 from shangyian/alembic-migrations-release

Include alembic migrations as part of datajunction-server release
DataJunction · Jan 24, 2025 · f69a5f9 · f69a5f9
2 parents 2f0a28f + 5a9ce33
commit f69a5f9
Show file tree

Hide file tree

Showing 25 changed files with 32 additions and 41 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -91,7 +91,7 @@ jobs:
 
           # Run tests
           export MODULE=${{ matrix.library == 'server' && 'datajunction_server' || matrix.library == 'client' && 'datajunction' || matrix.library == 'djqs' && 'djqs' || matrix.library == 'djrs' && 'datajunction_reflection'}}
-          pdm run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration
+          pdm run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py
 
   build-javascript:
     runs-on: ubuntu-latest

diff --git a/datajunction-server/.coveragerc b/datajunction-server/.coveragerc
@@ -11,6 +11,7 @@ omit =
     */datajunction_server/api/access/authentication/google.py
     */datajunction_server/api/djsql.py
     */datajunction_server/construction/dj_query.py
+    */datajunction_server/alembic/*
 
 [paths]
 source =

diff --git a/datajunction-server/alembic.ini → ...on-server/datajunction_server/alembic.ini b/datajunction-server/alembic.ini → ...on-server/datajunction_server/alembic.ini
diff --git a/datajunction-server/alembic/README → ...server/datajunction_server/alembic/README b/datajunction-server/alembic/README → ...server/datajunction_server/alembic/README
diff --git a/datajunction-server/alembic/env.py → ...server/datajunction_server/alembic/env.py b/datajunction-server/alembic/env.py → ...server/datajunction_server/alembic/env.py
@@ -3,11 +3,12 @@
 """
 # pylint: disable=no-member, unused-import, no-name-in-module, import-error
 
+import os
 from logging.config import fileConfig
 
+import alembic
 from sqlalchemy import create_engine
 
-from alembic import context
 from datajunction_server.database import (
     AttributeType,
     Catalog,
@@ -29,11 +30,14 @@
 from datajunction_server.database.base import Base
 from datajunction_server.database.column import Column
 
-DEFAULT_URI = "postgresql+psycopg://dj:dj@postgres_metadata:5432/dj"
+DEFAULT_URI = os.getenv(
+    "DATABASE_URI",
+    "postgresql+psycopg://dj:dj@postgres_metadata:5432/dj",
+)
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
-config = context.config
+config = alembic.context.config
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
@@ -64,17 +68,17 @@ def run_migrations_offline():
     script output.
 
     """
-    x_args = context.get_x_argument(as_dictionary=True)
-    context.configure(
+    x_args = alembic.context.get_x_argument(as_dictionary=True)
+    alembic.context.configure(
         url=x_args.get("uri") or DEFAULT_URI,
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
         render_as_batch=True,
     )
 
-    with context.begin_transaction():
-        context.run_migrations()
+    with alembic.context.begin_transaction():
+        alembic.context.run_migrations()
 
 
 def run_migrations_online():
@@ -84,21 +88,21 @@ def run_migrations_online():
     and associate a connection with the context.
 
     """
-    x_args = context.get_x_argument(as_dictionary=True)
+    x_args = alembic.context.get_x_argument(as_dictionary=True)
     connectable = create_engine(x_args.get("uri") or DEFAULT_URI)
 
     with connectable.connect() as connection:
-        context.configure(
+        alembic.context.configure(
             connection=connection,
             target_metadata=target_metadata,
             render_as_batch=True,
         )
 
-        with context.begin_transaction():
-            context.run_migrations()
+        with alembic.context.begin_transaction():
+            alembic.context.run_migrations()
 
 
-if context.is_offline_mode():
+if alembic.context.is_offline_mode():
     run_migrations_offline()
 else:
     run_migrations_online()
diff --git a/datajunction-server/alembic/script.py.mako → ...atajunction_server/alembic/script.py.mako b/datajunction-server/alembic/script.py.mako → ...atajunction_server/alembic/script.py.mako
diff --git a/...20_1829-724445d2b29d_initial_migration.py → ...20_1829-724445d2b29d_initial_migration.py b/...20_1829-724445d2b29d_initial_migration.py → ...20_1829-724445d2b29d_initial_migration.py
@@ -9,7 +9,6 @@
 
 import sqlalchemy as sa
 import sqlalchemy_utils
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/..._2034-945d44abcd32_add_dimension_links.py → ..._2034-945d44abcd32_add_dimension_links.py b/..._2034-945d44abcd32_add_dimension_links.py → ..._2034-945d44abcd32_add_dimension_links.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/..._11_2032-c74b11566d82_add_column_order.py → ..._11_2032-c74b11566d82_add_column_order.py b/..._11_2032-c74b11566d82_add_column_order.py → ..._11_2032-c74b11566d82_add_column_order.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "c74b11566d82"

diff --git a/...8_2011-20f060b02772_switch_enum_values.py → ...8_2011-20f060b02772_switch_enum_values.py b/...8_2011-20f060b02772_switch_enum_values.py → ...8_2011-20f060b02772_switch_enum_values.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "20f060b02772"

diff --git a/...add_missing_table_attribute_to_source_.py → ...add_missing_table_attribute_to_source_.py b/...add_missing_table_attribute_to_source_.py → ...add_missing_table_attribute_to_source_.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...availability_state_s_valid_through_ts_.py → ...availability_state_s_valid_through_ts_.py b/...availability_state_s_valid_through_ts_.py → ...availability_state_s_valid_through_ts_.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "a8e22109be24"

diff --git a/...cc3_cascade_deletes_to_dimension_links.py → ...cc3_cascade_deletes_to_dimension_links.py b/...cc3_cascade_deletes_to_dimension_links.py → ...cc3_cascade_deletes_to_dimension_links.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "d61fb7e48cc3"

diff --git a/...0_1556-de7ec1c82fe0_add_query_requests.py → ...0_1556-de7ec1c82fe0_add_query_requests.py b/...0_1556-de7ec1c82fe0_add_query_requests.py → ...0_1556-de7ec1c82fe0_add_query_requests.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "de7ec1c82fe0"

diff --git a/...1420-9b1227ff17f4_update_backfill_spec.py → ...1420-9b1227ff17f4_update_backfill_spec.py b/...1420-9b1227ff17f4_update_backfill_spec.py → ...1420-9b1227ff17f4_update_backfill_spec.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "9b1227ff17f4"

diff --git a/...c93ef6947_add_query_id_to_queryrequest.py → ...c93ef6947_add_query_id_to_queryrequest.py b/...c93ef6947_add_query_id_to_queryrequest.py → ...c93ef6947_add_query_id_to_queryrequest.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...301-640a814db2d8_add_collection_tables.py → ...301-640a814db2d8_add_collection_tables.py b/...301-640a814db2d8_add_collection_tables.py → ...301-640a814db2d8_add_collection_tables.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...c92dd6d_set_user_username_to_be_unique.py → ...c92dd6d_set_user_username_to_be_unique.py b/...c92dd6d_set_user_username_to_be_unique.py → ...c92dd6d_set_user_username_to_be_unique.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...add_create_by_to_nodes_node_revisions_.py → ...add_create_by_to_nodes_node_revisions_.py b/...add_create_by_to_nodes_node_revisions_.py → ...add_create_by_to_nodes_node_revisions_.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...d_a_map_of_links_to_availability_state.py → ...d_a_map_of_links_to_availability_state.py b/...d_a_map_of_links_to_availability_state.py → ...d_a_map_of_links_to_availability_state.py
@@ -8,7 +8,6 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

diff --git a/...add_indexes_on_history_and_node_tables.py → ...add_indexes_on_history_and_node_tables.py b/...add_indexes_on_history_and_node_tables.py → ...add_indexes_on_history_and_node_tables.py
@@ -8,9 +8,8 @@
 # pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
 
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "70904373eab3"

diff --git a/datajunction-server/pyproject.toml b/datajunction-server/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [tool.hatch.build.targets.wheel]
 packages = ["datajunction_server"]
-
+include = ["alembic/**", "alembic.ini"]
 
 [tool.pdm]
 [tool.pdm.build]

diff --git a/datajunction-server/tests/migrations_test.py b/datajunction-server/tests/migrations_test.py
@@ -1,14 +1,14 @@
 """Verify alembic migrations."""
 import pytest
-from sqlalchemy import create_engine
-from sqlalchemy.engine.base import Connection
-from testcontainers.postgres import PostgresContainer
-
 from alembic.autogenerate import compare_metadata
 from alembic.config import Config
 from alembic.runtime.environment import EnvironmentContext
 from alembic.runtime.migration import MigrationContext
 from alembic.script import ScriptDirectory
+from sqlalchemy import create_engine
+from sqlalchemy.engine.base import Connection
+from testcontainers.postgres import PostgresContainer
+
 from datajunction_server.database.base import Base
 
 
@@ -34,7 +34,7 @@ def test_migrations_are_current(connection):  # pylint: disable=redefined-outer-
     target_metadata = Base.metadata
 
     config = Config("alembic.ini")
-    config.set_main_option("script_location", "alembic")
+    config.set_main_option("script_location", "datajunction_server/alembic")
     script = ScriptDirectory.from_config(config)
 
     context = EnvironmentContext(

diff --git a/datajunction-server/tests/pytest.ini b/datajunction-server/tests/pytest.ini
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -97,6 +97,7 @@ services:
       context: ./datajunction-server
     volumes:
       - ./datajunction-server:/code
+    working_dir: /code/datajunction_server
     command: /bin/bash -c "alembic -x uri="postgresql+psycopg://dj:dj@postgres_metadata:5432/dj" upgrade head"
     restart: on-failure
     depends_on: