Support launching images via fly.io (#281)

- **In python images, refactor to call wait-for-postgres as entrypoint** - **Consolidate migration image into API image**
shaldengeki · Jul 23, 2024 · ed918ca · ed918ca
1 parent 1edfefd
commit ed918ca
Show file tree

Hide file tree

Showing 41 changed files with 236 additions and 327 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -33,6 +33,22 @@ jobs:
         uses: actions/checkout@v4
       - name: bazel test //...
         run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //...
+  push_ark_nova_stats:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Login to Docker Hub
+        if: ${{ github.ref == 'refs/heads/main' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push to Docker Hub
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //ark_nova_stats/api:api_image_image_dockerhub
   push_fitbit_challenges:
     needs: build
     runs-on: ubuntu-latest
@@ -49,7 +65,6 @@ jobs:
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/api:api_image_image_dockerhub
-          bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/api/migrations:image_dockerhub
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/worker:image_dockerhub
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/frontend:production_cross_platform_image_dockerhub
   push_home_api:
@@ -68,7 +83,6 @@ jobs:
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/api:api_image_image_dockerhub
-          bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/api/migrations:image_dockerhub
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/frontend:production_cross_platform_image_dockerhub
   push_mc_manager:
     needs: build
@@ -86,6 +100,5 @@ jobs:
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/api:api_image_image_dockerhub
-          bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/api/migrations:image_dockerhub
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/worker:image_dockerhub
           bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/frontend:production_cross_platform_image_dockerhub
diff --git a/ark_nova_stats/api/migrations/BUILD.bazel b/ark_nova_stats/api/migrations/BUILD.bazel
@@ -1,14 +1,13 @@
 load("@py_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
-load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image")
-load("//tools/build_rules:py_layer.bzl", "py_oci_image")
 
 py_library(
     name = "migrate_lib",
     srcs = glob(["**/*.py"]),  # keep
     imports = [".."],
     visibility = ["//:__subpackages__"],
     deps = [
+        "//base:flask_app_py",
         requirement("alembic"),
         requirement("Flask"),
     ],
@@ -20,52 +19,16 @@ py_binary(
     data = ["alembic.ini"],
     imports = [".."],
     main = "__main__.py",
-    visibility = ["//:__subpackages__"],
+    visibility = ["//ark_nova_stats/api:__subpackages__"],
     deps = [
         "//ark_nova_stats:config_py",
+        "//base:flask_app_py",
         "//scripts:wait_for_postgres",  # keep
         "@py_deps//flask_migrate",
         "@rules_python//python/runfiles",
     ],
 )
 
-py_oci_image(
-    name = "base_image",
-    base = "@python3_image",
-    binary = ":binary",
-    cmd = [
-        "/ark_nova_stats/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres",
-        "/ark_nova_stats/api/migrations/binary",
-    ],
-    env = {
-        "FLASK_APP": "app.py",
-        "FLASK_DEBUG": "True",
-        "API_PORT": "5000",
-        "FRONTEND_PROTOCOL": "http",
-        "FRONTEND_HOST": "frontend",
-        "FRONTEND_PORT": "5001",
-        "DB_HOST": "pg",
-        "DB_USERNAME": "admin",
-        "DB_PASSWORD": "development",
-        "DATABASE_NAME": "api_development",
-        "FITBIT_CLIENT_ID": "testing",
-        "FITBIT_CLIENT_SECRET": "testing",
-        "FITBIT_VERIFICATION_CODE": "testing",
-        "FLASK_SECRET_KEY": "testing",
-    },
-    tags = ["manual"],
-)
-
-# $ bazel run //ark_nova_stats/api/migrations:image_tarball
-# $ docker run --rm shaldengeki/ark-nova-stats-api-migrations:latest
-cross_platform_image(
-    name = "image",
-    image = ":base_image",
-    repo_tags = ["shaldengeki/ark-nova-stats-api-migrations:latest"],
-    repository = "docker.io/shaldengeki/ark-nova-stats-api-migrations",
-    visibility = ["//ark_nova_stats/api/migrations:__subpackages__"],
-)
-
 py_library(
     name = "env",
     srcs = ["env.py"],

diff --git a/ark_nova_stats/api/migrations/__main__.py b/ark_nova_stats/api/migrations/__main__.py
@@ -1,15 +1,15 @@
 import shutil
 
 from flask_migrate import upgrade
-from python.runfiles import Runfiles
 
 from ark_nova_stats.config import app
 
 if __name__ == "__main__":
     # Copy the alembic.ini.
-    r = Runfiles.Create()
-    alembic_ini_src = r.Rlocation("_main/ark_nova_stats/api/migrations/alembic.ini")
-    shutil.copyfile(alembic_ini_src, "/ark_nova_stats/api/migrations/alembic.ini")
+    shutil.copyfile(
+        "/ark_nova_stats/api/migrations/binary.runfiles/_main/ark_nova_stats/api/migrations/alembic.ini",
+        "/ark_nova_stats/api/migrations/alembic.ini",
+    )
 
     with app.app_context():
         upgrade(directory="/ark_nova_stats/api/migrations")
diff --git a/ark_nova_stats/api/migrations/env.py b/ark_nova_stats/api/migrations/env.py
@@ -5,6 +5,8 @@
 from alembic import context
 from flask import current_app
 
+from base.flask_app import database_uri
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -26,7 +28,7 @@ def get_engine():
 
 def get_engine_url():
     # Set the database URL directly from environment variables.
-    return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}"
+    return database_uri().replace("postgresql+pg8000://", "postgres://")
     # try:
     #     return get_engine().url.render_as_string(hide_password=False).replace(
     #         '%', '%%')

diff --git a/ark_nova_stats/docker-compose.override.yaml b/ark_nova_stats/docker-compose.override.yaml
@@ -7,7 +7,6 @@ services:
       - "5000:5000"
   migration:
     <<: *api
-    image: shaldengeki/ark-nova-stats-api-migrations
     ports: []
   worker:
     <<: *api

diff --git a/ark_nova_stats/docker-compose.prod.yaml b/ark_nova_stats/docker-compose.prod.yaml
@@ -7,7 +7,6 @@ services:
       - env/.api.prod.env
   migration:
     <<: *api
-    image: shaldengeki/ark-nova-stats-api-migrations:latest
     ports: []
   worker:
     <<: *api

diff --git a/ark_nova_stats/docker-compose.yaml b/ark_nova_stats/docker-compose.yaml
@@ -9,7 +9,7 @@ services:
       - pg
   migration:
     <<: *api
-    image: shaldengeki/ark-nova-stats-api-migrations:latest
+    command: [ "/ark_nova_stats/api/migrations/binary"]
     restart: no
   worker:
     <<: *api

diff --git a/base/flask_app.py b/base/flask_app.py
@@ -8,6 +8,20 @@
 from flask_sqlalchemy import SQLAlchemy
 
 
+def database_uri() -> str:
+    if os.getenv("DATABASE_URL", None) is not None:
+        return os.getenv("DATABASE_URL", "").replace(
+            "postgres://", "postgresql+pg8000://"
+        )
+    else:
+        return "postgresql+pg8000://{user}:{password}@{host}/{db}".format(
+            user=os.getenv("DB_USERNAME", "admin"),
+            password=os.getenv("DB_PASSWORD", "development"),
+            host=os.getenv("DB_HOST", "pg"),
+            db=os.getenv("DATABASE_NAME", "api_development"),
+        )
+
+
 def FlaskApp(name) -> tuple[Flask, CORS, SQLAlchemy, Migrate]:
     app = Flask(name)
 
@@ -24,12 +38,7 @@ def FlaskApp(name) -> tuple[Flask, CORS, SQLAlchemy, Migrate]:
     app.config.update(
         FRONTEND_URL=frontend_url,
         SECRET_KEY=os.getenv("FLASK_SECRET_KEY", "testing"),
-        SQLALCHEMY_DATABASE_URI="postgresql+pg8000://{user}:{password}@{host}/{db}".format(
-            user=os.getenv("DB_USERNAME", "admin"),
-            password=os.getenv("DB_PASSWORD", "development"),
-            host=os.getenv("DB_HOST", "pg"),
-            db=os.getenv("DATABASE_NAME", "api_development"),
-        ),
+        SQLALCHEMY_DATABASE_URI=database_uri(),
         SESSION_REFRESH_EACH_REQUEST=True,
         PERMANENT_SESSION_LIFETIME=datetime.timedelta(days=365),
     )

diff --git a/fitbit_challenges/api/migrations/BUILD.bazel b/fitbit_challenges/api/migrations/BUILD.bazel
@@ -1,14 +1,13 @@
 load("@py_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_binary", "py_library")
-load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image")
-load("//tools/build_rules:py_layer.bzl", "py_oci_image")
 
 py_library(
     name = "migrate_lib",
     srcs = glob(["**/*.py"]),  # keep
     imports = [".."],
     visibility = ["//:__subpackages__"],
     deps = [
+        "//base:flask_app_py",
         requirement("alembic"),
         requirement("Flask"),
     ],
@@ -20,47 +19,12 @@ py_binary(
     data = ["alembic.ini"],
     imports = [".."],
     main = "__main__.py",
+    visibility = ["//fitbit_challenges/api:__subpackages__"],
     deps = [
+        "//base:flask_app_py",
         "//fitbit_challenges:config_py",
         "//scripts:wait_for_postgres",
         "@rules_python//python/runfiles",
         requirement("Flask-Migrate"),
     ],
 )
-
-py_oci_image(
-    name = "base_image",
-    base = "@python3_image",
-    binary = ":binary",
-    cmd = [
-        "/fitbit_challenges/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres",
-        "/fitbit_challenges/api/migrations/binary",
-    ],
-    env = {
-        "FLASK_APP": "app.py",
-        "FLASK_DEBUG": "True",
-        "API_PORT": "5000",
-        "FRONTEND_PROTOCOL": "http",
-        "FRONTEND_HOST": "frontend",
-        "FRONTEND_PORT": "5001",
-        "DB_HOST": "pg",
-        "DB_USERNAME": "admin",
-        "DB_PASSWORD": "development",
-        "DATABASE_NAME": "api_development",
-        "FITBIT_CLIENT_ID": "testing",
-        "FITBIT_CLIENT_SECRET": "testing",
-        "FITBIT_VERIFICATION_CODE": "testing",
-        "FLASK_SECRET_KEY": "testing",
-    },
-    tags = ["manual"],
-)
-
-# $ bazel run //fitbit_challenges/api/migrations:image_tarball
-# $ docker run --rm shaldengeki/fitbit-challenges-api-migrations:latest
-cross_platform_image(
-    name = "image",
-    image = ":base_image",
-    repo_tags = ["shaldengeki/fitbit-challenges-migrations:latest"],
-    repository = "docker.io/shaldengeki/fitbit-challenges-migrations",
-    visibility = ["//fitbit_challenges/api/migrations:__subpackages__"],
-)
diff --git a/fitbit_challenges/api/migrations/__main__.py b/fitbit_challenges/api/migrations/__main__.py
@@ -1,15 +1,15 @@
 import shutil
 
 from flask_migrate import upgrade
-from python.runfiles import Runfiles
 
 from fitbit_challenges.config import app
 
 if __name__ == "__main__":
     # Copy the alembic.ini.
-    r = Runfiles.Create()
-    alembic_ini_src = r.Rlocation("_main/fitbit_challenges/api/migrations/alembic.ini")
-    shutil.copyfile(alembic_ini_src, "/fitbit_challenges/api/migrations/alembic.ini")
+    shutil.copyfile(
+        "/fitbit_challenges/api/migrations/binary.runfiles/_main/fitbit_challenges/api/migrations/alembic.ini",
+        "/fitbit_challenges/api/migrations/alembic.ini",
+    )
 
     with app.app_context():
         upgrade(directory="/fitbit_challenges/api/migrations")
diff --git a/fitbit_challenges/api/migrations/env.py b/fitbit_challenges/api/migrations/env.py
@@ -5,6 +5,8 @@
 from alembic import context
 from flask import current_app
 
+from base.flask_app import database_uri
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -26,7 +28,7 @@ def get_engine():
 
 def get_engine_url():
     # Set the database URL directly from environment variables.
-    return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}"
+    return database_uri().replace("postgresql+pg8000://", "postgres://")
     # try:
     #     return get_engine().url.render_as_string(hide_password=False).replace(
     #         '%', '%%')

diff --git a/fitbit_challenges/docker-compose.override.yaml b/fitbit_challenges/docker-compose.override.yaml
@@ -7,7 +7,6 @@ services:
       - "5000:5000"
   migration:
     <<: *api
-    image: shaldengeki/fitbit-challenges-api-migrations
     ports: []
   worker:
     <<: *api

diff --git a/fitbit_challenges/docker-compose.prod.yaml b/fitbit_challenges/docker-compose.prod.yaml
@@ -7,7 +7,6 @@ services:
       - env/.api.prod.env
   migration:
     <<: *api
-    image: shaldengeki/fitbit-challenges-api-migrations:latest
     ports: []
   worker:
     <<: *api

diff --git a/fitbit_challenges/docker-compose.yaml b/fitbit_challenges/docker-compose.yaml
@@ -9,7 +9,7 @@ services:
       - pg
   migration:
     <<: *api
-    image: shaldengeki/fitbit-challenges-api-migrations:latest
+    command: [ "/fitbit_challenges/api/migrations/binary"]
     restart: no
   worker:
     <<: *api

diff --git a/fitbit_challenges/worker/BUILD.bazel b/fitbit_challenges/worker/BUILD.bazel
@@ -25,9 +25,12 @@ py_binary(
 py_oci_image(
     name = "base_image",
     base = "@python3_image",
-    binary = ":binary",
+    binaries = [
+        "//scripts:wait_for_postgres",
+        ":binary",
+    ],
     cmd = [
-        "/fitbit_challenges/worker/binary.runfiles/_main/scripts/wait_for_postgres",
+        "/scripts/wait_for_postgres",
         "/fitbit_challenges/worker/binary",
     ],
     env = {