From ed918ca6bfbc0201e65729d20129b2e07bc0c3a9 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Tue, 23 Jul 2024 00:46:18 -0400 Subject: [PATCH] Support launching images via fly.io (#281) - **In python images, refactor to call wait-for-postgres as entrypoint** - **Consolidate migration image into API image** --- .github/workflows/main.yml | 19 ++++++-- ark_nova_stats/api/migrations/BUILD.bazel | 43 ++----------------- ark_nova_stats/api/migrations/__main__.py | 8 ++-- ark_nova_stats/api/migrations/env.py | 4 +- ark_nova_stats/docker-compose.override.yaml | 1 - ark_nova_stats/docker-compose.prod.yaml | 1 - ark_nova_stats/docker-compose.yaml | 2 +- base/flask_app.py | 21 ++++++--- fitbit_challenges/api/migrations/BUILD.bazel | 42 ++---------------- fitbit_challenges/api/migrations/__main__.py | 8 ++-- fitbit_challenges/api/migrations/env.py | 4 +- .../docker-compose.override.yaml | 1 - fitbit_challenges/docker-compose.prod.yaml | 1 - fitbit_challenges/docker-compose.yaml | 2 +- fitbit_challenges/worker/BUILD.bazel | 7 ++- home_api/api/migrations/BUILD.bazel | 37 +--------------- home_api/api/migrations/__main__.py | 8 ++-- home_api/docker-compose.override.yaml | 19 ++++++++ home_api/docker-compose.yaml | 2 +- mc_manager/api/migrations/BUILD.bazel | 39 ++--------------- mc_manager/api/migrations/__main__.py | 8 ++-- mc_manager/api/migrations/env.py | 4 +- mc_manager/docker-compose.override.yaml | 25 +++++++++++ mc_manager/docker-compose.yaml | 2 +- mc_manager/worker/BUILD.bazel | 21 +++++---- mc_manager/worker/tests/BUILD.bazel | 18 +++----- proto_registry/api/migrations/BUILD.bazel | 39 ++--------------- proto_registry/api/migrations/__main__.py | 8 ++-- proto_registry/api/migrations/env.py | 4 +- proto_registry/docker-compose.override.yaml | 13 ++++++ proto_registry/docker-compose.yaml | 2 +- scripts/BUILD.bazel | 1 + scripts/rebuild-and-run | 1 - scripts/wait_for_postgres.py | 38 +++++++++++++--- skeleton/api/migrations/BUILD.bazel | 41 +----------------- skeleton/api/migrations/__main__.py | 8 ++-- skeleton/docker-compose.prod.yaml | 1 - skeleton/docker-compose.yaml | 2 +- tools/build_rules/api_image.bzl | 14 +++++- tools/build_rules/py_layer.bzl | 30 +++++++++---- tools/build_rules/templates/__main__.py.tpl | 14 ------ 41 files changed, 236 insertions(+), 327 deletions(-) create mode 100644 home_api/docker-compose.override.yaml create mode 100644 mc_manager/docker-compose.override.yaml create mode 100644 proto_registry/docker-compose.override.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0e3bbd5..139124f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,6 +33,22 @@ jobs: uses: actions/checkout@v4 - name: bazel test //... run: bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test //... + push_ark_nova_stats: + needs: build + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Login to Docker Hub + if: ${{ github.ref == 'refs/heads/main' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push to Docker Hub + if: ${{ github.ref == 'refs/heads/main' }} + run: | + bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //ark_nova_stats/api:api_image_image_dockerhub push_fitbit_challenges: needs: build runs-on: ubuntu-latest @@ -49,7 +65,6 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: | bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/api:api_image_image_dockerhub - bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/api/migrations:image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/worker:image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //fitbit_challenges/frontend:production_cross_platform_image_dockerhub push_home_api: @@ -68,7 +83,6 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: | bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/api:api_image_image_dockerhub - bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/api/migrations:image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //home_api/frontend:production_cross_platform_image_dockerhub push_mc_manager: needs: build @@ -86,6 +100,5 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} run: | bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/api:api_image_image_dockerhub - bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/api/migrations:image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/worker:image_dockerhub bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc run --build_tag_filters=manual --stamp --embed_label $(git rev-parse HEAD) //mc_manager/frontend:production_cross_platform_image_dockerhub diff --git a/ark_nova_stats/api/migrations/BUILD.bazel b/ark_nova_stats/api/migrations/BUILD.bazel index 57b58451..b9d788c6 100644 --- a/ark_nova_stats/api/migrations/BUILD.bazel +++ b/ark_nova_stats/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -9,6 +7,7 @@ py_library( imports = [".."], visibility = ["//:__subpackages__"], deps = [ + "//base:flask_app_py", requirement("alembic"), requirement("Flask"), ], @@ -20,52 +19,16 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", - visibility = ["//:__subpackages__"], + visibility = ["//ark_nova_stats/api:__subpackages__"], deps = [ "//ark_nova_stats:config_py", + "//base:flask_app_py", "//scripts:wait_for_postgres", # keep "@py_deps//flask_migrate", "@rules_python//python/runfiles", ], ) -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/ark_nova_stats/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/ark_nova_stats/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FITBIT_CLIENT_ID": "testing", - "FITBIT_CLIENT_SECRET": "testing", - "FITBIT_VERIFICATION_CODE": "testing", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //ark_nova_stats/api/migrations:image_tarball -# $ docker run --rm shaldengeki/ark-nova-stats-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/ark-nova-stats-api-migrations:latest"], - repository = "docker.io/shaldengeki/ark-nova-stats-api-migrations", - visibility = ["//ark_nova_stats/api/migrations:__subpackages__"], -) - py_library( name = "env", srcs = ["env.py"], diff --git a/ark_nova_stats/api/migrations/__main__.py b/ark_nova_stats/api/migrations/__main__.py index 972889e5..d8250b72 100644 --- a/ark_nova_stats/api/migrations/__main__.py +++ b/ark_nova_stats/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from ark_nova_stats.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/ark_nova_stats/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/ark_nova_stats/api/migrations/alembic.ini") + shutil.copyfile( + "/ark_nova_stats/api/migrations/binary.runfiles/_main/ark_nova_stats/api/migrations/alembic.ini", + "/ark_nova_stats/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/ark_nova_stats/api/migrations") diff --git a/ark_nova_stats/api/migrations/env.py b/ark_nova_stats/api/migrations/env.py index 78d62879..8035c795 100644 --- a/ark_nova_stats/api/migrations/env.py +++ b/ark_nova_stats/api/migrations/env.py @@ -5,6 +5,8 @@ from alembic import context from flask import current_app +from base.flask_app import database_uri + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -26,7 +28,7 @@ def get_engine(): def get_engine_url(): # Set the database URL directly from environment variables. - return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}" + return database_uri().replace("postgresql+pg8000://", "postgres://") # try: # return get_engine().url.render_as_string(hide_password=False).replace( # '%', '%%') diff --git a/ark_nova_stats/docker-compose.override.yaml b/ark_nova_stats/docker-compose.override.yaml index 8b7c989b..45eb2144 100644 --- a/ark_nova_stats/docker-compose.override.yaml +++ b/ark_nova_stats/docker-compose.override.yaml @@ -7,7 +7,6 @@ services: - "5000:5000" migration: <<: *api - image: shaldengeki/ark-nova-stats-api-migrations ports: [] worker: <<: *api diff --git a/ark_nova_stats/docker-compose.prod.yaml b/ark_nova_stats/docker-compose.prod.yaml index c091a888..e5cbea4f 100644 --- a/ark_nova_stats/docker-compose.prod.yaml +++ b/ark_nova_stats/docker-compose.prod.yaml @@ -7,7 +7,6 @@ services: - env/.api.prod.env migration: <<: *api - image: shaldengeki/ark-nova-stats-api-migrations:latest ports: [] worker: <<: *api diff --git a/ark_nova_stats/docker-compose.yaml b/ark_nova_stats/docker-compose.yaml index 7cb28a44..cf669319 100644 --- a/ark_nova_stats/docker-compose.yaml +++ b/ark_nova_stats/docker-compose.yaml @@ -9,7 +9,7 @@ services: - pg migration: <<: *api - image: shaldengeki/ark-nova-stats-api-migrations:latest + command: [ "/ark_nova_stats/api/migrations/binary"] restart: no worker: <<: *api diff --git a/base/flask_app.py b/base/flask_app.py index 6ce9bb84..40b7981c 100644 --- a/base/flask_app.py +++ b/base/flask_app.py @@ -8,6 +8,20 @@ from flask_sqlalchemy import SQLAlchemy +def database_uri() -> str: + if os.getenv("DATABASE_URL", None) is not None: + return os.getenv("DATABASE_URL", "").replace( + "postgres://", "postgresql+pg8000://" + ) + else: + return "postgresql+pg8000://{user}:{password}@{host}/{db}".format( + user=os.getenv("DB_USERNAME", "admin"), + password=os.getenv("DB_PASSWORD", "development"), + host=os.getenv("DB_HOST", "pg"), + db=os.getenv("DATABASE_NAME", "api_development"), + ) + + def FlaskApp(name) -> tuple[Flask, CORS, SQLAlchemy, Migrate]: app = Flask(name) @@ -24,12 +38,7 @@ def FlaskApp(name) -> tuple[Flask, CORS, SQLAlchemy, Migrate]: app.config.update( FRONTEND_URL=frontend_url, SECRET_KEY=os.getenv("FLASK_SECRET_KEY", "testing"), - SQLALCHEMY_DATABASE_URI="postgresql+pg8000://{user}:{password}@{host}/{db}".format( - user=os.getenv("DB_USERNAME", "admin"), - password=os.getenv("DB_PASSWORD", "development"), - host=os.getenv("DB_HOST", "pg"), - db=os.getenv("DATABASE_NAME", "api_development"), - ), + SQLALCHEMY_DATABASE_URI=database_uri(), SESSION_REFRESH_EACH_REQUEST=True, PERMANENT_SESSION_LIFETIME=datetime.timedelta(days=365), ) diff --git a/fitbit_challenges/api/migrations/BUILD.bazel b/fitbit_challenges/api/migrations/BUILD.bazel index e4fb47b4..86c03386 100644 --- a/fitbit_challenges/api/migrations/BUILD.bazel +++ b/fitbit_challenges/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -9,6 +7,7 @@ py_library( imports = [".."], visibility = ["//:__subpackages__"], deps = [ + "//base:flask_app_py", requirement("alembic"), requirement("Flask"), ], @@ -20,47 +19,12 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", + visibility = ["//fitbit_challenges/api:__subpackages__"], deps = [ + "//base:flask_app_py", "//fitbit_challenges:config_py", "//scripts:wait_for_postgres", "@rules_python//python/runfiles", requirement("Flask-Migrate"), ], ) - -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/fitbit_challenges/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/fitbit_challenges/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FITBIT_CLIENT_ID": "testing", - "FITBIT_CLIENT_SECRET": "testing", - "FITBIT_VERIFICATION_CODE": "testing", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //fitbit_challenges/api/migrations:image_tarball -# $ docker run --rm shaldengeki/fitbit-challenges-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/fitbit-challenges-migrations:latest"], - repository = "docker.io/shaldengeki/fitbit-challenges-migrations", - visibility = ["//fitbit_challenges/api/migrations:__subpackages__"], -) diff --git a/fitbit_challenges/api/migrations/__main__.py b/fitbit_challenges/api/migrations/__main__.py index 07f5239a..9c1580ae 100644 --- a/fitbit_challenges/api/migrations/__main__.py +++ b/fitbit_challenges/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from fitbit_challenges.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/fitbit_challenges/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/fitbit_challenges/api/migrations/alembic.ini") + shutil.copyfile( + "/fitbit_challenges/api/migrations/binary.runfiles/_main/fitbit_challenges/api/migrations/alembic.ini", + "/fitbit_challenges/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/fitbit_challenges/api/migrations") diff --git a/fitbit_challenges/api/migrations/env.py b/fitbit_challenges/api/migrations/env.py index 78d62879..8035c795 100644 --- a/fitbit_challenges/api/migrations/env.py +++ b/fitbit_challenges/api/migrations/env.py @@ -5,6 +5,8 @@ from alembic import context from flask import current_app +from base.flask_app import database_uri + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -26,7 +28,7 @@ def get_engine(): def get_engine_url(): # Set the database URL directly from environment variables. - return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}" + return database_uri().replace("postgresql+pg8000://", "postgres://") # try: # return get_engine().url.render_as_string(hide_password=False).replace( # '%', '%%') diff --git a/fitbit_challenges/docker-compose.override.yaml b/fitbit_challenges/docker-compose.override.yaml index 4c771e04..8483d4e9 100644 --- a/fitbit_challenges/docker-compose.override.yaml +++ b/fitbit_challenges/docker-compose.override.yaml @@ -7,7 +7,6 @@ services: - "5000:5000" migration: <<: *api - image: shaldengeki/fitbit-challenges-api-migrations ports: [] worker: <<: *api diff --git a/fitbit_challenges/docker-compose.prod.yaml b/fitbit_challenges/docker-compose.prod.yaml index d2f761ff..9b0747ed 100644 --- a/fitbit_challenges/docker-compose.prod.yaml +++ b/fitbit_challenges/docker-compose.prod.yaml @@ -7,7 +7,6 @@ services: - env/.api.prod.env migration: <<: *api - image: shaldengeki/fitbit-challenges-api-migrations:latest ports: [] worker: <<: *api diff --git a/fitbit_challenges/docker-compose.yaml b/fitbit_challenges/docker-compose.yaml index f68b8bec..4435b421 100644 --- a/fitbit_challenges/docker-compose.yaml +++ b/fitbit_challenges/docker-compose.yaml @@ -9,7 +9,7 @@ services: - pg migration: <<: *api - image: shaldengeki/fitbit-challenges-api-migrations:latest + command: [ "/fitbit_challenges/api/migrations/binary"] restart: no worker: <<: *api diff --git a/fitbit_challenges/worker/BUILD.bazel b/fitbit_challenges/worker/BUILD.bazel index c87cea5e..627a9a15 100644 --- a/fitbit_challenges/worker/BUILD.bazel +++ b/fitbit_challenges/worker/BUILD.bazel @@ -25,9 +25,12 @@ py_binary( py_oci_image( name = "base_image", base = "@python3_image", - binary = ":binary", + binaries = [ + "//scripts:wait_for_postgres", + ":binary", + ], cmd = [ - "/fitbit_challenges/worker/binary.runfiles/_main/scripts/wait_for_postgres", + "/scripts/wait_for_postgres", "/fitbit_challenges/worker/binary", ], env = { diff --git a/home_api/api/migrations/BUILD.bazel b/home_api/api/migrations/BUILD.bazel index ca592525..b66d760d 100644 --- a/home_api/api/migrations/BUILD.bazel +++ b/home_api/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -21,6 +19,7 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", + visibility = ["//home_api/api:__subpackages__"], deps = [ "//home_api/api:config_py", "//scripts:wait_for_postgres", @@ -28,37 +27,3 @@ py_binary( requirement("Flask-Migrate"), ], ) - -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/home_api/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/home_api/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //home_api/api/migrations:image_tarball -# $ docker run --rm shaldengeki/home-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/home-api-api-migrations:latest"], - repository = "docker.io/shaldengeki/home-api-api-migrations", - visibility = ["//home_api/api/migrations:__subpackages__"], -) diff --git a/home_api/api/migrations/__main__.py b/home_api/api/migrations/__main__.py index d4a2a4eb..9d89bc01 100644 --- a/home_api/api/migrations/__main__.py +++ b/home_api/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from home_api.api.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/home_api/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/home_api/api/migrations/alembic.ini") + shutil.copyfile( + "/home_api/api/migrations/binary.runfiles/_main/home_api/api/migrations/alembic.ini", + "/home_api/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/home_api/api/migrations") diff --git a/home_api/docker-compose.override.yaml b/home_api/docker-compose.override.yaml new file mode 100644 index 00000000..eaeb0ce1 --- /dev/null +++ b/home_api/docker-compose.override.yaml @@ -0,0 +1,19 @@ +version: "3" +services: + api: &api + env_file: + - env/.api.env + ports: + - "5000:5000" + migration: + <<: *api + ports: [] + pg: + env_file: + - env/.postgres.env + frontend: + image: shaldengeki/home-api-frontend:latest + env_file: + - env/.frontend.env + ports: + - "5001:80" diff --git a/home_api/docker-compose.yaml b/home_api/docker-compose.yaml index 755f63f8..8b776591 100644 --- a/home_api/docker-compose.yaml +++ b/home_api/docker-compose.yaml @@ -20,7 +20,7 @@ services: - "5000:5000" migration: <<: *api - image: shaldengeki/home-api-api-migrations:latest + command: [ "/home_api/api/migrations/binary"] restart: no depends_on: - pg diff --git a/mc_manager/api/migrations/BUILD.bazel b/mc_manager/api/migrations/BUILD.bazel index 9022d36a..63f37c20 100644 --- a/mc_manager/api/migrations/BUILD.bazel +++ b/mc_manager/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -9,6 +7,7 @@ py_library( imports = [".."], visibility = ["//:__subpackages__"], deps = [ + "//base:flask_app_py", requirement("alembic"), requirement("Flask"), ], @@ -20,44 +19,12 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", + visibility = ["//mc_manager/api:__subpackages__"], deps = [ + "//base:flask_app_py", "//mc_manager/api:config_py", "//scripts:wait_for_postgres", "@rules_python//python/runfiles", requirement("Flask-Migrate"), ], ) - -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/mc_manager/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/mc_manager/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //mc_manager/api/migrations:image_tarball -# $ docker run --rm shaldengeki/mc-manager-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/mc-manager-api-migrations:latest"], - repository = "docker.io/shaldengeki/mc-manager-api-migrations", - visibility = ["//mc_manager/api/migrations:__subpackages__"], -) diff --git a/mc_manager/api/migrations/__main__.py b/mc_manager/api/migrations/__main__.py index f7f3f75c..e3c934f4 100644 --- a/mc_manager/api/migrations/__main__.py +++ b/mc_manager/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from mc_manager.api.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/mc_manager/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/mc_manager/api/migrations/alembic.ini") + shutil.copyfile( + "/mc_manager/api/migrations/binary.runfiles/_main/mc_manager/api/migrations/alembic.ini", + "/mc_manager/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/mc_manager/api/migrations") diff --git a/mc_manager/api/migrations/env.py b/mc_manager/api/migrations/env.py index 78d62879..8035c795 100644 --- a/mc_manager/api/migrations/env.py +++ b/mc_manager/api/migrations/env.py @@ -5,6 +5,8 @@ from alembic import context from flask import current_app +from base.flask_app import database_uri + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -26,7 +28,7 @@ def get_engine(): def get_engine_url(): # Set the database URL directly from environment variables. - return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}" + return database_uri().replace("postgresql+pg8000://", "postgres://") # try: # return get_engine().url.render_as_string(hide_password=False).replace( # '%', '%%') diff --git a/mc_manager/docker-compose.override.yaml b/mc_manager/docker-compose.override.yaml new file mode 100644 index 00000000..c35d8279 --- /dev/null +++ b/mc_manager/docker-compose.override.yaml @@ -0,0 +1,25 @@ +version: "3" +services: + api: &api + env_file: + - env/.api.env + ports: + - "5000:5000" + migration: + <<: *api + ports: [] + worker: + <<: *api + image: shaldengeki/mc-manager-worker:latest + ports: [] + deploy: + replicas: 0 + pg: + env_file: + - env/.postgres.env + frontend: + image: shaldengeki/mc-manager-frontend:latest + env_file: + - env/.frontend.env + ports: + - "5001:80" diff --git a/mc_manager/docker-compose.yaml b/mc_manager/docker-compose.yaml index 393aba8a..ad93dca6 100644 --- a/mc_manager/docker-compose.yaml +++ b/mc_manager/docker-compose.yaml @@ -9,7 +9,7 @@ services: - pg migration: <<: *api - image: shaldengeki/mc-manager-api-migrations:latest + command: [ "/mc_manager/api/migrations/binary"] restart: no worker: <<: *api diff --git a/mc_manager/worker/BUILD.bazel b/mc_manager/worker/BUILD.bazel index b3d958e8..bfe10018 100644 --- a/mc_manager/worker/BUILD.bazel +++ b/mc_manager/worker/BUILD.bazel @@ -1,5 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") -load("@rules_python//python:defs.bzl", "py_binary") +load("@rules_python//python:defs.bzl", "py_binary", "py_library") load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") load("//tools/build_rules:py_layer.bzl", "py_oci_image") @@ -19,21 +19,26 @@ py_library( py_binary( name = "binary", - srcs = [ - "__init__.py", - "worker.py", - ], + srcs = ["worker.py"], imports = [".."], main = "worker.py", - deps = DEPS, + visibility = ["//:__subpackages__"], + deps = [ + "@py_deps//boto3", + "@py_deps//docker", + "@py_deps//requests", + ], ) py_oci_image( name = "base_image", base = "@python3_image", - binary = ":binary", + binaries = [ + "//scripts:wait_for_postgres", + ":binary", + ], cmd = [ - "/mc_manager/worker/binary.runfiles/_main/scripts/wait_for_postgres", + "/scripts/wait_for_postgres", "/mc_manager/worker/binary", ], env = { diff --git a/mc_manager/worker/tests/BUILD.bazel b/mc_manager/worker/tests/BUILD.bazel index 6a27b338..ef17f59e 100644 --- a/mc_manager/worker/tests/BUILD.bazel +++ b/mc_manager/worker/tests/BUILD.bazel @@ -1,19 +1,11 @@ -load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_test") -_RUNTIME_DEPS = [ - requirement("boto3"), - requirement("docker"), - requirement("requests"), -] - py_test( - name = "default_tests", + name = "worker_test", srcs = ["worker_test.py"], - main = "worker_test.py", deps = [ - requirement("pytest"), - requirement("pysocks"), - "//mc_manager/worker:library", - ] + _RUNTIME_DEPS, + "//mc_manager/worker:binary", + "@py_deps//pytest", + "@py_deps//requests", + ], ) diff --git a/proto_registry/api/migrations/BUILD.bazel b/proto_registry/api/migrations/BUILD.bazel index b3932e78..4095eb44 100644 --- a/proto_registry/api/migrations/BUILD.bazel +++ b/proto_registry/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -9,6 +7,7 @@ py_library( imports = [".."], visibility = ["//:__subpackages__"], deps = [ + "//base:flask_app_py", requirement("alembic"), requirement("Flask"), ], @@ -20,44 +19,12 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", + visibility = ["//proto_registry/api:__subpackages__"], deps = [ + "//base:flask_app_py", "//proto_registry/api:config_py", "//scripts:wait_for_postgres", "@rules_python//python/runfiles", requirement("Flask-Migrate"), ], ) - -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/proto_registry/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/proto_registry/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //proto_registry/api/migrations:image_tarball -# $ docker run --rm shaldengeki/proto-registry-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/proto-registry-api-migrations:latest"], - repository = "docker.io/shaldengeki/proto-registry-api-migrations", - visibility = ["//proto_registry/api/migrations:__subpackages__"], -) diff --git a/proto_registry/api/migrations/__main__.py b/proto_registry/api/migrations/__main__.py index f3f6da58..a167ba41 100644 --- a/proto_registry/api/migrations/__main__.py +++ b/proto_registry/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from proto_registry.api.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/proto_registry/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/proto_registry/api/migrations/alembic.ini") + shutil.copyfile( + "/proto_registry/api/migrations/binary.runfiles/_main/proto_registry/api/migrations/alembic.ini", + "/proto_registry/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/proto_registry/api/migrations") diff --git a/proto_registry/api/migrations/env.py b/proto_registry/api/migrations/env.py index 78d62879..8035c795 100644 --- a/proto_registry/api/migrations/env.py +++ b/proto_registry/api/migrations/env.py @@ -5,6 +5,8 @@ from alembic import context from flask import current_app +from base.flask_app import database_uri + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -26,7 +28,7 @@ def get_engine(): def get_engine_url(): # Set the database URL directly from environment variables. - return f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}" + return database_uri().replace("postgresql+pg8000://", "postgres://") # try: # return get_engine().url.render_as_string(hide_password=False).replace( # '%', '%%') diff --git a/proto_registry/docker-compose.override.yaml b/proto_registry/docker-compose.override.yaml new file mode 100644 index 00000000..279eb1ef --- /dev/null +++ b/proto_registry/docker-compose.override.yaml @@ -0,0 +1,13 @@ +version: "3" +services: + api: &api + env_file: + - env/.api.env + ports: + - "5000:5000" + migration: + <<: *api + ports: [] + pg: + env_file: + - env/.postgres.env diff --git a/proto_registry/docker-compose.yaml b/proto_registry/docker-compose.yaml index 510c2250..ada2af2e 100644 --- a/proto_registry/docker-compose.yaml +++ b/proto_registry/docker-compose.yaml @@ -11,7 +11,7 @@ services: - pg migration: <<: *api - image: shaldengeki/proto-registry-api-migrations:latest + command: [ "/proto_registry/api/migrations/binary"] ports: [] restart: no pg: diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a4dd05e2..a71ab5df 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -6,6 +6,7 @@ py_binary( main = "wait_for_postgres.py", visibility = ["//visibility:public"], deps = [ + "//base:flask_app_py", "@py_deps//pg8000", ], ) diff --git a/scripts/rebuild-and-run b/scripts/rebuild-and-run index 83fedf28..b16987bf 100755 --- a/scripts/rebuild-and-run +++ b/scripts/rebuild-and-run @@ -6,6 +6,5 @@ set -euxo pipefail bazel run api:api_image_image_tarball -bazel run api/migrations:image_tarball docker compose -f docker-compose.yaml -f docker-compose.override.yaml up --no-deps -d api migration pg diff --git a/scripts/wait_for_postgres.py b/scripts/wait_for_postgres.py index 09755458..50ee4ca1 100755 --- a/scripts/wait_for_postgres.py +++ b/scripts/wait_for_postgres.py @@ -4,19 +4,47 @@ import subprocess import sys import time +from urllib.parse import urlparse import pg8000.native +from base.flask_app import database_uri + def wait_for_postgres() -> None: + uri = database_uri() + parsed_uri = urlparse(uri) + username = "admin" + password = "development" + host = "localhost" + port = "5432" + database = "postgres" + if "@" not in parsed_uri.netloc: + # no user/password. + host_parts = parsed_uri.netloc + else: + auth_parts, host_parts = parsed_uri.netloc.split("@") + if ":" not in auth_parts: + username = auth_parts + else: + username, password = auth_parts.split(":") + + if ":" not in host_parts: + host = host_parts + else: + host, port = host_parts.split(":") + + if parsed_uri.path and parsed_uri.path != "/": + database = parsed_uri.path[1:] + while True: try: pg8000.native.Connection( - os.getenv("DB_USERNAME", "admin"), - host=os.getenv("DB_HOST", "pg"), - port=os.getenv("DB_PORT", 5432), - database=os.getenv("DATABASE_NAME", "api_development"), - password=os.getenv("DB_PASSWORD", "development"), + username, + host=host, + port=int(port), + database=database, + password=password, ) except pg8000.exceptions.InterfaceError: print("Postgres is unavailable - sleeping") diff --git a/skeleton/api/migrations/BUILD.bazel b/skeleton/api/migrations/BUILD.bazel index cbf8c81d..e5fde8d9 100644 --- a/skeleton/api/migrations/BUILD.bazel +++ b/skeleton/api/migrations/BUILD.bazel @@ -1,7 +1,5 @@ load("@py_deps//:requirements.bzl", "requirement") load("@rules_python//python:defs.bzl", "py_binary", "py_library") -load("//tools/build_rules:cross_platform_image.bzl", "cross_platform_image") -load("//tools/build_rules:py_layer.bzl", "py_oci_image") py_library( name = "migrate_lib", @@ -20,7 +18,7 @@ py_binary( data = ["alembic.ini"], imports = [".."], main = "__main__.py", - visibility = ["//:__subpackages__"], + visibility = ["//skeleton/api:__subpackages__"], deps = [ "//scripts:wait_for_postgres", # keep "//skeleton:config_py", @@ -29,43 +27,6 @@ py_binary( ], ) -py_oci_image( - name = "base_image", - base = "@python3_image", - binary = ":binary", - cmd = [ - "/skeleton/api/migrations/binary.runfiles/_main/scripts/wait_for_postgres", - "/skeleton/api/migrations/binary", - ], - env = { - "FLASK_APP": "app.py", - "FLASK_DEBUG": "True", - "API_PORT": "5000", - "FRONTEND_PROTOCOL": "http", - "FRONTEND_HOST": "frontend", - "FRONTEND_PORT": "5001", - "DB_HOST": "pg", - "DB_USERNAME": "admin", - "DB_PASSWORD": "development", - "DATABASE_NAME": "api_development", - "FITBIT_CLIENT_ID": "testing", - "FITBIT_CLIENT_SECRET": "testing", - "FITBIT_VERIFICATION_CODE": "testing", - "FLASK_SECRET_KEY": "testing", - }, - tags = ["manual"], -) - -# $ bazel run //skeleton/api/migrations:image_tarball -# $ docker run --rm shaldengeki/skeleton-api-migrations:latest -cross_platform_image( - name = "image", - image = ":base_image", - repo_tags = ["shaldengeki/skeleton-api-migrations:latest"], - repository = "docker.io/shaldengeki/skeleton-api-migrations", - visibility = ["//skeleton/api/migrations:__subpackages__"], -) - py_library( name = "env", srcs = ["env.py"], diff --git a/skeleton/api/migrations/__main__.py b/skeleton/api/migrations/__main__.py index 76d1290e..2e865998 100644 --- a/skeleton/api/migrations/__main__.py +++ b/skeleton/api/migrations/__main__.py @@ -1,15 +1,15 @@ import shutil from flask_migrate import upgrade -from python.runfiles import Runfiles from skeleton.config import app if __name__ == "__main__": # Copy the alembic.ini. - r = Runfiles.Create() - alembic_ini_src = r.Rlocation("_main/skeleton/api/migrations/alembic.ini") - shutil.copyfile(alembic_ini_src, "/skeleton/api/migrations/alembic.ini") + shutil.copyfile( + "/skeleton/api/migrations/binary.runfiles/_main/skeleton/api/migrations/alembic.ini", + "/skeleton/api/migrations/alembic.ini", + ) with app.app_context(): upgrade(directory="/skeleton/api/migrations") diff --git a/skeleton/docker-compose.prod.yaml b/skeleton/docker-compose.prod.yaml index 3ecc8346..64dfb954 100644 --- a/skeleton/docker-compose.prod.yaml +++ b/skeleton/docker-compose.prod.yaml @@ -7,7 +7,6 @@ services: - env/.api.prod.env migration: <<: *api - image: shaldengeki/skeleton-api-migrations:latest ports: [] worker: <<: *api diff --git a/skeleton/docker-compose.yaml b/skeleton/docker-compose.yaml index 8607d3e2..88aa1cdf 100644 --- a/skeleton/docker-compose.yaml +++ b/skeleton/docker-compose.yaml @@ -9,7 +9,7 @@ services: - pg migration: <<: *api - image: shaldengeki/skeleton-api-migrations:latest + command: [ "/skeleton/api/migrations/binary"] restart: no worker: <<: *api diff --git a/tools/build_rules/api_image.bzl b/tools/build_rules/api_image.bzl index 9a8b5637..8e4dcebe 100644 --- a/tools/build_rules/api_image.bzl +++ b/tools/build_rules/api_image.bzl @@ -15,6 +15,7 @@ def api_image( deps, repo_tags, docker_hub_repository, + migration_binary = None, env = None, stamp_file = "//:stamped", base_image = "@python3_image", @@ -29,6 +30,7 @@ def api_image( env (dict[str, str]): Environment variables to set in the image. repo_tags (list[str]): List of repo + tag pairs that the container images should be loaded under. docker_hub_repository (str): Repository on Docker Hub that the container images should be pushed to. + migration_binary (label): Binary target for this API's database migrations. Defaults to //your/api/package/migrations:binary. stamp_file (file): File containing image tags that the image should be pushed under. base_image (label): Base container image to use. visibility (list[str]): Visibility to set on all the targets. @@ -37,6 +39,9 @@ def api_image( if visibility == None: visibility = ["//visibility:public"] + if migration_binary == None: + migration_binary = Label("//" + native.package_name() + "/migrations:binary") + main_py( name = name + "_main_py", app_package = app_package, @@ -76,7 +81,14 @@ def api_image( py_oci_image( name = name + "_base_image", base = base_image, - binary = name + "_binary", + binaries = [ + "//scripts:wait_for_postgres", + name + "_binary", + migration_binary, + ], + entrypoint = [ + "/scripts/wait_for_postgres", + ], cmd = [ "/" + native.package_name() + "/" + name + "_binary", ], diff --git a/tools/build_rules/py_layer.bzl b/tools/build_rules/py_layer.bzl index 1e677392..3f29821b 100644 --- a/tools/build_rules/py_layer.bzl +++ b/tools/build_rules/py_layer.bzl @@ -13,15 +13,15 @@ PY_INTERPRETER_REGEX = "\\.runfiles/.*python.*-.*" # match *only* external pip like repositories that contain the string "site-packages" SITE_PACKAGES_REGEX = "\\.runfiles/.*/site-packages/.*" -def py_layers(name, binary): - """Create three layers for a py_binary target: interpreter, third-party dependencies, and application code. +def py_layers(name, binaries): + """Create three layers for a list of py_binary targets: interpreter, third-party dependencies, and application code. This allows a container image to have smaller uploads, since the application layer usually changes more than the other two. Args: name: prefix for generated targets, to ensure they are unique within the package - binary: a py_binary target + binaries: a list of py_binary targets Returns: a list of labels for the layers, which are tar files """ @@ -33,7 +33,8 @@ def py_layers(name, binary): # into fine-grained layers for better docker performance. mtree_spec( name = name + ".mf", - srcs = [binary], + srcs = binaries, + tags = ["manual"], ) native.genrule( @@ -41,6 +42,7 @@ def py_layers(name, binary): srcs = [name + ".mf"], outs = [name + ".interpreter_tar_manifest.spec"], cmd = "grep '{}' $< >$@".format(PY_INTERPRETER_REGEX), + tags = ["manual"], ) native.genrule( @@ -48,6 +50,7 @@ def py_layers(name, binary): srcs = [name + ".mf"], outs = [name + ".packages_tar_manifest.spec"], cmd = "grep '{}' $< >$@".format(SITE_PACKAGES_REGEX), + tags = ["manual"], ) # Any lines that didn't match one of the two grep above @@ -56,6 +59,7 @@ def py_layers(name, binary): srcs = [name + ".mf"], outs = [name + ".app_tar_manifest.spec"], cmd = "grep -v '{}' $< | grep -v '{}' >$@".format(SITE_PACKAGES_REGEX, PY_INTERPRETER_REGEX), + tags = ["manual"], ) result = [] @@ -64,16 +68,26 @@ def py_layers(name, binary): result.append(layer_target) tar( name = layer_target, - srcs = [binary], + srcs = binaries, mtree = "{}.{}_tar_manifest".format(name, layer), + tags = ["manual"], ) return result -def py_oci_image(name, binary, tars = [], **kwargs): - "Wrapper around oci_image that splits the py_binary into layers." +def py_oci_image(name, binaries, tars = [], **kwargs): + """ + Wrapper around oci_image that splits the py_binary into layers. + + Args: + name: Name that the generated oci_image should have. + binaries: list of py_binary targets that should be a part of this image. + tars: (optional) list of tar targets to also bundle into this image. + **kwargs: Arguments to pass to oci_image. + """ + oci_image( name = name, - tars = tars + py_layers(name, binary), + tars = tars + py_layers(name, binaries), **kwargs ) diff --git a/tools/build_rules/templates/__main__.py.tpl b/tools/build_rules/templates/__main__.py.tpl index 99bd30a3..c4f3a208 100644 --- a/tools/build_rules/templates/__main__.py.tpl +++ b/tools/build_rules/templates/__main__.py.tpl @@ -1,18 +1,4 @@ -from python.runfiles import Runfiles -import os -import subprocess - from {app_package} import app if __name__ == "__main__": - # First, run wait-for-postgres. - r = Runfiles.Create() - env = os.environ - env.update(r.EnvVars()) - p = subprocess.run( - [r.Rlocation("_main/scripts/wait_for_postgres")], - env = env, - ) - - # Then run the app. app.run(host="0.0.0.0")