From f61364738fb7a79f304df39e6086ba3c1b661d16 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Wed, 20 Jul 2022 23:42:15 -0400 Subject: [PATCH 01/10] Initial commit --- proto-registry/.gitignore | 129 ++++++++++++++++++++++++++++++++++++++ proto-registry/LICENSE | 21 +++++++ 2 files changed, 150 insertions(+) create mode 100644 proto-registry/.gitignore create mode 100644 proto-registry/LICENSE diff --git a/proto-registry/.gitignore b/proto-registry/.gitignore new file mode 100644 index 00000000..b6e47617 --- /dev/null +++ b/proto-registry/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/proto-registry/LICENSE b/proto-registry/LICENSE new file mode 100644 index 00000000..4220c1dc --- /dev/null +++ b/proto-registry/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Charles OuGuo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From e7908032f1402e280ea306001e92220af0be539d Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 21 Jul 2022 05:15:27 +0000 Subject: [PATCH 02/10] Add API skeleton, with GET endpoints --- proto-registry/registry/.gitignore | 6 ++ proto-registry/registry/BUILD | 0 proto-registry/registry/README.md | 1 + proto-registry/registry/WORKSPACE | 84 ++++++++++++++++++ proto-registry/registry/api/.dockerignore | 2 + proto-registry/registry/api/BUILD | 38 +++++++++ proto-registry/registry/api/Dockerfile | 22 +++++ proto-registry/registry/api/__init__.py | 1 + proto-registry/registry/api/app.py | 39 +++++++++ proto-registry/registry/api/config.py | 33 +++++++ proto-registry/registry/api/migrations/README | 1 + .../registry/api/migrations/alembic.ini | 85 +++++++++++++++++++ proto-registry/registry/api/migrations/env.py | 81 ++++++++++++++++++ .../registry/api/migrations/script.py.mako | 24 ++++++ ...c1c2d94b4_create_subject_versions_table.py | 56 ++++++++++++ .../9da1de15cd6a_create_subjects_table.py | 35 ++++++++ .../registry/api/models/__init__.py | 2 + proto-registry/registry/api/models/subject.py | 24 ++++++ .../registry/api/models/subject_version.py | 36 ++++++++ proto-registry/registry/api/requirements.txt | 12 +++ .../registry/api/scripts/wait-for-postgres.sh | 14 +++ proto-registry/registry/api/views/__init__.py | 0 proto-registry/registry/docker-compose.yaml | 67 +++++++++++++++ proto-registry/registry/env.development | 18 ++++ proto-registry/registry/env.production | 18 ++++ .../registry/scripts/start-server.sh | 3 + proto-registry/registry/toolchain/BUILD | 30 +++++++ 27 files changed, 732 insertions(+) create mode 100644 proto-registry/registry/.gitignore create mode 100644 proto-registry/registry/BUILD create mode 100644 proto-registry/registry/README.md create mode 100644 proto-registry/registry/WORKSPACE create mode 100644 proto-registry/registry/api/.dockerignore create mode 100644 proto-registry/registry/api/BUILD create mode 100644 proto-registry/registry/api/Dockerfile create mode 100644 proto-registry/registry/api/__init__.py create mode 100644 proto-registry/registry/api/app.py create mode 100644 proto-registry/registry/api/config.py create mode 100644 proto-registry/registry/api/migrations/README create mode 100644 proto-registry/registry/api/migrations/alembic.ini create mode 100644 proto-registry/registry/api/migrations/env.py create mode 100644 proto-registry/registry/api/migrations/script.py.mako create mode 100644 proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py create mode 100644 proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py create mode 100644 proto-registry/registry/api/models/__init__.py create mode 100644 proto-registry/registry/api/models/subject.py create mode 100644 proto-registry/registry/api/models/subject_version.py create mode 100644 proto-registry/registry/api/requirements.txt create mode 100755 proto-registry/registry/api/scripts/wait-for-postgres.sh create mode 100644 proto-registry/registry/api/views/__init__.py create mode 100644 proto-registry/registry/docker-compose.yaml create mode 100644 proto-registry/registry/env.development create mode 100644 proto-registry/registry/env.production create mode 100755 proto-registry/registry/scripts/start-server.sh create mode 100644 proto-registry/registry/toolchain/BUILD diff --git a/proto-registry/registry/.gitignore b/proto-registry/registry/.gitignore new file mode 100644 index 00000000..00c5ebd8 --- /dev/null +++ b/proto-registry/registry/.gitignore @@ -0,0 +1,6 @@ +postgres-data +env.production.real +bazel-bin +bazel-out +bazel-registry +bazel-testlogs diff --git a/proto-registry/registry/BUILD b/proto-registry/registry/BUILD new file mode 100644 index 00000000..e69de29b diff --git a/proto-registry/registry/README.md b/proto-registry/registry/README.md new file mode 100644 index 00000000..8015af31 --- /dev/null +++ b/proto-registry/registry/README.md @@ -0,0 +1 @@ +# mc-manager diff --git a/proto-registry/registry/WORKSPACE b/proto-registry/registry/WORKSPACE new file mode 100644 index 00000000..d7ddbe02 --- /dev/null +++ b/proto-registry/registry/WORKSPACE @@ -0,0 +1,84 @@ +workspace( + name = "proto-registry-service", +) + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +rules_python_version = "740825b7f74930c62f44af95c9a4c1bd428d2c53" # Latest @ 2021-06-23 + +http_archive( + name = "rules_python", + sha256 = "09a3c4791c61b62c2cbc5b2cbea4ccc32487b38c7a2cc8f87a794d7a659cc742", + strip_prefix = "rules_python-{}".format(rules_python_version), + url = "https://github.com/bazelbuild/rules_python/archive/{}.zip".format(rules_python_version), +) + +load("@rules_python//python:pip.bzl", "pip_install") + +pip_install( + name = "worker_deps", + requirements="//worker:requirements.txt" +) +pip_install( + name = "api_deps", + requirements="//api:requirements.txt" +) + +http_archive( + name = "io_bazel_rules_docker", + sha256 = "59d5b42ac315e7eadffa944e86e90c2990110a1c8075f1cd145f487e999d22b3", + strip_prefix = "rules_docker-0.17.0", + urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.17.0/rules_docker-v0.17.0.tar.gz"], +) + +load( + "@io_bazel_rules_docker//repositories:repositories.bzl", + container_repositories = "repositories", +) +container_repositories() + +load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") + +container_deps() + +load("@io_bazel_rules_docker//container:pull.bzl", "container_pull") + +container_pull( + name = "py3_image", + registry = "index.docker.io", + repository = "library/python", + tag = "3.9-alpine", +) + +register_toolchains( + "//toolchain:container_py_toolchain", +) + +register_execution_platforms( + "@local_config_platform//:host", + "@io_bazel_rules_docker//platforms:local_container_platform", +) + +load( + "@io_bazel_rules_docker//python3:image.bzl", + _py_image_repos = "repositories", +) + +_py_image_repos() + +http_archive( + name = "bazel_skylib", + sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", + ], +) + +container_pull( + name = "nginx_image", + registry = "index.docker.io", + repository = "library/nginx", + tag = "1-alpine", + digest = "sha256:c35699d53f03ff9024ce2c8f6730567f183a15cc27b24453c5d90f0e7542daea", +) diff --git a/proto-registry/registry/api/.dockerignore b/proto-registry/registry/api/.dockerignore new file mode 100644 index 00000000..d637b18c --- /dev/null +++ b/proto-registry/registry/api/.dockerignore @@ -0,0 +1,2 @@ +/__pycache__ +/venv diff --git a/proto-registry/registry/api/BUILD b/proto-registry/registry/api/BUILD new file mode 100644 index 00000000..850ee5a2 --- /dev/null +++ b/proto-registry/registry/api/BUILD @@ -0,0 +1,38 @@ +load("@rules_python//python:defs.bzl", "py_binary") +load("@api_deps//:requirements.bzl", "requirement") +load("@io_bazel_rules_docker//python3:image.bzl", "py3_image") + +py_binary( + name = "binary", + srcs = glob(["**/*.py"]), + main = "app.py", + deps = [ + requirement("Flask"), + requirement("Flask-Migrate"), + requirement("importlib-metadata"), + requirement("importlib-resources"), + requirement("typing-extensions"), + requirement("psycopg2"), + requirement("sqlalchemy"), + requirement("Flask-SQLAlchemy"), + requirement("flask-cors") + ] +) + +py3_image( + name = "image", + srcs = glob(["**/*.py"]), + main = "app.py", + base = "@py3_image//image", + deps = [ + requirement("Flask"), + requirement("Flask-Migrate"), + requirement("importlib-metadata"), + requirement("importlib-resources"), + requirement("typing-extensions"), + requirement("psycopg2"), + requirement("sqlalchemy"), + requirement("Flask-SQLAlchemy"), + requirement("flask-cors") + ] +) diff --git a/proto-registry/registry/api/Dockerfile b/proto-registry/registry/api/Dockerfile new file mode 100644 index 00000000..a34efc11 --- /dev/null +++ b/proto-registry/registry/api/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.9-alpine + +EXPOSE 5000/tcp + +RUN apk update && \ + apk add --virtual build-deps gcc python3-dev musl-dev g++ && \ + apk add postgresql-dev postgresql-client + +WORKDIR /usr/src/app + +ENV FLASK_APP=app.py +ENV API_PORT=5000 +ENV FRONTEND_HOST=home_frontend +ENV FRONTEND_PORT=5001 + +COPY requirements.txt ./ + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +CMD flask run --host 0.0.0.0 --port $API_PORT diff --git a/proto-registry/registry/api/__init__.py b/proto-registry/registry/api/__init__.py new file mode 100644 index 00000000..0f5e7204 --- /dev/null +++ b/proto-registry/registry/api/__init__.py @@ -0,0 +1 @@ +from .config import app diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py new file mode 100644 index 00000000..579ffb00 --- /dev/null +++ b/proto-registry/registry/api/app.py @@ -0,0 +1,39 @@ +import json +from flask import abort +from sqlalchemy import asc + +from .config import app +from . import models + +@app.route("/") +def hello_world(): + return "

Hello, World!

" + +@app.route("/subjects") +def subjects(): + return json.dumps([dict(x) for x in models.Subject.query.order_by(asc(models.Subject.id)).all()]) + +@app.route("/subjects//versions") +def subject_versions(subject_name): + subject = models.Subject.query.filter(models.Subject.name == subject_name).first() + if subject is None: + abort(404) + + return json.dumps([ + version.version_id + for version in subject.versions + ]) + +@app.route("/subjects//versions/") +def subject_version(subject_name, version_id): + subject = models.Subject.query.filter(models.Subject.name == subject_name).first() + if subject is None: + abort(404) + + versions = [v for v in subject.versions if v.version_id == version_id] + if not versions: + abort(404) + return next(versions).schema + +if __name__ == "__main__": + app.run() diff --git a/proto-registry/registry/api/config.py b/proto-registry/registry/api/config.py new file mode 100644 index 00000000..40738327 --- /dev/null +++ b/proto-registry/registry/api/config.py @@ -0,0 +1,33 @@ +import os + +from flask import Flask +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate +from flask_cors import CORS + +app = Flask(__name__) +app.config[ + "SQLALCHEMY_DATABASE_URI" +] = "postgresql://{user}:{password}@{host}/{db}".format( + user=os.getenv("DB_USER", "admin"), + password=os.getenv("DB_PASS", "development"), + host=os.getenv("DB_HOST", "pg"), + db=os.getenv("DB_NAME", "api_development"), +) +CORS( + app, + resources={ + "/graphql": { + "origins": [ + "http://{host}:{port}".format( + host=os.getenv("FRONTEND_HOST", "localhost"), + port=os.getenv("FRONTEND_PORT", 5001), + ) + ], + } + }, +) + +db = SQLAlchemy(app) + +migrate = Migrate(app, db) diff --git a/proto-registry/registry/api/migrations/README b/proto-registry/registry/api/migrations/README new file mode 100644 index 00000000..2500aa1b --- /dev/null +++ b/proto-registry/registry/api/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/proto-registry/registry/api/migrations/alembic.ini b/proto-registry/registry/api/migrations/alembic.ini new file mode 100644 index 00000000..0f9a0b1f --- /dev/null +++ b/proto-registry/registry/api/migrations/alembic.ini @@ -0,0 +1,85 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = . + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migrations/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://admin:development@pg/api_development + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks=black +# black.type=console_scripts +# black.entrypoint=black +# black.options=-l 79 + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/proto-registry/registry/api/migrations/env.py b/proto-registry/registry/api/migrations/env.py new file mode 100644 index 00000000..18155fdd --- /dev/null +++ b/proto-registry/registry/api/migrations/env.py @@ -0,0 +1,81 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +import os + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +database_url = f"postgresql://{os.environ.get('DB_USERNAME', 'admin')}:{os.environ.get('DB_PASSWORD', 'development')}@{os.environ.get('DB_HOST', 'pg')}/{os.environ.get('DATABASE_NAME', 'api_development')}" + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + + context.configure( + url=database_url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + config_section = config.get_section(config.config_ini_section) + config_section["sqlalchemy.url"] = database_url + connectable = engine_from_config( + config_section, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/proto-registry/registry/api/migrations/script.py.mako b/proto-registry/registry/api/migrations/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/proto-registry/registry/api/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py b/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py new file mode 100644 index 00000000..4098b3f4 --- /dev/null +++ b/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py @@ -0,0 +1,56 @@ +"""create subject_versions table + +Revision ID: 86ac1c2d94b4 +Revises: 9da1de15cd6a +Create Date: 2022-07-21 04:39:32.310089 + +""" +from alembic import op +import datetime +import enum +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '86ac1c2d94b4' +down_revision = '9da1de15cd6a' +branch_labels = None +depends_on = None + +class SchemaType(enum.Enum): + AVRO = 1 + PROTOBUF = 2 + JSONSCHEMA = 3 + + +def upgrade(): + op.create_table( + 'subject_versions', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('created', sa.DateTime, nullable=False, default=datetime.datetime.utcnow), + sa.Column('version_id', sa.Integer, nullable=False), + sa.Column('subject_id', sa.Integer, nullable=False), + sa.Column('schema_type', sa.Enum(SchemaType), nullable=False, default=SchemaType.AVRO), + sa.Column('schema', sa.Unicode(20000), nullable=False), + ) + op.create_unique_constraint( + "subject_versions_version_id", "subject_versions", ["subject_id", "version_id"] + ) + op.create_foreign_key( + "subject_versions_subjects_subject_id", + "subject_versions", + "subjects", + ["subject_id"], + ["id"], + "CASCADE", + "CASCADE", + ) + op.create_index( + "subject_versions_subject_id", + "subject_versions", + ["subject_id", "version_id"], + ) + + +def downgrade(): + op.drop_table("subject_versions") diff --git a/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py b/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py new file mode 100644 index 00000000..6f5551b8 --- /dev/null +++ b/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py @@ -0,0 +1,35 @@ +"""create subjects table + +Revision ID: 9da1de15cd6a +Revises: +Create Date: 2022-07-21 04:15:20.943034 + +""" +from alembic import op +import datetime +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '9da1de15cd6a' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'subjects', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('created', sa.DateTime, nullable=False, default=datetime.datetime.utcnow), + sa.Column('name', sa.Unicode(2000), nullable=False), + ) + op.create_index( + "subjects_name", + "subjects", + ["name"], + ) + + +def downgrade(): + op.drop_table("subjects") diff --git a/proto-registry/registry/api/models/__init__.py b/proto-registry/registry/api/models/__init__.py new file mode 100644 index 00000000..27d43ec1 --- /dev/null +++ b/proto-registry/registry/api/models/__init__.py @@ -0,0 +1,2 @@ +from .subject import Subject +from .subject_version import SubjectVersion \ No newline at end of file diff --git a/proto-registry/registry/api/models/subject.py b/proto-registry/registry/api/models/subject.py new file mode 100644 index 00000000..1baf0126 --- /dev/null +++ b/proto-registry/registry/api/models/subject.py @@ -0,0 +1,24 @@ +from ..config import db +import datetime +import json + + +class Subject(db.Model): + __tablename__ = "subjects" + id = db.Column(db.Integer, primary_key=True) + created = db.Column( + db.TIMESTAMP(timezone=True), default=datetime.datetime.utcnow, nullable=False + ) + name = db.Column(db.Unicode(2000), nullable=False, unique=True) + versions = db.relationship( + "SubjectVersion", back_populates="subject", order_by="desc(SubjectVersion.version_id)" + ) + + def __repr__(self): + return "".format(id=self.id) + + def __dict__(self): + return {c.name: getattr(self, c.name) for c in self.__table__.columns} + + def to_json(self): + return json.dumps(dict(self)) \ No newline at end of file diff --git a/proto-registry/registry/api/models/subject_version.py b/proto-registry/registry/api/models/subject_version.py new file mode 100644 index 00000000..1e57885b --- /dev/null +++ b/proto-registry/registry/api/models/subject_version.py @@ -0,0 +1,36 @@ +from ..config import db +from .subject import Subject +from sqlalchemy import Enum +import datetime +import enum +import json + +class SchemaType(enum.Enum): + AVRO = 1 + PROTOBUF = 2 + JSONSCHEMA = 3 + + +class SubjectVersion(db.Model): + __tablename__ = "subject_versions" + id = db.Column(db.Integer, primary_key=True) + created = db.Column( + db.TIMESTAMP(timezone=True), default=datetime.datetime.utcnow, nullable=False + ) + version_id = db.Column(db.Integer, nullable=False) + schema_type = db.Column(Enum(SchemaType), nullable=True, default=SchemaType.AVRO) + schema = db.Column(db.Unicode(20000), nullable=False) + + subject_id = db.Column(db.Integer, db.ForeignKey(Subject.id), nullable=False) + subject = db.relationship( + "Subject", back_populates="versions", order_by="desc(SubjectVersion.version_id)" + ) + + def __repr__(self): + return "".format(id=self.id) + + def __dict__(self): + return {c.name: getattr(self, c.name) for c in self.__table__.columns} + + def to_json(self): + return json.dumps(dict(self)) \ No newline at end of file diff --git a/proto-registry/registry/api/requirements.txt b/proto-registry/registry/api/requirements.txt new file mode 100644 index 00000000..a3b1b7e2 --- /dev/null +++ b/proto-registry/registry/api/requirements.txt @@ -0,0 +1,12 @@ +Flask +Flask-Migrate +importlib-metadata +importlib-resources +typing-extensions +psycopg2 +pylint +sqlalchemy +Flask-SQLAlchemy +flask-cors +pre-commit +black diff --git a/proto-registry/registry/api/scripts/wait-for-postgres.sh b/proto-registry/registry/api/scripts/wait-for-postgres.sh new file mode 100755 index 00000000..05177201 --- /dev/null +++ b/proto-registry/registry/api/scripts/wait-for-postgres.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# wait-for-postgres.sh + +set -e + +cmd="$@" + +until PGPASSWORD=$DB_PASSWORD psql -h "$DB_HOST" -U "$DB_USERNAME" -d "$DATABASE_NAME" -c '\q'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" +exec $cmd diff --git a/proto-registry/registry/api/views/__init__.py b/proto-registry/registry/api/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/proto-registry/registry/docker-compose.yaml b/proto-registry/registry/docker-compose.yaml new file mode 100644 index 00000000..59d8db52 --- /dev/null +++ b/proto-registry/registry/docker-compose.yaml @@ -0,0 +1,67 @@ +version: '3' +services: + api: + build: + context: ./api + dockerfile: Dockerfile + image: shaldengeki/proto-registry + ports: + - "17283:17283" + volumes: + - ./api:/usr/src/app + command: ["./scripts/wait-for-postgres.sh", "flask", "run", "--host", "0.0.0.0", "--port", "$API_PORT"] + environment: + - FRONTEND_HOST + - FRONTEND_PORT + - API_PORT + - FLASK_ENV + - DB_HOST + - DB_USERNAME + - DB_PASSWORD + - DATABASE_NAME + restart: always + migration: + build: + context: ./api + dockerfile: Dockerfile + image: shaldengeki/proto-registry + volumes: + - ./api:/usr/src/app + command: ["./scripts/wait-for-postgres.sh", "flask", "db", "upgrade"] + environment: + - FLASK_ENV + - DB_HOST + - DB_USERNAME + - DB_PASSWORD + - DATABASE_NAME + depends_on: + - pg + worker: + build: + context: ./worker + dockerfile: Dockerfile + image: shaldengeki/proto-registry-worker + volumes: + - ./worker:/usr/src/app + - /var/run/docker.sock:/var/run/docker.sock + - /var/proto-registry:/var/proto-registry + environment: + - API_HOST + - API_PORT + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - HOST_PATH + - S3_BUCKET + depends_on: + - api + command: ["./wait-for", "$API_HOST:$API_PORT", "--", "python", "worker.py"] + restart: always + pg: + image: postgres:alpine + restart: always + environment: + - POSTGRES_PASSWORD + - POSTGRES_USER + - POSTGRES_DB + volumes: + - ./postgres-data:/var/lib/postgresql/data diff --git a/proto-registry/registry/env.development b/proto-registry/registry/env.development new file mode 100644 index 00000000..30adfe40 --- /dev/null +++ b/proto-registry/registry/env.development @@ -0,0 +1,18 @@ +# API variables. +REACT_APP_API_SCHEME=http +REACT_APP_API_HOST=localhost +REACT_APP_API_PORT=17283 +REACT_APP_API_PATH=graphql +API_HOST=api +API_PORT=17283 +FLASK_ENV=development +# Postgres variables. +DB_HOST=pg +DB_USERNAME=admin +DB_PASSWORD=development +DATABASE_NAME=api_development +POSTGRES_PASSWORD=development +POSTGRES_USER=admin +POSTGRES_DB=api_development +# Worker variables. +HOST_PATH=/var/proto-registry diff --git a/proto-registry/registry/env.production b/proto-registry/registry/env.production new file mode 100644 index 00000000..c54ba455 --- /dev/null +++ b/proto-registry/registry/env.production @@ -0,0 +1,18 @@ +# API variables. +REACT_APP_API_SCHEME=https +REACT_APP_API_HOST=api.proto-registry.ouguo.us +REACT_APP_API_PORT=443 +REACT_APP_API_PATH=api +API_HOST=api +API_PORT=17283 +FLASK_ENV=production +# Postgres variables. +DB_HOST=pg +DB_USERNAME=admin +DB_PASSWORD= +DATABASE_NAME=api_production +POSTGRES_PASSWORD= +POSTGRES_USER=admin +POSTGRES_DB=api_production +# Worker variables. +HOST_PATH=/var/proto-registry diff --git a/proto-registry/registry/scripts/start-server.sh b/proto-registry/registry/scripts/start-server.sh new file mode 100755 index 00000000..c1606515 --- /dev/null +++ b/proto-registry/registry/scripts/start-server.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker compose --env-file env.development up \ No newline at end of file diff --git a/proto-registry/registry/toolchain/BUILD b/proto-registry/registry/toolchain/BUILD new file mode 100644 index 00000000..b4910540 --- /dev/null +++ b/proto-registry/registry/toolchain/BUILD @@ -0,0 +1,30 @@ +load("@rules_python//python:defs.bzl", "py_runtime_pair") + +py_runtime( + name = "container_py2_runtime", + interpreter_path = "/usr/local/bin/python", + python_version = "PY2", +) + +# Path in the python:slim-3.7 image +py_runtime( + name = "container_py3_runtime", + interpreter_path = "/usr/local/bin/python3", + python_version = "PY3", +) + +py_runtime_pair( + name = "container_py_runtime_pair", + py2_runtime = ":container_py2_runtime", + py3_runtime = ":container_py3_runtime", +) + +toolchain( + name = "container_py_toolchain", + exec_compatible_with = [ + "@io_bazel_rules_docker//platforms:run_in_container", + ], + toolchain = ":container_py_runtime_pair", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", +) + From 774dd19f3c710cc9844b146a456cf50900c38014 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 21 Jul 2022 05:21:56 +0000 Subject: [PATCH 03/10] Update README --- proto-registry/registry/README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/proto-registry/registry/README.md b/proto-registry/registry/README.md index 8015af31..5e59005d 100644 --- a/proto-registry/registry/README.md +++ b/proto-registry/registry/README.md @@ -1 +1,15 @@ -# mc-manager +# registry service + +This is a minimal web service that registers & exposes schemas, [adhering to the Confluent Schema Registry API](https://docs.confluent.io/platform/current/schema-registry/develop/api.html#subjects). It is intended to serve as a reference implementation and is _not_ production-ready. + +## Development + +We support building & running via OCI-compatible container images (via Bazel) or via Docker. + +### Bazel + +Do `bazel build //api:image`. Run on the container runtime of your choice. + +### Docker + +Install docker compose on your platform, then do `docker compose --env-file env.development up`. From a525b735134df41e23dd19a509d9ac39a821b195 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 01:56:14 +0000 Subject: [PATCH 04/10] Fix subject version endpoints, and add subject-deletion/version-creation endpoints --- proto-registry/.gitignore | 8 ++ proto-registry/registry/.gitignore | 6 -- proto-registry/registry/api/app.py | 87 +++++++++++++++++-- .../registry/api/models/__init__.py | 2 +- 4 files changed, 88 insertions(+), 15 deletions(-) delete mode 100644 proto-registry/registry/.gitignore diff --git a/proto-registry/.gitignore b/proto-registry/.gitignore index b6e47617..703bcfe5 100644 --- a/proto-registry/.gitignore +++ b/proto-registry/.gitignore @@ -127,3 +127,11 @@ dmypy.json # Pyre type checker .pyre/ + +postgres-data +env.production.real +bazel-bin +bazel-out +bazel-registry +bazel-rules +bazel-testlogs diff --git a/proto-registry/registry/.gitignore b/proto-registry/registry/.gitignore deleted file mode 100644 index 00c5ebd8..00000000 --- a/proto-registry/registry/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -postgres-data -env.production.real -bazel-bin -bazel-out -bazel-registry -bazel-testlogs diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index 579ffb00..b2743bc9 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -1,8 +1,8 @@ import json -from flask import abort +from flask import abort, request from sqlalchemy import asc -from .config import app +from .config import app, db from . import models @app.route("/") @@ -13,8 +13,21 @@ def hello_world(): def subjects(): return json.dumps([dict(x) for x in models.Subject.query.order_by(asc(models.Subject.id)).all()]) +@app.route("/subjects/", methods=['DELETE']) +def delete_subject(subject_name: str) -> str: + subject = models.Subject.query.filter(models.Subject.name == subject_name).first() + if subject is None: + abort(404) + + versions = json.dumps([version.version_id for version in subject.versions]) + + db.session.delete(subject) + db.session.commit() + + return versions + @app.route("/subjects//versions") -def subject_versions(subject_name): +def subject_versions(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: abort(404) @@ -24,16 +37,74 @@ def subject_versions(subject_name): for version in subject.versions ]) -@app.route("/subjects//versions/") -def subject_version(subject_name, version_id): +@app.route("/subjects//versions", methods=["POST"]) +def create_subject_version(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: abort(404) - versions = [v for v in subject.versions if v.version_id == version_id] - if not versions: + data = request.get_json() + + if 'schema' not in data or not data['schema']: + abort(400) + schema: str = data['schema'] + + schema_type_name: str = data.get('schemaType', 'AVRO') + if schema_type_name not in models.SchemaType: + abort(400) + + schema_type = models.SchemaType[schema_type_name] + + references: list[dict] = data.get('references', []) + + next_version = max(version.version_id for version in subject.versions) + 1 + + new_version = models.SubjectVersion( + version_id = next_version, + schema_type = schema_type, + schema = schema, + subject_id = subject.id + ) + db.session.add(new_version) + db.session.commit() + + +@app.route("/subjects//versions/") +def subject_version(subject_name: str, version_id: int) -> str: + version = models.SubjectVersion.query\ + .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + .filter( + models.Subject.name == subject_name and \ + models.SubjectVersion.version_id == version_id + )\ + .first() + + if version is None: + abort(404) + + return json.dumps({ + "subject": version.subject.name, + "id": version.id, + "version": version.version_id, + "schemaType": version.schema_type.name, + "schema": version.schema, + }) + +@app.route("/subjects//versions//schema") +def subject_version_schema(subject_name: str, version_id: int) -> str: + version = models.SubjectVersion.query\ + .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + .filter( + models.Subject.name == subject_name and \ + models.SubjectVersion.version_id == version_id + )\ + .first() + + if version is None: abort(404) - return next(versions).schema + + return version.schema + if __name__ == "__main__": app.run() diff --git a/proto-registry/registry/api/models/__init__.py b/proto-registry/registry/api/models/__init__.py index 27d43ec1..e6bb231d 100644 --- a/proto-registry/registry/api/models/__init__.py +++ b/proto-registry/registry/api/models/__init__.py @@ -1,2 +1,2 @@ from .subject import Subject -from .subject_version import SubjectVersion \ No newline at end of file +from .subject_version import SubjectVersion, SchemaType \ No newline at end of file From 3e73904aa3c30144f191706de03f2921f6e90446 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 02:50:31 +0000 Subject: [PATCH 05/10] Add references to SubjectVersion --- ..._create_subject_version_referehce_table.py | 61 +++++++++++++++++++ .../registry/api/models/subject_version.py | 16 ++++- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py diff --git a/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py b/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py new file mode 100644 index 00000000..9d78d764 --- /dev/null +++ b/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py @@ -0,0 +1,61 @@ +"""create subject_version_referehce table + +Revision ID: 009086ba9d09 +Revises: 86ac1c2d94b4 +Create Date: 2022-07-28 02:17:55.026039 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '009086ba9d09' +down_revision = '86ac1c2d94b4' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'subject_version_references', + sa.Column('referrer_id', sa.Integer), + sa.Column('referred_id', sa.Integer), + ) + op.create_unique_constraint( + "subject_version_references_referrer_referred", "subject_version_references", ["referrer_id", "referred_id"] + ) + op.create_foreign_key( + "subject_version_references_subject_versions_referrer_id", + "subject_version_references", + "subject_versions", + ["referrer_id"], + ["id"], + "CASCADE", + "CASCADE", + ) + op.create_foreign_key( + "subject_version_references_subject_versions_referred_id", + "subject_version_references", + "subject_versions", + ["referred_id"], + ["id"], + "CASCADE", + "CASCADE", + ) + op.create_index( + "subject_version_references_index_referrer_referred", + "subject_version_references", + ["referrer_id", "referred_id"], + unique=True, + ) + op.create_index( + "subject_version_references_index_referred_referrer", + "subject_version_references", + ["referred_id", "referrer_id"], + unique=True, + ) + + +def downgrade(): + op.drop_table("subject_version_references") diff --git a/proto-registry/registry/api/models/subject_version.py b/proto-registry/registry/api/models/subject_version.py index 1e57885b..b874b959 100644 --- a/proto-registry/registry/api/models/subject_version.py +++ b/proto-registry/registry/api/models/subject_version.py @@ -10,6 +10,11 @@ class SchemaType(enum.Enum): PROTOBUF = 2 JSONSCHEMA = 3 +subject_version_reference_table = db.Table( + "subject_version_references", + db.Column("referrer_id", db.Integer, db.ForeignKey("subject_versions.id"), primary_key=True), + db.Column("referred_id", db.Integer, db.ForeignKey("subject_versions.id"), primary_key=True), +) class SubjectVersion(db.Model): __tablename__ = "subject_versions" @@ -20,7 +25,16 @@ class SubjectVersion(db.Model): version_id = db.Column(db.Integer, nullable=False) schema_type = db.Column(Enum(SchemaType), nullable=True, default=SchemaType.AVRO) schema = db.Column(db.Unicode(20000), nullable=False) - + + references = db.relationship( + "SubjectVersion", + secondary=subject_version_reference_table, + primaryjoin=id==subject_version_reference_table.c.referrer_id, + secondaryjoin=id==subject_version_reference_table.c.referred_id, + lazy="subquery", + backref=db.backref("referrers", lazy=True), + ) + subject_id = db.Column(db.Integer, db.ForeignKey(Subject.id), nullable=False) subject = db.relationship( "Subject", back_populates="versions", order_by="desc(SubjectVersion.version_id)" From f6d7019d2e7205812050c406776159ae928a707d Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 03:10:10 +0000 Subject: [PATCH 06/10] Transform reference names in requests --- proto-registry/registry/api/app.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index b2743bc9..caba6e16 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -45,29 +45,41 @@ def create_subject_version(subject_name: str) -> str: data = request.get_json() - if 'schema' not in data or not data['schema']: + if "schema" not in data or not data["schema"]: abort(400) - schema: str = data['schema'] + schema: str = data["schema"] - schema_type_name: str = data.get('schemaType', 'AVRO') + schema_type_name: str = data.get("schemaType", "AVRO") if schema_type_name not in models.SchemaType: abort(400) schema_type = models.SchemaType[schema_type_name] - references: list[dict] = data.get('references', []) + references: list[dict] = data.get("references", []) + reference_names = [f"{reference['subject']}/{reference['version']}" for reference in references] + reference_versions = models.SubjectVersion.query\ + .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + .filter( + f"{models.Subject.name}/{models.SubjectVersion.version_id}" in reference_names + )\ + .all() next_version = max(version.version_id for version in subject.versions) + 1 new_version = models.SubjectVersion( - version_id = next_version, - schema_type = schema_type, - schema = schema, - subject_id = subject.id + version_id=next_version, + schema_type=schema_type, + schema=schema, + subject_id=subject.id, + references=reference_versions, ) db.session.add(new_version) db.session.commit() + return json.dumps({ + "id": new_version.id, + }) + @app.route("/subjects//versions/") def subject_version(subject_name: str, version_id: int) -> str: From dfa0da86f97f9a002db886bac83d8e6374b3f6d8 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 03:24:48 +0000 Subject: [PATCH 07/10] Add additional endpoints --- proto-registry/registry/api/app.py | 64 ++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index caba6e16..7decd9c2 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -26,6 +26,50 @@ def delete_subject(subject_name: str) -> str: return versions +@app.route("/subjects/", methods=['POST']) +def check_subject_schema(subject_name: str) -> str: + subject = models.Subject.query.filter(models.Subject.name == subject_name).first() + if subject is None: + abort(404) + + data = request.get_json() + + if "schema" not in data: + abort(400) + + schema = data["schema"] + schema_type_name: str = data.get("schemaType", "AVRO") + if schema_type_name not in models.SchemaType: + abort(400) + + schema_type = models.SchemaType[schema_type_name] + + references: list[dict] = data.get("references", []) + reference_names = set([f"{reference['subject']}/{reference['version']}" for reference in references]) + + version = models.SubjectVersion.query\ + .filter( + models.SubjectVersion.schema == schema and \ + models.SubjectVersion.schema_type == schema_type + )\ + .first() + + if version is None: + abort(404) + + version_reference_names = set([ + f"{reference['subject']}/{reference['version']}" for reference in version.references + ]) + if reference_names != version_reference_names: + abort(404) + + return json.dumps({ + "subject": version.subject.name, + "id": version.id, + "version": version.version_id, + "schema": version.schema, + }) + @app.route("/subjects//versions") def subject_versions(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() @@ -63,6 +107,8 @@ def create_subject_version(subject_name: str) -> str: f"{models.Subject.name}/{models.SubjectVersion.version_id}" in reference_names )\ .all() + if len(reference_versions) != len(references): + abort(400) next_version = max(version.version_id for version in subject.versions) + 1 @@ -102,6 +148,24 @@ def subject_version(subject_name: str, version_id: int) -> str: "schema": version.schema, }) +@app.route("/subjects//versions//referencedby") +def subject_version_referencedby(subject_name: str, version_id: int) -> str: + version = models.SubjectVersion.query\ + .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + .filter( + models.Subject.name == subject_name and \ + models.SubjectVersion.version_id == version_id + )\ + .first() + + if version is None: + abort(404) + + return json.dumps([ + referer.id for referer in version.referrers + ]) + + @app.route("/subjects//versions//schema") def subject_version_schema(subject_name: str, version_id: int) -> str: version = models.SubjectVersion.query\ From ecd73cc962ac9f295f565bc40ab01774318eef4c Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 03:29:41 +0000 Subject: [PATCH 08/10] Add references to endpoint --- proto-registry/registry/api/app.py | 3 +++ proto-registry/registry/api/models/subject_version.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index 7decd9c2..2095ddfc 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -140,12 +140,15 @@ def subject_version(subject_name: str, version_id: int) -> str: if version is None: abort(404) + reference_names = [reference.unique_name() for reference in version.references] + return json.dumps({ "subject": version.subject.name, "id": version.id, "version": version.version_id, "schemaType": version.schema_type.name, "schema": version.schema, + "references": reference_names, }) @app.route("/subjects//versions//referencedby") diff --git a/proto-registry/registry/api/models/subject_version.py b/proto-registry/registry/api/models/subject_version.py index b874b959..0d72e5ca 100644 --- a/proto-registry/registry/api/models/subject_version.py +++ b/proto-registry/registry/api/models/subject_version.py @@ -47,4 +47,7 @@ def __dict__(self): return {c.name: getattr(self, c.name) for c in self.__table__.columns} def to_json(self): - return json.dumps(dict(self)) \ No newline at end of file + return json.dumps(dict(self)) + + def unique_name(self): + return f"{self.subject.name}/{self.version_id}" \ No newline at end of file From b4d4b0749e51668386d52564d34020f24c302900 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 01:21:28 -0400 Subject: [PATCH 09/10] In version-creation endpoint, also create subjects; fix some bugs too (#4) --- proto-registry/registry/api/app.py | 46 ++++++++++++------- proto-registry/registry/api/models/subject.py | 8 +--- .../registry/api/models/subject_version.py | 8 +--- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index 2095ddfc..618e2739 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -10,8 +10,11 @@ def hello_world(): return "

Hello, World!

" @app.route("/subjects") -def subjects(): - return json.dumps([dict(x) for x in models.Subject.query.order_by(asc(models.Subject.id)).all()]) +def get_subjects(): + subjects = models.Subject.query.order_by(asc(models.Subject.id)).all() + return json.dumps([ + subject.name for subject in subjects + ]) @app.route("/subjects/", methods=['DELETE']) def delete_subject(subject_name: str) -> str: @@ -71,7 +74,7 @@ def check_subject_schema(subject_name: str) -> str: }) @app.route("/subjects//versions") -def subject_versions(subject_name: str) -> str: +def get_subject_versions(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: abort(404) @@ -85,50 +88,59 @@ def subject_versions(subject_name: str) -> str: def create_subject_version(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: - abort(404) + subject = models.Subject(name=subject_name) + db.session.add(subject) - data = request.get_json() + json_data = request.get_json() - if "schema" not in data or not data["schema"]: + if "schema" not in json_data or not json_data["schema"]: abort(400) - schema: str = data["schema"] + schema: str = json_data["schema"] - schema_type_name: str = data.get("schemaType", "AVRO") - if schema_type_name not in models.SchemaType: + schema_type_name: str = json_data.get("schemaType", "AVRO") + if models.SchemaType[schema_type_name] is None: abort(400) schema_type = models.SchemaType[schema_type_name] - references: list[dict] = data.get("references", []) + references: list[dict] = json_data.get("references", []) + + reference_subjects = models.Subject.query.filter(models.Subject.name in [reference['subject'] for reference in references]).all() + reference_names = [f"{reference['subject']}/{reference['version']}" for reference in references] reference_versions = models.SubjectVersion.query\ .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ .filter( - f"{models.Subject.name}/{models.SubjectVersion.version_id}" in reference_names + models.SubjectVersion.subject_id in [subject.id for subject in reference_subjects] )\ .all() + reference_versions = [version for version in reference_versions if f"{version.subject.name}/{version.version_id}" in reference_names] if len(reference_versions) != len(references): abort(400) - next_version = max(version.version_id for version in subject.versions) + 1 + subject_versions = subject.versions + if not subject_versions: + next_version = 1 + else: + next_version = max(version.version_id for version in subject.versions) + 1 new_version = models.SubjectVersion( version_id=next_version, schema_type=schema_type, schema=schema, - subject_id=subject.id, + subject=subject, references=reference_versions, ) db.session.add(new_version) db.session.commit() return json.dumps({ - "id": new_version.id, + "id": subject.id, }) @app.route("/subjects//versions/") -def subject_version(subject_name: str, version_id: int) -> str: +def get_subject_version(subject_name: str, version_id: int) -> str: version = models.SubjectVersion.query\ .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ .filter( @@ -152,7 +164,7 @@ def subject_version(subject_name: str, version_id: int) -> str: }) @app.route("/subjects//versions//referencedby") -def subject_version_referencedby(subject_name: str, version_id: int) -> str: +def get_subject_version_referencedby(subject_name: str, version_id: int) -> str: version = models.SubjectVersion.query\ .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ .filter( @@ -170,7 +182,7 @@ def subject_version_referencedby(subject_name: str, version_id: int) -> str: @app.route("/subjects//versions//schema") -def subject_version_schema(subject_name: str, version_id: int) -> str: +def get_subject_version_schema(subject_name: str, version_id: int) -> str: version = models.SubjectVersion.query\ .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ .filter( diff --git a/proto-registry/registry/api/models/subject.py b/proto-registry/registry/api/models/subject.py index 1baf0126..d1ad3b73 100644 --- a/proto-registry/registry/api/models/subject.py +++ b/proto-registry/registry/api/models/subject.py @@ -15,10 +15,4 @@ class Subject(db.Model): ) def __repr__(self): - return "".format(id=self.id) - - def __dict__(self): - return {c.name: getattr(self, c.name) for c in self.__table__.columns} - - def to_json(self): - return json.dumps(dict(self)) \ No newline at end of file + return f"" diff --git a/proto-registry/registry/api/models/subject_version.py b/proto-registry/registry/api/models/subject_version.py index 0d72e5ca..9b8fe933 100644 --- a/proto-registry/registry/api/models/subject_version.py +++ b/proto-registry/registry/api/models/subject_version.py @@ -41,13 +41,7 @@ class SubjectVersion(db.Model): ) def __repr__(self): - return "".format(id=self.id) - - def __dict__(self): - return {c.name: getattr(self, c.name) for c in self.__table__.columns} - - def to_json(self): - return json.dumps(dict(self)) + return f"" def unique_name(self): return f"{self.subject.name}/{self.version_id}" \ No newline at end of file From 6daa4b7e22b598b3b345c5ed6e7ac363776be086 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Thu, 28 Jul 2022 02:11:17 -0400 Subject: [PATCH 10/10] Add pre-commit config, and run it over everything (#5) --- proto-registry/.pre-commit-config.yaml | 16 ++ proto-registry/registry/api/app.py | 166 +++++++++++------- ..._create_subject_version_referehce_table.py | 14 +- ...c1c2d94b4_create_subject_versions_table.py | 23 ++- .../9da1de15cd6a_create_subjects_table.py | 14 +- .../registry/api/models/__init__.py | 2 +- proto-registry/registry/api/models/subject.py | 4 +- .../registry/api/models/subject_version.py | 23 ++- proto-registry/registry/api/requirements.txt | 10 +- .../registry/scripts/start-server.sh | 2 +- proto-registry/registry/toolchain/BUILD | 1 - 11 files changed, 172 insertions(+), 103 deletions(-) create mode 100644 proto-registry/.pre-commit-config.yaml diff --git a/proto-registry/.pre-commit-config.yaml b/proto-registry/.pre-commit-config.yaml new file mode 100644 index 00000000..82860ee3 --- /dev/null +++ b/proto-registry/.pre-commit-config.yaml @@ -0,0 +1,16 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: no-commit-to-branch + args: [--branch, staging, --branch, main] + - id: requirements-txt-fixer + - repo: https://github.com/psf/black + rev: 22.6.0 + hooks: + - id: black diff --git a/proto-registry/registry/api/app.py b/proto-registry/registry/api/app.py index 618e2739..5039cfba 100644 --- a/proto-registry/registry/api/app.py +++ b/proto-registry/registry/api/app.py @@ -5,18 +5,19 @@ from .config import app, db from . import models + @app.route("/") def hello_world(): return "

Hello, World!

" -@app.route("/subjects") + +@app.route("/subjects/") def get_subjects(): subjects = models.Subject.query.order_by(asc(models.Subject.id)).all() - return json.dumps([ - subject.name for subject in subjects - ]) + return json.dumps([subject.name for subject in subjects]) + -@app.route("/subjects/", methods=['DELETE']) +@app.route("/subjects//", methods=["DELETE"]) def delete_subject(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: @@ -29,7 +30,8 @@ def delete_subject(subject_name: str) -> str: return versions -@app.route("/subjects/", methods=['POST']) + +@app.route("/subjects//", methods=["POST"]) def check_subject_schema(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: @@ -48,43 +50,47 @@ def check_subject_schema(subject_name: str) -> str: schema_type = models.SchemaType[schema_type_name] references: list[dict] = data.get("references", []) - reference_names = set([f"{reference['subject']}/{reference['version']}" for reference in references]) + reference_names = set( + [f"{reference['subject']}/{reference['version']}" for reference in references] + ) - version = models.SubjectVersion.query\ - .filter( - models.SubjectVersion.schema == schema and \ - models.SubjectVersion.schema_type == schema_type - )\ - .first() + version = models.SubjectVersion.query.filter( + models.SubjectVersion.schema == schema + and models.SubjectVersion.schema_type == schema_type + ).first() if version is None: abort(404) - version_reference_names = set([ - f"{reference['subject']}/{reference['version']}" for reference in version.references - ]) + version_reference_names = set( + [ + f"{reference['subject']}/{reference['version']}" + for reference in version.references + ] + ) if reference_names != version_reference_names: abort(404) - return json.dumps({ - "subject": version.subject.name, - "id": version.id, - "version": version.version_id, - "schema": version.schema, - }) + return json.dumps( + { + "subject": version.subject.name, + "id": version.id, + "version": version.version_id, + "schema": version.schema, + } + ) + -@app.route("/subjects//versions") +@app.route("/subjects//versions/") def get_subject_versions(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: abort(404) - return json.dumps([ - version.version_id - for version in subject.versions - ]) + return json.dumps([version.version_id for version in subject.versions]) -@app.route("/subjects//versions", methods=["POST"]) + +@app.route("/subjects//versions/", methods=["POST"]) def create_subject_version(subject_name: str) -> str: subject = models.Subject.query.filter(models.Subject.name == subject_name).first() if subject is None: @@ -105,16 +111,28 @@ def create_subject_version(subject_name: str) -> str: references: list[dict] = json_data.get("references", []) - reference_subjects = models.Subject.query.filter(models.Subject.name in [reference['subject'] for reference in references]).all() - - reference_names = [f"{reference['subject']}/{reference['version']}" for reference in references] - reference_versions = models.SubjectVersion.query\ - .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + reference_subjects = models.Subject.query.filter( + models.Subject.name in [reference["subject"] for reference in references] + ).all() + + reference_names = [ + f"{reference['subject']}/{reference['version']}" for reference in references + ] + reference_versions = ( + models.SubjectVersion.query.join( + models.Subject, models.SubjectVersion.subject_id == models.Subject.id + ) .filter( - models.SubjectVersion.subject_id in [subject.id for subject in reference_subjects] - )\ + models.SubjectVersion.subject_id + in [subject.id for subject in reference_subjects] + ) .all() - reference_versions = [version for version in reference_versions if f"{version.subject.name}/{version.version_id}" in reference_names] + ) + reference_versions = [ + version + for version in reference_versions + if f"{version.subject.name}/{version.version_id}" in reference_names + ] if len(reference_versions) != len(references): abort(400) @@ -134,62 +152,74 @@ def create_subject_version(subject_name: str) -> str: db.session.add(new_version) db.session.commit() - return json.dumps({ - "id": subject.id, - }) + return json.dumps( + { + "id": subject.id, + } + ) -@app.route("/subjects//versions/") +@app.route("/subjects//versions//") def get_subject_version(subject_name: str, version_id: int) -> str: - version = models.SubjectVersion.query\ - .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + version = ( + models.SubjectVersion.query.join( + models.Subject, models.SubjectVersion.subject_id == models.Subject.id + ) .filter( - models.Subject.name == subject_name and \ - models.SubjectVersion.version_id == version_id - )\ + models.Subject.name == subject_name + and models.SubjectVersion.version_id == version_id + ) .first() + ) if version is None: abort(404) reference_names = [reference.unique_name() for reference in version.references] - return json.dumps({ - "subject": version.subject.name, - "id": version.id, - "version": version.version_id, - "schemaType": version.schema_type.name, - "schema": version.schema, - "references": reference_names, - }) + return json.dumps( + { + "subject": version.subject.name, + "id": version.id, + "version": version.version_id, + "schemaType": version.schema_type.name, + "schema": version.schema, + "references": reference_names, + } + ) + -@app.route("/subjects//versions//referencedby") +@app.route("/subjects//versions//referencedby/") def get_subject_version_referencedby(subject_name: str, version_id: int) -> str: - version = models.SubjectVersion.query\ - .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + version = ( + models.SubjectVersion.query.join( + models.Subject, models.SubjectVersion.subject_id == models.Subject.id + ) .filter( - models.Subject.name == subject_name and \ - models.SubjectVersion.version_id == version_id - )\ + models.Subject.name == subject_name + and models.SubjectVersion.version_id == version_id + ) .first() + ) if version is None: abort(404) - return json.dumps([ - referer.id for referer in version.referrers - ]) + return json.dumps([referer.id for referer in version.referrers]) -@app.route("/subjects//versions//schema") +@app.route("/subjects//versions//schema/") def get_subject_version_schema(subject_name: str, version_id: int) -> str: - version = models.SubjectVersion.query\ - .join(models.Subject, models.SubjectVersion.subject_id == models.Subject.id)\ + version = ( + models.SubjectVersion.query.join( + models.Subject, models.SubjectVersion.subject_id == models.Subject.id + ) .filter( - models.Subject.name == subject_name and \ - models.SubjectVersion.version_id == version_id - )\ + models.Subject.name == subject_name + and models.SubjectVersion.version_id == version_id + ) .first() + ) if version is None: abort(404) diff --git a/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py b/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py index 9d78d764..336db7f7 100644 --- a/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py +++ b/proto-registry/registry/api/migrations/versions/009086ba9d09_create_subject_version_referehce_table.py @@ -10,20 +10,22 @@ # revision identifiers, used by Alembic. -revision = '009086ba9d09' -down_revision = '86ac1c2d94b4' +revision = "009086ba9d09" +down_revision = "86ac1c2d94b4" branch_labels = None depends_on = None def upgrade(): op.create_table( - 'subject_version_references', - sa.Column('referrer_id', sa.Integer), - sa.Column('referred_id', sa.Integer), + "subject_version_references", + sa.Column("referrer_id", sa.Integer), + sa.Column("referred_id", sa.Integer), ) op.create_unique_constraint( - "subject_version_references_referrer_referred", "subject_version_references", ["referrer_id", "referred_id"] + "subject_version_references_referrer_referred", + "subject_version_references", + ["referrer_id", "referred_id"], ) op.create_foreign_key( "subject_version_references_subject_versions_referrer_id", diff --git a/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py b/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py index 4098b3f4..fab1a876 100644 --- a/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py +++ b/proto-registry/registry/api/migrations/versions/86ac1c2d94b4_create_subject_versions_table.py @@ -12,11 +12,12 @@ # revision identifiers, used by Alembic. -revision = '86ac1c2d94b4' -down_revision = '9da1de15cd6a' +revision = "86ac1c2d94b4" +down_revision = "9da1de15cd6a" branch_labels = None depends_on = None + class SchemaType(enum.Enum): AVRO = 1 PROTOBUF = 2 @@ -25,13 +26,17 @@ class SchemaType(enum.Enum): def upgrade(): op.create_table( - 'subject_versions', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('created', sa.DateTime, nullable=False, default=datetime.datetime.utcnow), - sa.Column('version_id', sa.Integer, nullable=False), - sa.Column('subject_id', sa.Integer, nullable=False), - sa.Column('schema_type', sa.Enum(SchemaType), nullable=False, default=SchemaType.AVRO), - sa.Column('schema', sa.Unicode(20000), nullable=False), + "subject_versions", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow + ), + sa.Column("version_id", sa.Integer, nullable=False), + sa.Column("subject_id", sa.Integer, nullable=False), + sa.Column( + "schema_type", sa.Enum(SchemaType), nullable=False, default=SchemaType.AVRO + ), + sa.Column("schema", sa.Unicode(20000), nullable=False), ) op.create_unique_constraint( "subject_versions_version_id", "subject_versions", ["subject_id", "version_id"] diff --git a/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py b/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py index 6f5551b8..a132b7d7 100644 --- a/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py +++ b/proto-registry/registry/api/migrations/versions/9da1de15cd6a_create_subjects_table.py @@ -1,7 +1,7 @@ """create subjects table Revision ID: 9da1de15cd6a -Revises: +Revises: Create Date: 2022-07-21 04:15:20.943034 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. -revision = '9da1de15cd6a' +revision = "9da1de15cd6a" down_revision = None branch_labels = None depends_on = None @@ -19,10 +19,12 @@ def upgrade(): op.create_table( - 'subjects', - sa.Column('id', sa.Integer, primary_key=True), - sa.Column('created', sa.DateTime, nullable=False, default=datetime.datetime.utcnow), - sa.Column('name', sa.Unicode(2000), nullable=False), + "subjects", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "created", sa.DateTime, nullable=False, default=datetime.datetime.utcnow + ), + sa.Column("name", sa.Unicode(2000), nullable=False), ) op.create_index( "subjects_name", diff --git a/proto-registry/registry/api/models/__init__.py b/proto-registry/registry/api/models/__init__.py index e6bb231d..cf84fb46 100644 --- a/proto-registry/registry/api/models/__init__.py +++ b/proto-registry/registry/api/models/__init__.py @@ -1,2 +1,2 @@ from .subject import Subject -from .subject_version import SubjectVersion, SchemaType \ No newline at end of file +from .subject_version import SubjectVersion, SchemaType diff --git a/proto-registry/registry/api/models/subject.py b/proto-registry/registry/api/models/subject.py index d1ad3b73..04c5985b 100644 --- a/proto-registry/registry/api/models/subject.py +++ b/proto-registry/registry/api/models/subject.py @@ -11,7 +11,9 @@ class Subject(db.Model): ) name = db.Column(db.Unicode(2000), nullable=False, unique=True) versions = db.relationship( - "SubjectVersion", back_populates="subject", order_by="desc(SubjectVersion.version_id)" + "SubjectVersion", + back_populates="subject", + order_by="desc(SubjectVersion.version_id)", ) def __repr__(self): diff --git a/proto-registry/registry/api/models/subject_version.py b/proto-registry/registry/api/models/subject_version.py index 9b8fe933..c723e5a7 100644 --- a/proto-registry/registry/api/models/subject_version.py +++ b/proto-registry/registry/api/models/subject_version.py @@ -5,17 +5,30 @@ import enum import json + class SchemaType(enum.Enum): AVRO = 1 PROTOBUF = 2 JSONSCHEMA = 3 + subject_version_reference_table = db.Table( "subject_version_references", - db.Column("referrer_id", db.Integer, db.ForeignKey("subject_versions.id"), primary_key=True), - db.Column("referred_id", db.Integer, db.ForeignKey("subject_versions.id"), primary_key=True), + db.Column( + "referrer_id", + db.Integer, + db.ForeignKey("subject_versions.id"), + primary_key=True, + ), + db.Column( + "referred_id", + db.Integer, + db.ForeignKey("subject_versions.id"), + primary_key=True, + ), ) + class SubjectVersion(db.Model): __tablename__ = "subject_versions" id = db.Column(db.Integer, primary_key=True) @@ -29,8 +42,8 @@ class SubjectVersion(db.Model): references = db.relationship( "SubjectVersion", secondary=subject_version_reference_table, - primaryjoin=id==subject_version_reference_table.c.referrer_id, - secondaryjoin=id==subject_version_reference_table.c.referred_id, + primaryjoin=id == subject_version_reference_table.c.referrer_id, + secondaryjoin=id == subject_version_reference_table.c.referred_id, lazy="subquery", backref=db.backref("referrers", lazy=True), ) @@ -44,4 +57,4 @@ def __repr__(self): return f"" def unique_name(self): - return f"{self.subject.name}/{self.version_id}" \ No newline at end of file + return f"{self.subject.name}/{self.version_id}" diff --git a/proto-registry/registry/api/requirements.txt b/proto-registry/registry/api/requirements.txt index a3b1b7e2..9e245627 100644 --- a/proto-registry/registry/api/requirements.txt +++ b/proto-registry/registry/api/requirements.txt @@ -1,12 +1,12 @@ +black Flask +flask-cors Flask-Migrate +Flask-SQLAlchemy importlib-metadata importlib-resources -typing-extensions +pre-commit psycopg2 pylint sqlalchemy -Flask-SQLAlchemy -flask-cors -pre-commit -black +typing-extensions diff --git a/proto-registry/registry/scripts/start-server.sh b/proto-registry/registry/scripts/start-server.sh index c1606515..cdb8b962 100755 --- a/proto-registry/registry/scripts/start-server.sh +++ b/proto-registry/registry/scripts/start-server.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -docker compose --env-file env.development up \ No newline at end of file +docker compose --env-file env.development up diff --git a/proto-registry/registry/toolchain/BUILD b/proto-registry/registry/toolchain/BUILD index b4910540..f3697a4b 100644 --- a/proto-registry/registry/toolchain/BUILD +++ b/proto-registry/registry/toolchain/BUILD @@ -27,4 +27,3 @@ toolchain( toolchain = ":container_py_runtime_pair", toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) -