From 67ce71d9c698bfc1f7258cf1f72474706d970f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Murawski?= Date: Sun, 14 Jul 2024 23:27:52 +0200 Subject: [PATCH] feat: Adde GHA IC --- .github/workflows/main.yml | 23 ------ .github/workflows/release.yml | 46 ++++++++++++ .github/workflows/test.yml | 23 ++++++ gen/client_generator.py | 68 +++++++++++++++--- gen/helpers.py | 4 +- gen/run_generation.py | 3 +- gen/tests_generator.py | 127 ++++++++++++++++++++++++---------- requirements.txt | 1 - setup.py | 46 ------------ 9 files changed, 222 insertions(+), 119 deletions(-) delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5637de9..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Tests -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: -jobs: - publish: - name: "Publish in PYPI" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Publish - env: - USERNAME: ${{ secrets.USERNAME }} - PASSWORD: ${{ secrets.PASSWORD }} - run: | - pip3 install wheel twine - python3 setup.py sdist bdist_wheel - python3 -m twine check dist/* - python3 -m twine upload --skip-existing dist/* --verbose -u ${USERNAME} -p ${PASSWORD} - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..11fa3f9 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Publish Python Package + +on: + release: + types: [published] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 + poetry config virtualenvs.create false + poetry install + - name: Install + run: poetry install --with test + - name: Run Pytest + run: poetry run pytest + deploy: + runs-on: ubuntu-latest + needs: [ tests ] + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 + poetry config virtualenvs.create false + poetry install + - name: Build package + run: poetry build + - name: Publish package + run: poetry publish + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..f0382e4 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,23 @@ +name: Test + +on: [pull_request, workflow_dispatch] + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + curl -sSL https://install.python-poetry.org | python3 - --version 1.8.2 + poetry config virtualenvs.create false + poetry install + - name: Install + run: poetry install --with test + - name: Run Pytest + run: poetry run pytest \ No newline at end of file diff --git a/gen/client_generator.py b/gen/client_generator.py index d230e68..403507b 100644 --- a/gen/client_generator.py +++ b/gen/client_generator.py @@ -63,19 +63,39 @@ def get_type_for_param(param: dict, swagger: dict | None = None) -> str: param_type = param.get("type") schema_type = get_path(param, "schema.type", None) - ref_path = get_path(param, "$ref", None) + ref_path = get_path(param, "$ref", get_path(param, "schema.$ref", None)) all_of = get_path(param, "allOf.0.$ref", None) any_of = get_path(param, "anyOf", None) + if any_of: - raise Exception(param) + types = [] + _list = any_of + + for i in _list: + obj = get_path( + swagger, + i["$ref"][2:], + None, + "/" + ) + _type = obj["type"] + items_type = get_path(obj, "items.type", "object") + + types.append( + COMPLEX_TYPES_MAP[(_type, items_type)] + ) + return "Union[%s]" % ", ".join(types) while ref_path or all_of: ref_path = ref_path or all_of ref = get_path(swagger, ref_path[2:], separator="/", default=None) - ref_path = get_path(ref, "$ref", None) - all_of = get_path(ref, "allOf.0.$ref", None) + if ref_path.endswith(".json"): + return dict.__name__ + + ref_path = get_path(ref, "$ref", default=None) + all_of = get_path(ref, "allOf.0.$ref", default=None) if not ref_path and not all_of: return SIMPLE_TYPES_MAP[ref["type"]] @@ -86,8 +106,36 @@ def get_type_for_param(param: dict, swagger: dict | None = None) -> str: if schema_type in SIMPLE_TYPES_MAP: return SIMPLE_TYPES_MAP[schema_type] + elif schema_type in COMPLEX_TYPES: + items_type_path = get_path( + param, + "schema.items.type", + get_path( + param, + "schema.items.$ref", + None + ) + ) + while True: + ref = get_path(swagger, items_type_path[2:], separator="/", default={}) + + if isinstance(ref, dict) and "$ref" in ref: + items_type_path = ref["$ref"] + continue + else: + _type = ref.get("type", items_type_path) + return COMPLEX_TYPES_MAP[(schema_type, _type)] + elif param_type in COMPLEX_TYPES: - items_type = get_path(param, "schema.items.type", default="object") + items_type = get_path( + param, + "schema.items.type", + default=get_path( + param, + "items.type", + default="object" + ) + ) return COMPLEX_TYPES_MAP[(param_type, items_type)] elif param_type in SIMPLE_TYPES_MAP: return SIMPLE_TYPES_MAP[param_type] @@ -164,7 +212,7 @@ def create_request_block( requests_lines[1] = requests_lines[1][:-1] + ", params=params)" if has_body: - requests_lines[1] = requests_lines[1][:-1] + ", json=data)" + requests_lines[1] = requests_lines[1][:-1] + ", json=request_data)" return "\n".join(requests_lines) @@ -398,7 +446,7 @@ def create_validation_block(path: str, method: str, swagger: dict, name: str) -> model_class = schema_path.rsplit("/", 1)[-1] lines = [ "if self.validation:", - "%svalidate_data(data, %s, %sAPIError)" % (INDENT, model_class, name), + "%svalidate_data(request_data, %s, %sAPIError)" % (INDENT, model_class, name), ] return "\n".join(lines) return "" @@ -411,10 +459,10 @@ def create_body_block(required: list[dict], not_required: list[dict]) -> str: return "" if not required: - lines.append("data = {}") + lines.append("request_data = {}") else: lines.append( - "data = {" + "request_data = {" ) for p in required: converted_name = convert_to_snake_case(p['name']) @@ -432,7 +480,7 @@ def create_body_block(required: list[dict], not_required: list[dict]) -> str: ) lines.append( - "%sdata['%s'] = %s" % (INDENT, p['name'], converted_name) + "%srequest_data['%s'] = %s" % (INDENT, p['name'], converted_name) ) return "\n".join(lines) diff --git a/gen/helpers.py b/gen/helpers.py index 531e4c9..7b1e89d 100644 --- a/gen/helpers.py +++ b/gen/helpers.py @@ -3,9 +3,11 @@ import sys from subprocess import PIPE, Popen from typing import Any -from .swagger import SwaggerDoc + import yaml +from .swagger import SwaggerDoc + NOT_SET = object() diff --git a/gen/run_generation.py b/gen/run_generation.py index 53b6e7b..c5d389c 100644 --- a/gen/run_generation.py +++ b/gen/run_generation.py @@ -4,7 +4,7 @@ import requests from gen.client_generator import generate_clients -from gen.helpers import generate_models, put_inits, ruff_format +from gen.helpers import generate_models, ruff_format WORKDIR = os.getcwd() @@ -48,7 +48,6 @@ def remove_path_if_empty(path: str): models_path = os.path.join(module_path, "models") rel_module_path = module_path.replace(WORKDIR, ".") os.makedirs(rel_module_path, exist_ok=True) - put_inits(rel_module_path) swagger_path = os.path.join(module_path, "swagger.yaml") diff --git a/gen/tests_generator.py b/gen/tests_generator.py index ae4ccc5..e0e95ac 100644 --- a/gen/tests_generator.py +++ b/gen/tests_generator.py @@ -1,5 +1,6 @@ import os import shutil +from importlib import import_module from gen.helpers import put_inits from osdu_client.client import get_service_client @@ -12,6 +13,7 @@ 'str | None': '"text"', 'str': '"text"', int | None: '10', + 'int | None': '10', dict: '{}', 'dict': '{}', bool | None: 'False', @@ -21,11 +23,16 @@ 'list[dict]': '[{}]', list[dict] | None: '[{}]', dict | None: '{}', - list[str] | None: '["text"]' + 'dict | None': '{}', + list[str] | None: '["text"]', + 'list[str] | None': '["text"]' } def create_conftest(module_path: str, name: str): + module = import_module(f"osdu_client.services.{name.lower()}") + versions = getattr(module, "VERSIONS", {}) + import_lines = [ "import pytest", "import os", @@ -62,6 +69,15 @@ def create_conftest(module_path: str, name: str): f"def {name.lower()}_client(auth_backend: AuthBackendInterface) -> OSDUAPIClient:", f'{INDENT}return OSDUAPI.client("{name.lower()}", auth_backend=auth_backend)', ] + if versions: + body = body[:-3] + for version in versions: + body += [ + '@pytest.fixture(scope="session")', + f"def {name.lower()}_client_{version}(auth_backend: AuthBackendInterface) -> OSDUAPIClient:", + f'{INDENT}return OSDUAPI.client("{name.lower()}", auth_backend=auth_backend, version="{version}")', + "\n" + ] with open(os.path.join(module_path, "conftest.py"), "w") as file: for line in import_lines: @@ -74,41 +90,80 @@ def create_tests( name: str, tests_path: str, ): - - client_class = get_service_client(name.lower()) - methods = [ - getattr(client_class, method_name) - for method_name in dir(client_class) - if callable(getattr(client_class, method_name)) and hasattr( - getattr(client_class, method_name), "__annotations__" - ) and method_name not in ("__init__") and getattr(client_class, method_name).__class__.__name__ == "function" - ] - - import_template = '\n'.join([ - "from {module_path} import {class_name}", - "\n", - ]) - test_template = '\n'.join([ - "def test_{module}_{method_name}({module}_api_server, {module}_client: {class_name}):", - "{indent}{module}_client.{method_name}(" - ]) - - with open(tests_path, "w") as f: - _import = import_template.format(module=name, class_name=client_class.__name__, - module_path=client_class.__module__) - f.write(_import) - for method in methods: - test = test_template.format( - module=name.lower(), - class_name=client_class.__name__, - method_name=method.__name__, - indent=INDENT, - ) - for k, v in method.__annotations__.items(): - if k != "return": - test += "\n"+(INDENT*2)+f'{k}={TYPE_DEFAULTS[v]},' - test += f"\n{INDENT})\n\n" - f.write(test) + module = import_module(f"osdu_client.services.{name.lower()}") + versions = getattr(module, "VERSIONS", {}) + if versions: + for version, client_class in versions.items(): + + methods = [ + getattr(client_class, method_name) + for method_name in dir(client_class) + if callable(getattr(client_class, method_name)) and hasattr( + getattr(client_class, method_name), "__annotations__" + ) and method_name not in ("__init__") and getattr(client_class, method_name).__class__.__name__ == "function" + ] + import_template = '\n'.join([ + "from {module_path} import {class_name}", + "\n", + ]) + test_template = '\n'.join([ + "def test_{module}_{method_name}({module}_api_server, {module}_client_{version}: {class_name}):", + "{indent}{module}_client_{version}.{method_name}(" + ]) + versioned_tests_path = tests_path.replace(name.lower()+".py", f"{name.lower()}_{version}.py") + with open(versioned_tests_path, "w") as f: + _import = import_template.format(module=name, class_name=client_class.__name__, + module_path=client_class.__module__) + f.write(_import) + for method in methods: + test = test_template.format( + module=name.lower(), + class_name=client_class.__name__, + method_name=method.__name__, + indent=INDENT, + version=version + ) + for k, v in method.__annotations__.items(): + if k != "return": + test += "\n"+(INDENT*2)+f'{k}={TYPE_DEFAULTS[v]},' + test += f"\n{INDENT})\n\n" + f.write(test) + + else: + client_class = get_service_client(name.lower()) + methods = [ + getattr(client_class, method_name) + for method_name in dir(client_class) + if callable(getattr(client_class, method_name)) and hasattr( + getattr(client_class, method_name), "__annotations__" + ) and method_name not in ("__init__") and getattr(client_class, method_name).__class__.__name__ == "function" + ] + + import_template = '\n'.join([ + "from {module_path} import {class_name}", + "\n", + ]) + test_template = '\n'.join([ + "def test_{module}_{method_name}({module}_api_server, {module}_client: {class_name}):", + "{indent}{module}_client.{method_name}(" + ]) + + with open(tests_path, "w") as f: + _import = import_template.format(module=name, class_name=client_class.__name__, + module_path=client_class.__module__) + f.write(_import) + for method in methods: + test = test_template.format( + module=name.lower(), + class_name=client_class.__name__, + method_name=method.__name__, + indent=INDENT, + ) + for k, v in method.__annotations__.items(): + if k != "return": + test += "\n"+(INDENT*2)+f'{k}={TYPE_DEFAULTS[v]},' + test += f"\n{INDENT})\n\n" + f.write(test) def generate_tests( diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 309476d..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests>=2.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 37493de..0000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -import codecs -import os.path -from signal import raise_signal - -from setuptools import find_packages, setup - -HERE = os.path.abspath(os.path.dirname(__file__)) - - -def read(*parts): - return codecs.open(os.path.join(HERE, *parts), 'r').read() - - -def get_dependencies(): - with open("requirements.txt", encoding="utf-8") as fh: - return fh.read().splitlines() - - -setup( - name='osdu_client', - version='0.3.2', - author="Michal Murawski", - author_email="mmurawski777@gmail.com", - description="OSDU API Client", - long_description=read('README.md'), - long_description_content_type="text/markdown", - url="https://github.com/micmurawski/osdu-client/", - #package_dir={"": "src"}, - packages=find_packages( - where=".", - exclude=( - 'build', - 'tests', - )), - install_requires=get_dependencies(), - include_package_data=True, - python_requires=">=3.7,<4.0", - license="MIT", - classifiers=[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8" - ], -)