From 36a6b608af1eea421ff33ac47f552ca5bb56a6cc Mon Sep 17 00:00:00 2001 From: hanyajun <1581532052@qq.com> Date: Fri, 5 Jul 2024 10:39:53 +0800 Subject: [PATCH] feat: apigw-manager support stages (#174) --- sdks/apigw-manager/README.md | 133 ++++++++++-------- sdks/apigw-manager/README.rst | 7 - .../files/support-files/definition.yaml | 87 +++++------- .../support-files/definition.yaml | 85 +++++------ .../support-files/definition.yaml | 42 ++---- sdks/apigw-manager/pyproject.toml | 4 +- .../src/apigw_manager/apigw/helper.py | 8 +- .../management/commands/sync_apigw_stage.py | 18 ++- 8 files changed, 186 insertions(+), 198 deletions(-) diff --git a/sdks/apigw-manager/README.md b/sdks/apigw-manager/README.md index 248477ed..66c57652 100644 --- a/sdks/apigw-manager/README.md +++ b/sdks/apigw-manager/README.md @@ -74,9 +74,63 @@ definition.yaml 中可以使用 Django 模版语法引用和渲染变量，内 推荐在一个文件中统一进行定义，用命名空间区分不同配置间的定义，definition.yaml 样例： +目前有两种配置文件版本：spec_version=1/2,主要区别就是stage相关的配置方式上有一些不一样。 +新接入系统请使用 spec_version=2， 旧有系统如果需要配置多个stage/配置多个backend， 建议也升级到spec_version=2并变更相关yaml配置。 +区别如下： +spec_version: 1 ```yaml -# definition.yaml 配置文件版本号，必填，固定值 1 +# definition.yaml 配置文件版本号，必填，固定值 1/2 +# 1：key为stage; 只支持单个 stage， 并且 proxy_http只能配置一个后端服务 spec_version: 1 +stage: + name: "prod" + description: "描述" + description_en: "English description" + proxy_http: + timeout: "65" + upstreams: + loadbalance: "roundrobin" + hosts: + - host: "http://httpbin.org" + weight: 100 +``` + +spec_version: 2 +```yaml +# definition.yaml 配置文件版本号，必填，固定值 1/2 +# 2：key为stages; 支持多个stages，并且每个stage可以配置多个backend后端服务 +spec_version: 2 +stages: + - name: "prod" + description: "描述" + description_en: "English description" + vars: + status_500: "500" + backends: + - name: "default" + config: + timeout: 60 + loadbalance: "roundrobin" + hosts: + - host: "http://httpbin.org" + weight: 100 + + - name: "backend1" + config: + timeout: 60 + loadbalance: "roundrobin" + hosts: + - host: "http://httpbin.org" + weight: 100 +``` +> 📢 注意：如果之前接入过的，建议将 sepc_version改成 2，并将原先 `stage:{}`改成 `stages: []` + + +整体的样例： + +```yaml +# definition.yaml 配置文件版本号，必填，固定值 1/2 +spec_version: 2 # 如果之前接入过的，建议将 sepc_version改成 2，并将原先 stage:改成 stages: [] # 定义发布内容，用于命令 `create_version_and_release_apigw` release: @@ -97,52 +151,35 @@ apigateway: is_public: true # 标记网关为官方网关，网关名需以 `bk-` 开头，可选；非官方网关，可去除此配置 api_type: 1 - # 应用请求网关时，是否允许从请求参数 (querystring, body) 中获取蓝鲸认证信息，默认值为 true； - # 如果为 false，则只能从请求头 X-Bkapi-Authorization 获取蓝鲸认证信息； - # 新接入的网关，可以设置为 false，已接入的网关，待推动所有调用者将认证信息放到请求头后，可设置为 false - allow_auth_from_params: false - # 网关请求后端时，是否删除请求参数 (querystring, body) 中的蓝鲸认证敏感信息，比如 bk_token，为 true 表示允许删除； - # 待请求网关的所有调用者，将认证参数放到请求头 X-Bkapi-Authorization 时，可将此值设置为 false - allow_delete_sensitive_params: false # 网关维护人员，仅维护人员有管理网关的权限 maintainers: - "admin" # 定义环境信息，用于命令 `sync_apigw_stage` -stage: - name: "prod" - description: "描述" - # 环境的英文名，蓝鲸官方网关需提供，以支持国际化 - description_en: "English description" - # 环境变量；如未使用，可去除此配置 - # vars: - # key: "value" - # 代理配置 - # proxy_http 与 backends 二选一， 推荐使用 backends 方式配置 - # 网关版本 <= 1.13.3, 只支持一个后端服务, 默认是 default - # proxy_http: - # timeout: 60 - # # 负载均衡类型 + Hosts - # upstreams: - # loadbalance: "roundrobin" - # hosts: - # # 网关调用后端服务的默认域名或IP，不包含Path，比如：http://api.example.com - # - host: "" - # weight: 100 - - # 网关版本 1.13.3之后引入 backends 配置方式,支持多后端服务 - # 注意: 资源中引用的 backend 一定要配置， 否则会导入失败,不配置则会选择 default 后端服务 - # 如果 backends 没有配置 default 且 resource 未指定 backend 则会导致版本发布校验失败 - backends: +stages: + - name: "prod" + description: "描述" + # 环境的英文名，蓝鲸官方网关需提供，以支持国际化 + description_en: "English description" + # 环境变量；如未使用，可去除此配置 + # vars: + # key: "value" + # 代理配置 + # proxy_http 与 backends 二选一， 推荐使用 backends 方式配置 + # 网关版本 <= 1.13.3, 只支持一个后端服务, 默认是 default + # 网关版本 1.13.3之后引入 backends 配置方式,支持多后端服务 + # 注意: 资源中引用的 backend 一定要配置， 否则会导入失败,不配置则会选 择 default 后端服务 + # 如果 backends 没有配置 default 且 resource 未指定 backend 则会导致版本发布校验失败 + backends: - name: "default" config: - timeout: 60 - loadbalance: "roundrobin" - hosts: - # 网关调用后端服务的默认域名或IP，不包含Path，比如：http://api.example.com - - host: "" - weight: 100 - + timeout: 60 + loadbalance: "roundrobin" + hosts: + # 网关调用后端服务的默认域名或IP，不包含Path，比如： http://api.example.com + - host: "" + weight: 100 + - name: "service1" config: timeout: 60 @@ -175,22 +212,6 @@ stage: # allow_credential: false -# 支持定义多个stage,如果定义多个，则同步脚本需要添加对应的同步命令，并指明：namespace(默认：stage) eg：stage2 -# 同步脚本 sync-apigateway.sh 需要新增以下命令: -# python manage.py sync_apigw_stage --gateway-name=${gateway_name} --file="${definition_file}" --namespace="stage2" - -#stage2: -# name: "test" -# description: "这是一个测试" -# description_en: "This is a test" -# proxy_http: -# timeout: 60 -# upstreams: -# loadbalance: "roundrobin" -# hosts: -# - host: "https://httpbin.org" -# weight: 100 - # 主动授权，网关主动给应用，添加访问网关所有资源或者具体某个资源的权限； # 用于命令 `grant_apigw_permissions` diff --git a/sdks/apigw-manager/README.rst b/sdks/apigw-manager/README.rst index 7d625c3d..a0ff248d 100644 --- a/sdks/apigw-manager/README.rst +++ b/sdks/apigw-manager/README.rst @@ -116,13 +116,6 @@ definition.yaml 中可以使用 Django 模版语法引用和渲染变量，内 is_public: true # 标记网关为官方网关，网关名需以 `bk-` 开头，可选；非官方网关，可去除此配置 api_type: 1 - # 应用请求网关时，是否允许从请求参数 (querystring, body) 中获取蓝鲸认证信息，默认值为 true； - # 如果为 false，则只能从请求头 X-Bkapi-Authorization 获取蓝鲸认证信息； - # 新接入的网关，可以设置为 false，已接入的网关，待推动所有调用者将认证信息放到请求头后，可设置为 false - allow_auth_from_params: false - # 网关请求后端时，是否删除请求参数 (querystring, body) 中的蓝鲸认证敏感信息，比如 bk_token，为 true 表示允许删除； - # 待请求网关的所有调用者，将认证参数放到请求头 X-Bkapi-Authorization 时，可将此值设置为 false - allow_delete_sensitive_params: false # 网关维护人员，仅维护人员有管理网关的权限 maintainers: - "admin" diff --git a/sdks/apigw-manager/examples/chart/use-configmap/files/support-files/definition.yaml b/sdks/apigw-manager/examples/chart/use-configmap/files/support-files/definition.yaml index 861ee689..ac5dc984 100644 --- a/sdks/apigw-manager/examples/chart/use-configmap/files/support-files/definition.yaml +++ b/sdks/apigw-manager/examples/chart/use-configmap/files/support-files/definition.yaml @@ -1,4 +1,4 @@ -spec_version: 1 +spec_version: 2 release: # 发布版本号 @@ -13,56 +13,41 @@ apigateway: maintainers: - "admin" -stage: - name: "prod" - description: "这是一个测试" - description_en: "This is a test" - proxy_http: - timeout: 60 - upstreams: - loadbalance: "roundrobin" - hosts: - - host: "https://httpbin.org" - weight: 100 - # 环境插件配置 - # plugin_configs: - # - type: bk-rate-limit - # yaml: |- - # rates: - # __default: - # - period: 1 - # tokens: 100 - # - type: bk-header-rewrite - # yaml: |- - # set: - # - key: test - # value: '2' - # remove: [] - # - type: bk-cors - # yaml: |- - # allow_origins: '*' - # allow_methods: '*' - # allow_headers: '*' - # expose_headers: '*' - # max_age: 86400 - # allow_credential: false - - -# 支持定义多个stage,如果定义多个，则同步脚本需要添加对应的同步命令，并指明：namespace(默认：stage) eg：stage2 -# 同步脚本 sync-apigateway.sh 需要新增以下命令: -# call_definition_command_or_exit sync_apigw_stage "${definition_file}" --gateway-name=${gateway_name} --namespace="stage2" - -#stage2: -# name: "test" -# description: "这是一个测试" -# description_en: "This is a test" -# proxy_http: -# timeout: 60 -# upstreams: -# loadbalance: "roundrobin" -# hosts: -# - host: "https://httpbin.org" -# weight: 100 +stages: + - name: "prod" + description: "这是一个测试" + description_en: "This is a test" + backends: + - name: "default" + config: + timeout: 60 + loadbalance: "roundrobin" + hosts: + # 网关调用后端服务的默认域名或IP，不包含Path，比如：http://api.example.com + - host: "https://httpbin.org" + weight: 100 + # 环境插件配置 + # plugin_configs: + # - type: bk-rate-limit + # yaml: |- + # rates: + # __default: + # - period: 1 + # tokens: 100 + # - type: bk-header-rewrite + # yaml: |- + # set: + # - key: test + # value: '2' + # remove: [] + # - type: bk-cors + # yaml: |- + # allow_origins: '*' + # allow_methods: '*' + # allow_headers: '*' + # expose_headers: '*' + # max_age: 86400 + # allow_credential: false diff --git a/sdks/apigw-manager/examples/chart/use-custom-docker-image/my-apigw-manager/support-files/definition.yaml b/sdks/apigw-manager/examples/chart/use-custom-docker-image/my-apigw-manager/support-files/definition.yaml index 8ca02115..e059acd1 100644 --- a/sdks/apigw-manager/examples/chart/use-custom-docker-image/my-apigw-manager/support-files/definition.yaml +++ b/sdks/apigw-manager/examples/chart/use-custom-docker-image/my-apigw-manager/support-files/definition.yaml @@ -1,4 +1,4 @@ -spec_version: 1 +spec_version: 2 release: # 发布版本号 @@ -13,55 +13,42 @@ apigateway: maintainers: - "admin" -stage: - name: "prod" - description: "这是一个测试" - description_en: "This is a test" - proxy_http: - timeout: 60 - upstreams: - loadbalance: "roundrobin" - hosts: - - host: "https://httpbin.org" - weight: 100 - # 环境插件配置 - # plugin_configs: - # - type: bk-rate-limit - # yaml: |- - # rates: - # __default: - # - period: 1 - # tokens: 100 - # - type: bk-header-rewrite - # yaml: |- - # set: - # - key: test - # value: '2' - # remove: [] - # - type: bk-cors - # yaml: |- - # allow_origins: '*' - # allow_methods: '*' - # allow_headers: '*' - # expose_headers: '*' - # max_age: 86400 - # allow_credential: false - -# 支持定义多个stage,如果定义多个，则同步脚本需要添加对应的同步命令，并指明：namespace(默认：stage) eg：stage2 -# 同步脚本 sync-apigateway.sh 需要新增以下命令: -# call_definition_command_or_exit sync_apigw_stage "${definition_file}" --gateway-name=${gateway_name} --namespace="stage2" +stages: + - name: "prod" + description: "这是一个测试" + description_en: "This is a test" + backends: + - name: "default" + config: + timeout: 60 + loadbalance: "roundrobin" + hosts: + # 网关调用后端服务的默认域名或IP，不包含Path，比如：http://api.example.com + - host: "https://httpbin.org" + weight: 100 -#stage2: -# name: "test" -# description: "这是一个测试" -# description_en: "This is a test" -# proxy_http: -# timeout: 60 -# upstreams: -# loadbalance: "roundrobin" -# hosts: -# - host: "https://httpbin.org" -# weight: 100 + # 环境插件配置 + # plugin_configs: + # - type: bk-rate-limit + # yaml: |- + # rates: + # __default: + # - period: 1 + # tokens: 100 + # - type: bk-header-rewrite + # yaml: |- + # set: + # - key: test + # value: '2' + # remove: [] + # - type: bk-cors + # yaml: |- + # allow_origins: '*' + # allow_methods: '*' + # allow_headers: '*' + # expose_headers: '*' + # max_age: 86400 + # allow_credential: false # 资源文档 # 资源文档的目录格式样例如下，en 为英文文档，zh 为中文文档： diff --git a/sdks/apigw-manager/examples/django/use-custom-script/support-files/definition.yaml b/sdks/apigw-manager/examples/django/use-custom-script/support-files/definition.yaml index ac8d172b..444efe01 100644 --- a/sdks/apigw-manager/examples/django/use-custom-script/support-files/definition.yaml +++ b/sdks/apigw-manager/examples/django/use-custom-script/support-files/definition.yaml @@ -1,4 +1,4 @@ -spec_version: 1 +spec_version: 2 release: # 发布版本号 @@ -13,33 +13,19 @@ apigateway: maintainers: - "admin" -stage: - name: "prod" - description: "这是一个测试" - description_en: "This is a test" - proxy_http: - timeout: 60 - upstreams: - loadbalance: "roundrobin" - hosts: - - host: "https://httpbin.org" - weight: 100 - -# 支持定义多个stage,如果定义多个，则同步脚本需要添加对应的同步命令，并指明：namespace(默认：stage) eg：stage2 -# 同步脚本 sync-apigateway.sh 需要新增以下命令: -# python manage.py sync_apigw_stage --gateway-name=${gateway_name} --file="${definition_file}" --namespace="stage2" - -#stage2: -# name: "test" -# description: "这是一个测试" -# description_en: "This is a test" -# proxy_http: -# timeout: 60 -# upstreams: -# loadbalance: "roundrobin" -# hosts: -# - host: "https://httpbin.org" -# weight: 100 +stages: + - name: "prod" + description: "这是一个测试" + description_en: "This is a test" + backends: + - name: "default" + config: + timeout: 60 + loadbalance: "roundrobin" + hosts: + # 网关调用后端服务的默认域名或IP，不包含Path，比如：http://api.example.com + - host: "https://httpbin.org" + weight: 100 # 资源文档 # 资源文档的目录格式样例如下，en 为英文文档，zh 为中文文档： diff --git a/sdks/apigw-manager/pyproject.toml b/sdks/apigw-manager/pyproject.toml index 34a1cc7e..c06d6d9c 100644 --- a/sdks/apigw-manager/pyproject.toml +++ b/sdks/apigw-manager/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "apigw-manager" -version = "3.0.3" +version = "3.0.4" description = "The SDK for managing blueking gateway resource." readme = "README.md" authors = ["blueking "] @@ -26,7 +26,7 @@ pyjwt = { version = ">=1.6.4", optional = true } django-environ = { version = ">=0.8.1", optional = true } Django = { version = ">=1.11.1", optional = true } cryptography = { version = ">=3.1.1", optional = true } -packaging = { version = ">=21.0" } +packaging = { version = ">=20.4" } PyMySQL = { version = "^1.0.2", optional = true } kubernetes = { version = "*", optional = true } diff --git a/sdks/apigw-manager/src/apigw_manager/apigw/helper.py b/sdks/apigw-manager/src/apigw_manager/apigw/helper.py index a4697f53..c89eba73 100644 --- a/sdks/apigw-manager/src/apigw_manager/apigw/helper.py +++ b/sdks/apigw-manager/src/apigw_manager/apigw/helper.py @@ -27,7 +27,8 @@ class Definition: """Gateway model definitions""" - valid_spec_versions = ["1"] + valid_spec_versions = ["1", "2"] + spec_version = "1" @classmethod def load_from(cls, path, dictionary): @@ -46,7 +47,7 @@ def __init__(self, definition): loaded = yaml_load(definition) self._check_spec_version(loaded) - + self.spec_version = loaded.get("spec_version") self.loaded = loaded def _check_spec_version(self, definition): @@ -55,7 +56,7 @@ def _check_spec_version(self, definition): return if str(spec_version) not in self.valid_spec_versions: - raise ValueError("spec_version configured in definition.yaml is wrong, choices: 1") + raise ValueError("spec_version configured in definition.yaml is wrong, choices: [1,2]") def _get_namespace_list(self, namespace): if not namespace: @@ -168,6 +169,7 @@ def increase(self, gateway_name): class ResourceSignatureManager(ContextManager): scope = "resource_signature" + # is_dirty 表示对环境资源进行了改动，但是还没有发布的状态，可能有两种更新的方式： # 1. 同步时，发现当前资源签名和上次不一致 # 2. 其他明确需要发布的场景，比如同步接口时，发现有资源的增删 diff --git a/sdks/apigw-manager/src/apigw_manager/apigw/management/commands/sync_apigw_stage.py b/sdks/apigw-manager/src/apigw_manager/apigw/management/commands/sync_apigw_stage.py index 1d02407d..80424117 100644 --- a/sdks/apigw-manager/src/apigw_manager/apigw/management/commands/sync_apigw_stage.py +++ b/sdks/apigw-manager/src/apigw_manager/apigw/management/commands/sync_apigw_stage.py @@ -16,6 +16,20 @@ class Command(SyncCommand): default_namespace = "stage" + def get_definition(self, define, file, namespace, **kwargs): + definition = self.load_definition(define, file, **kwargs) + if definition is None: + return {} + if definition.spec_version == 2: + namespace = "stages" + self.default_namespace = "stages" + return super().get_definition(define, file, namespace, **kwargs) + def do(self, manager, definition, *args, **kwargs): - result = manager.sync_stage_config(**definition) - print("API gateway stage synchronization completed, id %s, name %s" % (result["id"], result["name"])) + if self.default_namespace == "stages": + for stage_definition in definition: + result = manager.sync_stage_config(**stage_definition) + print("API gateway stage synchronization completed, id %s, name %s" % (result["id"], result["name"])) + else: + result = manager.sync_stage_config(**definition) + print("API gateway stage synchronization completed, id %s, name %s" % (result["id"], result["name"]))