From e2cd97d1ce811538ee62c3acaab01dc1e9497b60 Mon Sep 17 00:00:00 2001 From: charlie green Date: Thu, 1 Aug 2024 18:19:41 +0100 Subject: [PATCH] CTOOLS-294: add oneOf fix for workflow --- generate/fix-file-for-one-of.sh | 61 +++++++++++++++ generate/fix-files-for-one-of.sh | 22 ++++++ generate/fix-notifications-v2-sdk.sh | 84 --------------------- generate/notifications-one-of-fix-list.json | 24 ++++++ generate/templates/api_test.mustache | 22 ++++-- generate/workflow-one-of-fix-list.json | 24 ++++++ justfile | 21 ++---- 7 files changed, 156 insertions(+), 102 deletions(-) create mode 100644 generate/fix-file-for-one-of.sh create mode 100644 generate/fix-files-for-one-of.sh delete mode 100644 generate/fix-notifications-v2-sdk.sh create mode 100644 generate/notifications-one-of-fix-list.json create mode 100644 generate/workflow-one-of-fix-list.json diff --git a/generate/fix-file-for-one-of.sh b/generate/fix-file-for-one-of.sh new file mode 100644 index 0000000..5680747 --- /dev/null +++ b/generate/fix-file-for-one-of.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +# shellcheck disable=SC2089 + +set -eETuo pipefail + +failure() { + local lineno="$1" + local msg="$2" + echo "Failed at $lineno: $msg" +} +trap 'failure ${LINENO} "$BASH_COMMAND"' ERR + +file=$1 +find=$2 +replace=$3 + +# need the GNU version of sed on a mac +if [[ $(uname) == Darwin ]]; then + if gsed --version > /dev/null; then + shopt -s expand_aliases + alias sed=gsed + else + echo "GNU sed required for this script, please add it. See https://formulae.brew.sh/formula/gnu-sed" + exit 1 + fi +fi + +# check file exists +if ! [[ -f $file ]]; then + echo "expected file '$file' does not exist - unable to carry out fix for one of" + exit 1 +fi + +# 1. fix if statement +if_statement_text_to_replace="if value not in ($find):" + +# check that the expected text exists in the file +if ! grep -q "$if_statement_text_to_replace" "$file"; then + echo "did not find expected text '$if_statement_text_to_replace' in file '$file' - unable to carry out fix for one of" + exit 1 +fi +# make the replacement +if ! sed -i "s/$if_statement_text_to_replace/if not value == $replace:/" "$file"; then + echo "error updating file '$file' for one of fix" + exit 1 +fi + +# 2. fix if statement error message +error_msg_text_to_replace="raise ValueError(\"must be one of enum values ($find)\")" + +# check that the expected text exists in the file +if ! grep -q "$error_msg_text_to_replace" "$file"; then + echo "did not find expected text '$error_msg_text_to_replace' in file '$file' - unable to carry out fix for one of" + exit 1 +fi +# make the replacement +if ! sed -i "s/$error_msg_text_to_replace/raise ValueError(\"must be one of enum values ($replace)\")/" "$file"; then + echo "error updating file '$file' for one of fix" + exit 1 +fi diff --git a/generate/fix-files-for-one-of.sh b/generate/fix-files-for-one-of.sh new file mode 100644 index 0000000..fb58209 --- /dev/null +++ b/generate/fix-files-for-one-of.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eETuo pipefail + +failure() { + local lineno="$1" + local msg="$2" + echo "Failed at $lineno: $msg" +} +trap 'failure ${LINENO} "$BASH_COMMAND"' ERR + +justfile_dir=$1 +package_name=$2 +application=$3 + +while read -r item; do + # echo "item='$item'" + file="$(echo "$item" | jq -r '.file')" + find="$(echo "$item" | jq -r '.find')" + replace="$(echo "$item" | jq -r '.replace')" + bash "$justfile_dir/generate/fix-file-for-one-of.sh" "$justfile_dir/generate/.output/sdk/$package_name/models/$file.py" "$find" "$replace" +done <<< "$(jq -rc '.items[]' "$justfile_dir/generate/$application-one-of-fix-list.json")" \ No newline at end of file diff --git a/generate/fix-notifications-v2-sdk.sh b/generate/fix-notifications-v2-sdk.sh deleted file mode 100644 index b35d0f5..0000000 --- a/generate/fix-notifications-v2-sdk.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# shellcheck disable=SC2089 - -set -euo pipefail - -justfile_dir=$1 -package_name=$2 - -# need the GNU version of sed on a mac -if [[ $(uname) == Darwin ]]; then - if gsed --version > /dev/null; then - shopt -s expand_aliases - alias sed=gsed - else - echo "GNU sed required for this script, please add it. See https://formulae.brew.sh/formula/gnu-sed" - exit 1 - fi -fi - -# fix AmazonSqsNotificationType/Response files -amazon_sqs_notification_type_file="$justfile_dir/generate/.output/sdk/$package_name/models/amazon_sqs_notification_type.py" -amazon_sqs_notification_type_response_file="$justfile_dir/generate/.output/sdk/$package_name/models/amazon_sqs_notification_type_response.py" - -for file in $amazon_sqs_notification_type_file $amazon_sqs_notification_type_response_file; do - # check file exists - if ! [[ -f $file ]]; then - echo "expected file '$file' does not exist - unable to make notifications v2 sdk fix" - exit 1 - fi - - # 1. fix if statement - if_statement_text_to_replace="if value not in ('AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook'):" - - # check that the expected text exists in the file - if ! grep -q "$if_statement_text_to_replace" "$file"; then - echo "did not find expected text '$if_statement_text_to_replace' in file '$file' - unable to make notifications v2 sdk fix" - exit 1 - fi - # make the replacement - if ! sed -i "s/$if_statement_text_to_replace/if not value == 'AmazonSqs':/" "$file"; then - echo "error updating file '$file' for notifications v2 sdk fix" - exit 1 - fi - - # 2. fix if statement error message - error_msg_text_to_replace="raise ValueError(\"must be one of enum values ('AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook')\")" - - # check that the expected text exists in the file - if ! grep -q "$error_msg_text_to_replace" "$file"; then - echo "did not find expected text '$error_msg_text_to_replace' in file '$file' - unable to make notifications v2 sdk fix" - exit 1 - fi - # make the replacement - if ! sed -i "s/$error_msg_text_to_replace/raise ValueError(\"must be one of enum values ('AmazonSqs')\")/" "$file"; then - echo "error updating file '$file' for notifications v2 sdk fix" - exit 1 - fi -done - -# fix AmazonSqsPrincipalAuthNotificationType/Response files -amazon_sqs_principal_auth_notification_type_file="$justfile_dir/generate/.output/sdk/$package_name/models/amazon_sqs_principal_auth_notification_type.py" -amazon_sqs_principal_auth_notification_type_response_file="$justfile_dir/generate/.output/sdk/$package_name/models/amazon_sqs_principal_auth_notification_type_response.py" -for file in $amazon_sqs_principal_auth_notification_type_file $amazon_sqs_principal_auth_notification_type_response_file; do - # check file exists - if ! [[ -f $file ]]; then - echo "expected file '$file' does not exist - unable to make notifications v2 sdk fix" - exit 1 - fi - - # 1. fix if statement - if_statement_text_to_replace="if value not in ('AmazonSqsPrincipalAuth'):" - - # check that the expected text exists in the file - if ! grep -q "$if_statement_text_to_replace" "$file"; then - echo "did not find expected text '$if_statement_text_to_replace' in file '$file' - unable to make notifications v2 sdk fix" - exit 1 - fi - # make the replacement - if ! sed -i "s/$if_statement_text_to_replace/if not value == 'AmazonSqsPrincipalAuth':/" "$file"; then - echo "error updating file '$file' for notifications v2 sdk fix" - exit 1 - fi -done \ No newline at end of file diff --git a/generate/notifications-one-of-fix-list.json b/generate/notifications-one-of-fix-list.json new file mode 100644 index 0000000..4f0f23c --- /dev/null +++ b/generate/notifications-one-of-fix-list.json @@ -0,0 +1,24 @@ +{ + "items": [ + { + "file": "amazon_sqs_notification_type", + "find": "'AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook'", + "replace": "'AmazonSqs'" + }, + { + "file": "amazon_sqs_notification_type_response", + "find": "'AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook'", + "replace": "'AmazonSqs'" + }, + { + "file": "amazon_sqs_principal_auth_notification_type", + "find": "'AmazonSqsPrincipalAuth'", + "replace": "'AmazonSqsPrincipalAuth'" + }, + { + "file": "amazon_sqs_principal_auth_notification_type_response", + "find": "'AmazonSqsPrincipalAuth'", + "replace": "'AmazonSqsPrincipalAuth'" + } + ] +} \ No newline at end of file diff --git a/generate/templates/api_test.mustache b/generate/templates/api_test.mustache index 10b5b96..43d9a76 100644 --- a/generate/templates/api_test.mustache +++ b/generate/templates/api_test.mustache @@ -2,6 +2,7 @@ {{>partial_header}} +from typing import List import unittest import json from xeger import Xeger @@ -48,8 +49,8 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase): minLength = {{#schema.minLength}}{{schema.minLength}}{{/schema.minLength}}{{^schema.minLength}}0{{/schema.minLength}} maxLength = {{#schema.maxLength}}{{schema.maxLength}}{{/schema.maxLength}}{{^schema.maxLength}}10{{/schema.maxLength}} # the following is necessary to avoid url becoming longer than acceptable length - if maxLength > 20: - maxLength = 20 + if maxLength > 50: + maxLength = 50 xegerObj = Xeger(limit=maxLength) count = 0 while True: @@ -62,7 +63,16 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase): {{/vendorExtensions.x-regex}} {{^vendorExtensions.x-regex}} {{#isPrimitiveType}} + {{#isUuid}} + # this parameter must follow the uuid syntax - make sure it doesn't contain invalid characters + if "_" in {{{example}}}: + {{paramName}}: {{{dataType}}} = "71beb2cb-be3a-4662-b8ee-e5ded63b957a" + else: + {{paramName}}: {{{dataType}}} = {{{example}}} + {{/isUuid}} + {{^isUuid}} {{paramName}}: {{{dataType}}} = {{{example}}} + {{/isUuid}} {{/isPrimitiveType}} {{^isPrimitiveType}} {{#content}} @@ -83,12 +93,14 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase): {{paramName}}: {{dataType}} = example["value"] numExamples = len(examples) - # needed to correctly generate oneOf nested type - {{paramName}} = {{dataType}}.construct().from_dict({{paramName}}) + {{paramName}} = {{dataType}}.from_dict({{paramName}}) {{/content}} {{^content}} - {{paramName}}: {{{dataType}}} = json.loads("""{{{example}}}""") + {{#isArray}} + {{paramName}}: {{{dataType}}} = {{{example}}}{{/isArray}} + {{^isArray}} + {{paramName}}: {{{dataType}}} = json.loads("""{{{example}}}"""){{/isArray}} {{/content}} {{/isPrimitiveType}} {{/vendorExtensions.x-regex}} diff --git a/generate/workflow-one-of-fix-list.json b/generate/workflow-one-of-fix-list.json new file mode 100644 index 0000000..c39f817 --- /dev/null +++ b/generate/workflow-one-of-fix-list.json @@ -0,0 +1,24 @@ +{ + "items": [ + { + "file": "create_child_tasks_action", + "find": "'CreateChildTasks', 'RunWorker', 'TriggerParentTask'", + "replace": "'CreateChildTasks'" + }, + { + "file": "create_child_tasks_action_response", + "find": "'CreateChildTasks', 'RunWorker', 'TriggerParentTask'", + "replace": "'CreateChildTasks'" + }, + { + "file": "fail", + "find": "'Fail', 'HealthCheck', 'LuminesceView', 'SchedulerJob', 'Sleep'", + "replace": "'Fail'" + }, + { + "file": "fail_response", + "find": "'Fail', 'HealthCheck', 'LuminesceView', 'SchedulerJob', 'Sleep'", + "replace": "'Fail'" + } + ] +} \ No newline at end of file diff --git a/justfile b/justfile index d23d58a..a09200b 100644 --- a/justfile +++ b/justfile @@ -27,7 +27,6 @@ export GENERATE_API_TESTS := `echo ${GENERATE_API_TESTS:-false}` swagger_path := "./swagger.json" swagger_url := "https://fbn-prd.lusid.com/api/swagger/v0/swagger.json" -fix_notifications_v2_sdk_flag := "--fix-notifications-v2-sdk" get-swagger: echo {{swagger_url}} @@ -42,9 +41,6 @@ generate-templates: finbourne/lusid-sdk-gen-python:latest -- java -jar /opt/openapi-generator/modules/openapi-generator-cli/target/openapi-generator-cli.jar author template -g python -o /usr/src/templates generate-local FLAG="": - # check if the notifications fix flag has been set - if [ "{{FLAG}}" != "{{fix_notifications_v2_sdk_flag}}" ] && [ -n "{{FLAG}}" ]; then echo "unexpected flag '{{FLAG}}' ... did you mean '{{fix_notifications_v2_sdk_flag}}'?"; fi - # generate the sdk rm -r {{justfile_directory()}}/generate/.output || true # ensure a clean output dir before starting envsubst < generate/config-template.json > generate/.config.json @@ -63,8 +59,9 @@ generate-local FLAG="": # split the README into two, and move one up a level bash generate/split-readme.sh - # try to fix the notifications sdk if flag set - if [ "{{FLAG}}" = "{{fix_notifications_v2_sdk_flag}}" ]; then just fix-notifications-v2-sdk; fi + # make the necessary post-generation fixes to the sdks using the 'oneOf' openapi feature + # caused by a bug in the python generator + if [ "{{APPLICATION_NAME}}" = "notifications" ] || [ "{{APPLICATION_NAME}}" = "workflow" ]; then just make-fix-for-one-of; fi add-tests: mkdir -p {{justfile_directory()}}/generate/.output/sdk/test/ @@ -134,9 +131,6 @@ generate TARGET_DIR FLAG="": # Generate an SDK from a swagger.json and copy the output to the TARGET_DIR generate-cicd TARGET_DIR FLAG="": - # check if the notifications fix flag has been set - if [ "{{FLAG}}" != "{{fix_notifications_v2_sdk_flag}}" ] && [ -n "{{FLAG}}" ]; then echo "unexpected flag '{{FLAG}}' ... did you mean '{{fix_notifications_v2_sdk_flag}}'?"; fi - mkdir -p {{TARGET_DIR}} mkdir -p ./generate/.output envsubst < generate/config-template.json > generate/.config.json @@ -149,8 +143,9 @@ generate-cicd TARGET_DIR FLAG="": # split the README into two, and move one up a level bash generate/split-readme.sh - # try to fix the notifications sdk if flag set - if [ "{{FLAG}}" = "{{fix_notifications_v2_sdk_flag}}" ]; then just fix-notifications-v2-sdk; fi + # make the necessary post-generation fixes to the sdks using the 'oneOf' openapi feature + # caused by a bug in the python generator + if [ "{{APPLICATION_NAME}}" = "notifications" ] || [ "{{APPLICATION_NAME}}" = "workflow" ]; then just make-fix-for-one-of; fi # need to remove the created content before copying over the top of it. # this prevents deleted content from hanging around indefinitely. @@ -202,6 +197,6 @@ generate-and-publish-cicd OUT_DIR FLAG="": @just generate-cicd {{OUT_DIR}} {{FLAG}} @just publish-cicd {{OUT_DIR}} -fix-notifications-v2-sdk: - bash {{justfile_directory()}}/generate/fix-notifications-v2-sdk.sh {{justfile_directory()}} ${PACKAGE_NAME} +make-fix-for-one-of: + bash {{justfile_directory()}}/generate/fix-files-for-one-of.sh {{justfile_directory()}} ${PACKAGE_NAME} ${APPLICATION_NAME}