Skip to content

Commit

Permalink
CTOOLS-294: add oneOf fix for workflow
Browse files Browse the repository at this point in the history
  • Loading branch information
cg-finbourne committed Aug 5, 2024
1 parent 1d5ba75 commit e2cd97d
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 102 deletions.
61 changes: 61 additions & 0 deletions generate/fix-file-for-one-of.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash

# shellcheck disable=SC2089

set -eETuo pipefail

failure() {
local lineno="$1"
local msg="$2"
echo "Failed at $lineno: $msg"
}
trap 'failure ${LINENO} "$BASH_COMMAND"' ERR

file=$1
find=$2
replace=$3

# need the GNU version of sed on a mac
if [[ $(uname) == Darwin ]]; then
if gsed --version > /dev/null; then
shopt -s expand_aliases
alias sed=gsed
else
echo "GNU sed required for this script, please add it. See https://formulae.brew.sh/formula/gnu-sed"
exit 1
fi
fi

# check file exists
if ! [[ -f $file ]]; then
echo "expected file '$file' does not exist - unable to carry out fix for one of"
exit 1
fi

# 1. fix if statement
if_statement_text_to_replace="if value not in ($find):"

# check that the expected text exists in the file
if ! grep -q "$if_statement_text_to_replace" "$file"; then
echo "did not find expected text '$if_statement_text_to_replace' in file '$file' - unable to carry out fix for one of"
exit 1
fi
# make the replacement
if ! sed -i "s/$if_statement_text_to_replace/if not value == $replace:/" "$file"; then
echo "error updating file '$file' for one of fix"
exit 1
fi

# 2. fix if statement error message
error_msg_text_to_replace="raise ValueError(\"must be one of enum values ($find)\")"

# check that the expected text exists in the file
if ! grep -q "$error_msg_text_to_replace" "$file"; then
echo "did not find expected text '$error_msg_text_to_replace' in file '$file' - unable to carry out fix for one of"
exit 1
fi
# make the replacement
if ! sed -i "s/$error_msg_text_to_replace/raise ValueError(\"must be one of enum values ($replace)\")/" "$file"; then
echo "error updating file '$file' for one of fix"
exit 1
fi
22 changes: 22 additions & 0 deletions generate/fix-files-for-one-of.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash

set -eETuo pipefail

failure() {
local lineno="$1"
local msg="$2"
echo "Failed at $lineno: $msg"
}
trap 'failure ${LINENO} "$BASH_COMMAND"' ERR

justfile_dir=$1
package_name=$2
application=$3

while read -r item; do
# echo "item='$item'"
file="$(echo "$item" | jq -r '.file')"
find="$(echo "$item" | jq -r '.find')"
replace="$(echo "$item" | jq -r '.replace')"
bash "$justfile_dir/generate/fix-file-for-one-of.sh" "$justfile_dir/generate/.output/sdk/$package_name/models/$file.py" "$find" "$replace"
done <<< "$(jq -rc '.items[]' "$justfile_dir/generate/$application-one-of-fix-list.json")"
84 changes: 0 additions & 84 deletions generate/fix-notifications-v2-sdk.sh

This file was deleted.

24 changes: 24 additions & 0 deletions generate/notifications-one-of-fix-list.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"items": [
{
"file": "amazon_sqs_notification_type",
"find": "'AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook'",
"replace": "'AmazonSqs'"
},
{
"file": "amazon_sqs_notification_type_response",
"find": "'AmazonSqs', 'AmazonSqsPrincipalAuth', 'AzureServiceBus', 'Email', 'Sms', 'Webhook'",
"replace": "'AmazonSqs'"
},
{
"file": "amazon_sqs_principal_auth_notification_type",
"find": "'AmazonSqsPrincipalAuth'",
"replace": "'AmazonSqsPrincipalAuth'"
},
{
"file": "amazon_sqs_principal_auth_notification_type_response",
"find": "'AmazonSqsPrincipalAuth'",
"replace": "'AmazonSqsPrincipalAuth'"
}
]
}
22 changes: 17 additions & 5 deletions generate/templates/api_test.mustache
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

{{>partial_header}}

from typing import List
import unittest
import json
from xeger import Xeger
Expand Down Expand Up @@ -48,8 +49,8 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase):
minLength = {{#schema.minLength}}{{schema.minLength}}{{/schema.minLength}}{{^schema.minLength}}0{{/schema.minLength}}
maxLength = {{#schema.maxLength}}{{schema.maxLength}}{{/schema.maxLength}}{{^schema.maxLength}}10{{/schema.maxLength}}
# the following is necessary to avoid url becoming longer than acceptable length
if maxLength > 20:
maxLength = 20
if maxLength > 50:
maxLength = 50
xegerObj = Xeger(limit=maxLength)
count = 0
while True:
Expand All @@ -62,7 +63,16 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase):
{{/vendorExtensions.x-regex}}
{{^vendorExtensions.x-regex}}
{{#isPrimitiveType}}
{{#isUuid}}
# this parameter must follow the uuid syntax - make sure it doesn't contain invalid characters
if "_" in {{{example}}}:
{{paramName}}: {{{dataType}}} = "71beb2cb-be3a-4662-b8ee-e5ded63b957a"
else:
{{paramName}}: {{{dataType}}} = {{{example}}}
{{/isUuid}}
{{^isUuid}}
{{paramName}}: {{{dataType}}} = {{{example}}}
{{/isUuid}}
{{/isPrimitiveType}}
{{^isPrimitiveType}}
{{#content}}
Expand All @@ -83,12 +93,14 @@ class {{#operations}}Test{{classname}}(unittest.IsolatedAsyncioTestCase):
{{paramName}}: {{dataType}} = example["value"]
numExamples = len(examples)

# needed to correctly generate oneOf nested type
{{paramName}} = {{dataType}}.construct().from_dict({{paramName}})
{{paramName}} = {{dataType}}.from_dict({{paramName}})

{{/content}}
{{^content}}
{{paramName}}: {{{dataType}}} = json.loads("""{{{example}}}""")
{{#isArray}}
{{paramName}}: {{{dataType}}} = {{{example}}}{{/isArray}}
{{^isArray}}
{{paramName}}: {{{dataType}}} = json.loads("""{{{example}}}"""){{/isArray}}
{{/content}}
{{/isPrimitiveType}}
{{/vendorExtensions.x-regex}}
Expand Down
24 changes: 24 additions & 0 deletions generate/workflow-one-of-fix-list.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"items": [
{
"file": "create_child_tasks_action",
"find": "'CreateChildTasks', 'RunWorker', 'TriggerParentTask'",
"replace": "'CreateChildTasks'"
},
{
"file": "create_child_tasks_action_response",
"find": "'CreateChildTasks', 'RunWorker', 'TriggerParentTask'",
"replace": "'CreateChildTasks'"
},
{
"file": "fail",
"find": "'Fail', 'HealthCheck', 'LuminesceView', 'SchedulerJob', 'Sleep'",
"replace": "'Fail'"
},
{
"file": "fail_response",
"find": "'Fail', 'HealthCheck', 'LuminesceView', 'SchedulerJob', 'Sleep'",
"replace": "'Fail'"
}
]
}
21 changes: 8 additions & 13 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ export GENERATE_API_TESTS := `echo ${GENERATE_API_TESTS:-false}`
swagger_path := "./swagger.json"

swagger_url := "https://fbn-prd.lusid.com/api/swagger/v0/swagger.json"
fix_notifications_v2_sdk_flag := "--fix-notifications-v2-sdk"

get-swagger:
echo {{swagger_url}}
Expand All @@ -42,9 +41,6 @@ generate-templates:
finbourne/lusid-sdk-gen-python:latest -- java -jar /opt/openapi-generator/modules/openapi-generator-cli/target/openapi-generator-cli.jar author template -g python -o /usr/src/templates

generate-local FLAG="":
# check if the notifications fix flag has been set
if [ "{{FLAG}}" != "{{fix_notifications_v2_sdk_flag}}" ] && [ -n "{{FLAG}}" ]; then echo "unexpected flag '{{FLAG}}' ... did you mean '{{fix_notifications_v2_sdk_flag}}'?"; fi

# generate the sdk
rm -r {{justfile_directory()}}/generate/.output || true # ensure a clean output dir before starting
envsubst < generate/config-template.json > generate/.config.json
Expand All @@ -63,8 +59,9 @@ generate-local FLAG="":
# split the README into two, and move one up a level
bash generate/split-readme.sh

# try to fix the notifications sdk if flag set
if [ "{{FLAG}}" = "{{fix_notifications_v2_sdk_flag}}" ]; then just fix-notifications-v2-sdk; fi
# make the necessary post-generation fixes to the sdks using the 'oneOf' openapi feature
# caused by a bug in the python generator
if [ "{{APPLICATION_NAME}}" = "notifications" ] || [ "{{APPLICATION_NAME}}" = "workflow" ]; then just make-fix-for-one-of; fi

add-tests:
mkdir -p {{justfile_directory()}}/generate/.output/sdk/test/
Expand Down Expand Up @@ -134,9 +131,6 @@ generate TARGET_DIR FLAG="":

# Generate an SDK from a swagger.json and copy the output to the TARGET_DIR
generate-cicd TARGET_DIR FLAG="":
# check if the notifications fix flag has been set
if [ "{{FLAG}}" != "{{fix_notifications_v2_sdk_flag}}" ] && [ -n "{{FLAG}}" ]; then echo "unexpected flag '{{FLAG}}' ... did you mean '{{fix_notifications_v2_sdk_flag}}'?"; fi

mkdir -p {{TARGET_DIR}}
mkdir -p ./generate/.output
envsubst < generate/config-template.json > generate/.config.json
Expand All @@ -149,8 +143,9 @@ generate-cicd TARGET_DIR FLAG="":
# split the README into two, and move one up a level
bash generate/split-readme.sh

# try to fix the notifications sdk if flag set
if [ "{{FLAG}}" = "{{fix_notifications_v2_sdk_flag}}" ]; then just fix-notifications-v2-sdk; fi
# make the necessary post-generation fixes to the sdks using the 'oneOf' openapi feature
# caused by a bug in the python generator
if [ "{{APPLICATION_NAME}}" = "notifications" ] || [ "{{APPLICATION_NAME}}" = "workflow" ]; then just make-fix-for-one-of; fi

# need to remove the created content before copying over the top of it.
# this prevents deleted content from hanging around indefinitely.
Expand Down Expand Up @@ -202,6 +197,6 @@ generate-and-publish-cicd OUT_DIR FLAG="":
@just generate-cicd {{OUT_DIR}} {{FLAG}}
@just publish-cicd {{OUT_DIR}}

fix-notifications-v2-sdk:
bash {{justfile_directory()}}/generate/fix-notifications-v2-sdk.sh {{justfile_directory()}} ${PACKAGE_NAME}
make-fix-for-one-of:
bash {{justfile_directory()}}/generate/fix-files-for-one-of.sh {{justfile_directory()}} ${PACKAGE_NAME} ${APPLICATION_NAME}

0 comments on commit e2cd97d

Please sign in to comment.