-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #519 from JiriPapousek/fix-sha-extractor-bdd
[CCXDEV-11895] Fix SHA extractor tests
- Loading branch information
Showing
6 changed files
with
137 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| # Copyright © 2023, Jiří Papoušek, Red Hat, Inc. | ||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| """Implementation of logic behind usage of S3 and Kafka in SHA extractor tests.""" | ||
|
|
||
| import boto3 | ||
| import json | ||
| import logging | ||
| import os | ||
| from behave import when | ||
| from botocore.exceptions import ClientError | ||
| from src import kafka_util | ||
|
|
||
|
|
||
| def create_presigned_url(s3_client, bucket_name, object_name, expiration=3600): | ||
| """Generate a presigned URL to share an S3 object.""" | ||
| try: | ||
| response = s3_client.generate_presigned_url('get_object', | ||
| Params={'Bucket': bucket_name, | ||
| 'Key': object_name}, | ||
| ExpiresIn=expiration) | ||
| except ClientError as e: | ||
| logging.error(e) | ||
| return None | ||
|
|
||
| # The response contains the presigned URL | ||
| return response | ||
|
|
||
|
|
||
| def use_real_storage(context, archive_key, msg_path): | ||
| """Load data to real S3 storage and publish the JSON message to Kafka.""" | ||
| s3_host = os.getenv("S3_HOST", default="localhost") | ||
| s3_port = os.getenv("S3_PORT", default="9000") | ||
| s3_access_key = os.getenv("S3_ACCESS_KEY") | ||
| s3_secret_access_key = os.getenv("S3_SECRET_ACCESS_KEY") | ||
|
|
||
| s3_client = boto3.client('s3', | ||
| endpoint_url=f'http://{s3_host}:{s3_port}', | ||
| aws_access_key_id=s3_access_key, | ||
| aws_secret_access_key=s3_secret_access_key) | ||
|
|
||
| try: | ||
| s3_client.head_bucket(Bucket='test') | ||
| except ClientError: | ||
| s3_client.create_bucket(Bucket='test') | ||
|
|
||
| with open(f"test_data/{archive_key}.tar.gz", 'rb') as archive: | ||
| s3_client.put_object(Body=archive, Bucket="test", Key=archive_key) | ||
|
|
||
| topic_name = context.__dict__["_stack"][0]["incoming_topic"] | ||
| presigned_url = create_presigned_url(s3_client, "test", archive_key) | ||
|
|
||
| with open(msg_path, "r") as f: | ||
| msg = f.read().encode("utf-8") | ||
| event_data = json.loads(msg) | ||
| event_data["url"] = presigned_url | ||
| event_data = json.dumps(event_data).encode('utf-8') | ||
| headers = [("service", b"testareno")] | ||
| kafka_util.send_event(context.hostname, topic_name, event_data, headers) | ||
|
|
||
|
|
||
| def use_mock_storage(context, archive_key, msg_path): | ||
| """Publish JSON messages to Kafka with URLs for mock storage.""" | ||
| topic_name = context.__dict__["_stack"][0]["incoming_topic"] | ||
|
|
||
| with open(msg_path, "r") as f: | ||
| event_data = f.read().encode("utf-8") | ||
| headers = [("service", b"testareno")] | ||
| kafka_util.send_event(context.hostname, topic_name, event_data, headers) | ||
|
|
||
|
|
||
| @when('S3 and Kafka are populated with an archive {with_or_without} workload_info') | ||
| def populate_s3(context, with_or_without): | ||
| """Try to load archive to real S3 storage and publish JSON message to Kafka.""" | ||
| if with_or_without == "with": | ||
| archive_key = "archive" | ||
| msg_path = "test_data/upload.json" | ||
| else: | ||
| archive_key = "archive_no_workloadinfo" | ||
| msg_path = "test_data/upload_no_workloadinfo.json" | ||
|
|
||
| # use real storage and in case of failure, use mock s3 instead | ||
| try: | ||
| use_real_storage(context, archive_key, msg_path) | ||
| except Exception: | ||
| use_mock_storage(context, archive_key, msg_path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,6 +14,9 @@ | |
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| export PATH_TO_LOCAL_SHA_EXTRACTOR=${PATH_TO_LOCAL_SHA_EXTRACTOR:="../ccx-sha-extractor"} | ||
| exit_trap_command="" | ||
|
|
||
| function prepare_venv() { | ||
| echo "Preparing environment" | ||
| # shellcheck disable=SC1091 | ||
|
|
@@ -23,12 +26,16 @@ function prepare_venv() { | |
| python3 "$(which pip3)" install --no-cache -r requirements.in || exit 1 | ||
| python3 "$(which pip3)" install --no-cache -r requirements/insights_sha_extractor.txt || exit 1 | ||
|
|
||
| git clone --depth=1 [email protected]:ccx/ccx-sha-extractor.git | ||
| cd ccx-sha-extractor || exit | ||
| if [[ ! -d $PATH_TO_LOCAL_SHA_EXTRACTOR ]] ; then | ||
| git clone --depth=1 [email protected]:ccx/ccx-sha-extractor.git $PATH_TO_LOCAL_SHA_EXTRACTOR | ||
| add_trap "rm -rf ./ccx-sha-extractor" | ||
| fi | ||
| cwd=$(pwd) | ||
| cd $PATH_TO_LOCAL_SHA_EXTRACTOR || exit | ||
| pip install --no-cache-dir -U pip setuptools wheel | ||
| pip install --no-cache-dir -r requirements.txt | ||
| pip install -e . | ||
| cd .. | ||
| cd "$cwd" || exit 1 | ||
|
|
||
| echo "Environment ready" | ||
| } | ||
|
|
@@ -56,16 +63,33 @@ function run_kafka() { | |
| done | ||
|
|
||
| export kafka_cid | ||
| add_trap "docker kill ${kafka_cid}" | ||
| } | ||
|
|
||
| function run_mock_s3(){ | ||
| function run_mock_s3() { | ||
| uvicorn mocks.s3.s3:app & | ||
| s3_pid=$! | ||
| add_trap "kill -9 $s3_pid" | ||
| } | ||
|
|
||
| run_kafka | ||
| function cleanup { | ||
| eval "$exit_trap_command" | ||
| } | ||
|
|
||
| run_mock_s3 | ||
| function add_trap() { | ||
| local to_add=$1 | ||
| if [ -z $exit_trap_command ] ; then | ||
| exit_trap_command="$to_add" | ||
| else | ||
| exit_trap_command="$exit_trap_command; $to_add" | ||
| fi | ||
| trap cleanup EXIT | ||
| } | ||
|
|
||
| if ! [ "$ENV_DOCKER" ] ; then | ||
| run_kafka | ||
| run_mock_s3 | ||
| fi | ||
|
|
||
| prepare_venv | ||
|
|
||
|
|
@@ -74,7 +98,3 @@ PYTHONDONTWRITEBYTECODE=1 python3 -m behave --no-capture \ | |
| --format=progress2 \ | ||
| --tags=-skip --tags=-managed \ | ||
| -D dump_errors=true @test_list/insights_sha_extractor.txt "$@" | ||
|
|
||
| docker kill "$kafka_cid" | ||
| kill -9 $s3_pid | ||
| rm -rf ./ccx-sha-extractor | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters