From faa6a209a926d6510d3f3a481e6d01014270483c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20Papou=C5=A1ek?= Date: Fri, 27 Oct 2023 13:05:24 +0200 Subject: [PATCH] Replace mocked S3 with S3 in docker-compose --- docker-compose.yml | 2 + features/SHA_Extractor/sha_extractor.feature | 8 +- features/steps/insights_sha_extractor.py | 17 +--- features/steps/insights_sha_extractor_s3.py | 97 ++++++++++++++++++++ insights_sha_extractor_test.sh | 4 +- 5 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 features/steps/insights_sha_extractor_s3.py diff --git a/docker-compose.yml b/docker-compose.yml index 4a2a707c..1b562a5a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: "3.9" services: # "pod" with the tests bddtests: + build: . image: quay.io/cloudservices/insights-behavioral-spec:latest entrypoint: - /bin/sh @@ -37,6 +38,7 @@ services: minio: profiles: - test-exporter + - test-sha-extractor image: minio/minio command: - server diff --git a/features/SHA_Extractor/sha_extractor.feature b/features/SHA_Extractor/sha_extractor.feature index c43d97aa..1236f25d 100644 --- a/features/SHA_Extractor/sha_extractor.feature +++ b/features/SHA_Extractor/sha_extractor.feature @@ -19,7 +19,7 @@ Feature: SHA Extractor Scenario: Check if SHA extractor is able to consume messages from Kafka - When an archive without workload info is announced in "incoming_topic" topic + When S3 and Kafka are populated with an archive without workload_info And SHA extractor service is started in group "check" Then SHA extractor should consume message about this event And this message should contain following attributes @@ -35,7 +35,7 @@ Feature: SHA Extractor Scenario: Check if SHA extractor is able to consume messages from Kafka and then download tarball Given SHA extractor service is started - When an archive without workload info is announced in "incoming_topic" topic + When S3 and Kafka are populated with an archive without workload_info Then SHA extractor should consume message about this event And this message should contain following attributes | Attribute | Description | Type | @@ -51,7 +51,7 @@ Feature: SHA Extractor Scenario: Check if SHA extractor is able to consume messages from Kafka, download tarball, and take SHA images Given SHA extractor service is started - When an archive without workload info is announced in "incoming_topic" topic + When S3 and Kafka are populated with an archive without workload_info Then SHA extractor should consume message about this event And this message should contain following attributes | Attribute | Description | Type | @@ -69,7 +69,7 @@ Feature: SHA Extractor Scenario: Check if SHA extractor is able to finish the processing of SHA images Given SHA extractor service is started - When an archive with workload info is announced in "incoming_topic" topic + When S3 and Kafka are populated with an archive with workload_info Then SHA extractor should consume message about this event And this message should contain following attributes | Attribute | Description | Type | diff --git a/features/steps/insights_sha_extractor.py b/features/steps/insights_sha_extractor.py index a0de4906..24b6bce3 100644 --- a/features/steps/insights_sha_extractor.py +++ b/features/steps/insights_sha_extractor.py @@ -87,21 +87,6 @@ def start_sha_extractor(context, group_id=None): context.sha_extractor = sha_extractor -@when('an archive {with_or_without} workload info is announced in "{topic_var}" topic') -def produce_event(context, with_or_without, topic_var): - """Produce an event into specified topic.""" - topic_name = context.__dict__["_stack"][0][topic_var] - if with_or_without == "with": - msg_path = "test_data/upload.json" - else: - msg_path = "test_data/upload_no_workloadinfo.json" - - with open(msg_path, "r") as f: - event_data = f.read().encode("utf-8") - headers = [("service", b"testareno")] - kafka_util.send_event(context.hostname, topic_name, event_data, headers) - - @when('the file "config/workload_info.json" is not found') def check_workload_info_not_present(context): """Step when workload_info.json is not in the archive.""" @@ -184,7 +169,7 @@ def check_url(context): @then("SHA extractor should download tarball from given URL attribute") def check_start_download(context): """Check that sha extractor is able to start download.""" - expected_msg = "Downloading http://localhost:8000/" + expected_msg = "Downloading" assert message_in_buffer( expected_msg, context.sha_extractor.stdout ), "download not started" diff --git a/features/steps/insights_sha_extractor_s3.py b/features/steps/insights_sha_extractor_s3.py new file mode 100644 index 00000000..3c5ae0dc --- /dev/null +++ b/features/steps/insights_sha_extractor_s3.py @@ -0,0 +1,97 @@ +# Copyright © 2023, Jiří Papoušek, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Implementation of logic behind usage of S3 and Kafka in SHA extractor tests.""" + +import boto3 +import json +import logging +import os +from behave import when +from botocore.exceptions import ClientError +from src import kafka_util + + +def create_presigned_url(s3_client, bucket_name, object_name, expiration=3600): + """Generate a presigned URL to share an S3 object.""" + try: + response = s3_client.generate_presigned_url('get_object', + Params={'Bucket': bucket_name, + 'Key': object_name}, + ExpiresIn=expiration) + except ClientError as e: + logging.error(e) + return None + + # The response contains the presigned URL + return response + + +def use_real_storage(context, archive_key, msg_path): + """Load data to real S3 storage and publish the JSON message to Kafka.""" + s3_host = os.getenv("S3_HOST", default="localhost") + s3_port = os.getenv("S3_PORT", default="9000") + s3_access_key = os.getenv("S3_ACCESS_KEY") + s3_secret_access_key = os.getenv("S3_SECRET_ACCESS_KEY") + + s3_client = boto3.client('s3', + endpoint_url=f'http://{s3_host}:{s3_port}', + aws_access_key_id=s3_access_key, + aws_secret_access_key=s3_secret_access_key) + + try: + s3_client.head_bucket(Bucket='test') + except ClientError: + s3_client.create_bucket(Bucket='test') + + with open(f"test_data/{archive_key}.tar.gz", 'rb') as archive: + s3_client.put_object(Body=archive, Bucket="test", Key=archive_key) + + topic_name = context.__dict__["_stack"][0]["incoming_topic"] + presigned_url = create_presigned_url(s3_client, "test", archive_key) + + with open(msg_path, "r") as f: + msg = f.read().encode("utf-8") + event_data = json.loads(msg) + event_data["url"] = presigned_url + event_data = json.dumps(event_data).encode('utf-8') + headers = [("service", b"testareno")] + kafka_util.send_event(context.hostname, topic_name, event_data, headers) + + +def use_mock_storage(context, archive_key, msg_path): + """Publish JSON messages to Kafka with URLs for mock storage.""" + topic_name = context.__dict__["_stack"][0]["incoming_topic"] + + with open(msg_path, "r") as f: + event_data = f.read().encode("utf-8") + headers = [("service", b"testareno")] + kafka_util.send_event(context.hostname, topic_name, event_data, headers) + + +@when('S3 and Kafka are populated with an archive {with_or_without} workload_info') +def populate_s3(context, with_or_without): + """Try to load archive to real S3 storage and publish JSON message to Kafka.""" + if with_or_without == "with": + archive_key = "archive" + msg_path = "test_data/upload.json" + else: + archive_key = "archive_no_workloadinfo" + msg_path = "test_data/upload_no_workloadinfo.json" + + # use real storage and in case of failure, use mock s3 instead + try: + use_real_storage(context, archive_key, msg_path) + except Exception: + use_mock_storage(context, archive_key, msg_path) diff --git a/insights_sha_extractor_test.sh b/insights_sha_extractor_test.sh index 1e388fa3..ab9cd4c8 100755 --- a/insights_sha_extractor_test.sh +++ b/insights_sha_extractor_test.sh @@ -28,7 +28,7 @@ function prepare_venv() { if [[ ! -d $PATH_TO_LOCAL_SHA_EXTRACTOR ]] ; then git clone --depth=1 git@gitlab.cee.redhat.com:ccx/ccx-sha-extractor.git $PATH_TO_LOCAL_SHA_EXTRACTOR - add_trap "rm -rf ./ccx-sha-extractor" + add_trap "rm -rf ./ccx-sha-extractor" fi cwd=$(pwd) cd $PATH_TO_LOCAL_SHA_EXTRACTOR || exit @@ -88,9 +88,9 @@ function add_trap() { if ! [ "$ENV_DOCKER" ] ; then run_kafka + run_mock_s3 fi -run_mock_s3 prepare_venv # shellcheck disable=SC2068