From a2ec22af2f05129862cac759fdfad89d6cbfa1fc Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 28 Aug 2024 16:23:04 +0200 Subject: [PATCH 1/5] Extend bucket notification timeout and map to the test timeout Issue: ZENKO-4806 --- tests/ctst/steps/notifications.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/ctst/steps/notifications.ts b/tests/ctst/steps/notifications.ts index 54cad9e168..0fb75e90d9 100644 --- a/tests/ctst/steps/notifications.ts +++ b/tests/ctst/steps/notifications.ts @@ -1,11 +1,11 @@ import { Then, Given, When, After } from '@cucumber/cucumber'; import { strict as assert } from 'assert'; -import { S3, Utils, KafkaHelper, AWSVersionObject, NotificationDestination } from 'cli-testing'; +import { S3, Utils, KafkaHelper, AWSVersionObject, NotificationDestination, Constants } from 'cli-testing'; import { Message } from 'node-rdkafka'; import { cleanS3Bucket } from 'common/common'; import Zenko from 'world/Zenko'; -const KAFKA_TESTS_TIMEOUT = Number(process.env.KAFKA_TESTS_TIMEOUT) || 60000; +const KAFKA_TESTS_TIMEOUT = Number(process.env.KAFKA_TESTS_TIMEOUT) || Constants.DEFAULT_TIMEOUT * 1.5; const allNotificationTypes = [ 's3:ObjectCreated:Put', @@ -308,6 +308,7 @@ Then('notifications should be enabled for {string} event in destination {int}', }); Then('i should {string} a notification for {string} event in destination {int}', + { timeout: Constants.DEFAULT_TIMEOUT * 2 }, async function (this: Zenko, receive: string, notificationType: string, destination: number) { const receivedNotification = await KafkaHelper.consumeTopicUntilCondition( From 1b2f5f3d14d1b98f52aec84dc1d220645e21103b Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 3 Sep 2024 11:25:52 +0200 Subject: [PATCH 2/5] Ensure the KafkaCleaner test completes An issue where the kafkacleaner never completed was observed in the CI; as a result, the build took 6h before being cancelled. THis fix ensures that we tolerate up to 6min. Note that the current value is 30s * (1000 + 5000) so 3 minutes. Issue: ZENKO-4806 --- tests/ctst/common/common.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 5c991ab239..d2f0690bc2 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -237,14 +237,15 @@ Then('i {string} be able to add user metadata to object {string}', } }); -Then('kafka consumed messages should not take too much place on disk', { timeout: -1 }, +const kafkaCleanerTimeout = 1800000; // 30min +Then('kafka consumed messages should not take too much place on disk', { timeout: kafkaCleanerTimeout }, async function (this: Zenko) { const kfkcIntervalSeconds = parseInt(this.parameters.KafkaCleanerInterval); const checkInterval = kfkcIntervalSeconds * (1000 + 5000); const timeoutID = setTimeout(() => { assert.fail('Kafka cleaner did not clean the topics within the expected time'); - }, checkInterval * 10); // Timeout after 10 Kafka cleaner intervals + }, Math.min(kafkaCleanerTimeout, checkInterval * 10)); // Timeout after 10 Kafka cleaner intervals try { const ignoredTopics = ['dead-letter']; @@ -307,6 +308,9 @@ Then('kafka consumed messages should not take too much place on disk', { timeout // If a topic remains in this array, it means it has not been cleaned assert(topics.length === 0, `Topics ${topics.join(', ')} still have not been cleaned`); + } catch(err: any) { + this.logger.error('Error while checking Kafka cleaner', { err }); + assert.fail(err); } finally { clearTimeout(timeoutID); } From 0257dca5323f5c37ff903a4f59ca792ce56675eb Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 3 Sep 2024 11:28:23 +0200 Subject: [PATCH 3/5] Extend the duration checks for the DMF volume In some cases, the platform will take some time to process the event, so the test will fail, although there is no error. This change also add a sleep between two checks, to reduce the impact on the CPU when multiple tests are running this step in parallel. Issue: ZENKO-4806 --- tests/ctst/steps/dmf.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/ctst/steps/dmf.ts b/tests/ctst/steps/dmf.ts index 535e26da04..759f382870 100644 --- a/tests/ctst/steps/dmf.ts +++ b/tests/ctst/steps/dmf.ts @@ -1,12 +1,13 @@ import { Then, After } from '@cucumber/cucumber'; import assert from 'assert'; +import { Utils } from 'cli-testing'; import { execShellCommand } from 'common/utils'; async function cleanDmfVolume() { await execShellCommand('rm -rf /cold-data/*'); } -Then('dmf volume should contain {int} objects', async (objectCount: number) => { +Then('dmf volume should contain {int} objects', { timeout: 5 * 60 * 1000 }, async (objectCount: number) => { let conditionOk = false; while (!conditionOk) { // Getting the number of objects inside the volume used @@ -14,6 +15,9 @@ Then('dmf volume should contain {int} objects', async (objectCount: number) => { const outStr = await execShellCommand('find /cold-data -type f | wc -l'); // we store two files per object (content and manifest.json) conditionOk = Number(outStr) === objectCount * 2; + if (!conditionOk) { + await Utils.sleep(500); + } } assert(conditionOk); }); From e22a9deac7783aa18200ac9d97c7721fc1ec69fd Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 3 Sep 2024 11:54:11 +0200 Subject: [PATCH 4/5] Lint code Issue: ZENKO-4806 --- tests/ctst/common/common.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index d2f0690bc2..4feadb8dea 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -308,9 +308,9 @@ Then('kafka consumed messages should not take too much place on disk', { timeout // If a topic remains in this array, it means it has not been cleaned assert(topics.length === 0, `Topics ${topics.join(', ')} still have not been cleaned`); - } catch(err: any) { + } catch (err: unknown) { this.logger.error('Error while checking Kafka cleaner', { err }); - assert.fail(err); + assert.fail(err as Error); } finally { clearTimeout(timeoutID); } From 0125893e807c72653c7605fa4f0f67178692324f Mon Sep 17 00:00:00 2001 From: williamlardier Date: Tue, 3 Sep 2024 13:39:19 +0200 Subject: [PATCH 5/5] Increase frequency and checks of kafkacleaner Issue: ZENKO-4806 --- tests/ctst/common/common.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 4feadb8dea..f6d33739a1 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -237,15 +237,15 @@ Then('i {string} be able to add user metadata to object {string}', } }); -const kafkaCleanerTimeout = 1800000; // 30min +const kafkaCleanerTimeout = 900000; // 15min Then('kafka consumed messages should not take too much place on disk', { timeout: kafkaCleanerTimeout }, async function (this: Zenko) { const kfkcIntervalSeconds = parseInt(this.parameters.KafkaCleanerInterval); - const checkInterval = kfkcIntervalSeconds * (1000 + 5000); + const checkInterval = kfkcIntervalSeconds * 1000 + 5000; const timeoutID = setTimeout(() => { assert.fail('Kafka cleaner did not clean the topics within the expected time'); - }, Math.min(kafkaCleanerTimeout, checkInterval * 10)); // Timeout after 10 Kafka cleaner intervals + }, Math.min(kafkaCleanerTimeout, checkInterval * 25)); // Timeout after 25 Kafka cleaner intervals try { const ignoredTopics = ['dead-letter'];