From 287678e69e224daead28f37d0b6eb6de90086751 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Tue, 10 Oct 2023 08:42:38 -0700 Subject: [PATCH 01/11] Add Storage Extension test case, Changes to validator --- .../amazon/adot/testbed/LogsTests.java | 192 ++++++++++++++---- .../configurations/config-fileRotation.yaml | 19 ++ .../resources/logs/filerotationlogs/log4j.log | 1 + 3 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml create mode 100644 adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index a2056902b..370d48386 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -1,26 +1,34 @@ package software.amazon.adot.testbed; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.UUID; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.file.Path; +import java.net.URISyntaxException; +import java.time.Duration; +import java.time.Instant; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.github.rholder.retry.RetryerBuilder; import com.github.rholder.retry.StopStrategies; import com.github.rholder.retry.WaitStrategies; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; + import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.junit.jupiter.Testcontainers; @@ -28,17 +36,6 @@ import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsClient; import software.amazon.awssdk.services.cloudwatchlogs.model.GetLogEventsRequest; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileWriter; -import java.time.Duration; -import java.time.Instant; -import java.util.HashSet; -import java.util.UUID; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - import static org.assertj.core.api.Assertions.assertThat; @Testcontainers(disabledWithoutDocker = true) @@ -49,7 +46,7 @@ class LogsTests { : "public.ecr.aws/aws-observability/aws-otel-collector:latest"; private final Logger collectorLogger = LoggerFactory.getLogger("collector"); private static final String uniqueID = UUID.randomUUID().toString(); - + private final Path logDirectory = Files.createTempDirectory("filerotationlogs"); private GenericContainer collector; private GenericContainer createAndStartCollector(String configFilePath, String logFilePath, String logStreamName) throws IOException { @@ -65,16 +62,25 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin if (System.getenv("AWS_SESSION_TOKEN") != null) { envVariables.put("AWS_SESSION_TOKEN", System.getenv("AWS_SESSION_TOKEN")); } +// envVariables.put("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "true"); var collector = new GenericContainer<>(TEST_IMAGE) .withCopyFileToContainer(MountableFile.forClasspathResource(configFilePath), "/etc/collector/config.yaml") .withLogConsumer(new Slf4jLogConsumer(collectorLogger)) .waitingFor(Wait.forLogMessage(".*Everything is ready. Begin running and processing data.*", 1)) - .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.receiver.filelog,+adot.exporter.awscloudwatchlogs,+adot.extension.file_storage") - .withEnv(envVariables); + .withEnv(envVariables) + .withClasspathResourceMapping("/logs", "/logs", BindMode.READ_WRITE) + + .withPrivilegedMode(true) + .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.filelog.receiver,+adot.awscloudwatchlogs.exporter,+adot.file_storage.extension"); //Mount the log file for the file log receiver to parse - collector.withCopyFileToContainer(MountableFile.forClasspathResource(logFilePath), logFilePath ); + collector.withCopyFileToContainer(MountableFile.forClasspathResource(logFilePath), logFilePath); + collector.withFileSystemBind(logDirectory.toString(),"/filerotation", BindMode.READ_WRITE); +// collector.withFileSystemBind(tempFile.getAbsolutePath(), "/filerotation", BindMode.READ_WRITE); +// collector.withFileSystemBind(tempFileB.getAbsolutePath(), "/logs/filerotationlogs/logsB.log", BindMode.READ_WRITE); +// collector.withCopyFileToContainer(MountableFile.forClasspathResource("/logs/"), "/logs/", ); +// collector.withCommand("sh", "-c", "chmod -R a+rw " + "/logs"); collector.start(); collector.waitingFor(Wait.forHealthcheck()); @@ -86,7 +92,9 @@ void testSyslog() throws Exception { String logStreamName = "rfcsyslog-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", "/logs/RFC5424.log", logStreamName); - validateLogs(logStreamName , "/logs/RFC5424.log"); + List logFilePaths = new ArrayList<>(); + logFilePaths.add("/logs/testingJSON.log"); + validateLogs(logStreamName , logFilePaths, true); collector.stop(); } @@ -95,7 +103,9 @@ void testLog4j() throws Exception { String logStreamName = "log4j-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-log4j.yaml", "/logs/log4j.log", logStreamName); - validateLogs(logStreamName , "/logs/log4j.log"); + List logFilePaths = new ArrayList<>(); + logFilePaths.add("/logs/testingJSON.log"); + validateLogs(logStreamName , logFilePaths, true); collector.stop(); } @@ -104,22 +114,131 @@ void testJson() throws Exception { String logStreamName = "json-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-json.yaml", "/logs/testingJSON.log", logStreamName); - validateLogs(logStreamName , "/logs/testingJSON.log"); + List logFilePaths = new ArrayList<>(); + logFilePaths.add("/logs/testingJSON.log"); + validateLogs(logStreamName , logFilePaths, true); + collector.stop(); + } + + @Test + void testCollectorRestartAfterCrash() throws Exception { + String logStreamName = "storageExtension-logstream-" + uniqueID; + collector = createAndStartCollector("/configurations/config-storageExtension.yaml", "/logs/storageExtension.log", logStreamName); + String resourceFilePath = "/logs/storageExtension.log"; // Path to the resource file + try { + Thread.sleep(10000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + try (OutputStream outputStream = new FileOutputStream(new File(getClass().getResource(resourceFilePath).toURI()), true); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { + // Append the lines to the file + writer.write("[otel.javaagent 2023-09-28 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- First Entry"); + writer.newLine(); + writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Second Entry"); + writer.newLine(); + writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Third Entry"); + writer.newLine(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException("Error writing to the file: " + resourceFilePath, e); + } + + List logFilePaths = new ArrayList<>(); + logFilePaths.add(resourceFilePath); + validateLogs(logStreamName , logFilePaths, true); + collector.stop(); + try { + Thread.sleep(10000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + try (OutputStream outputStream = new FileOutputStream(new File(getClass().getResource(resourceFilePath).toURI()), true); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { + // Append the lines to the file + writer.write("[otel.javaagent 2023-09-28 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fourth Entry"); + writer.newLine(); + writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fifth Entry"); + writer.newLine(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException("Error writing to the file: " + resourceFilePath, e); + } + + collector.start(); + try { + Thread.sleep(10000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + validateLogs(logStreamName , logFilePaths, true); + collector.stop(); + } + + @Test + void testFileRotation() throws Exception { + String logStreamName = "fileRotation-logstream-" + uniqueID; + String resourceFilePath = "/logs/filerotationlogs/log4j.log"; // Path to the resource file + collector = createAndStartCollector("/configurations/config-fileRotation.yaml", resourceFilePath, logStreamName); + + Thread.sleep(10000); + + // Create and write data to File A + File tempFile = new File(logDirectory.toString(), "testlogA.log"); + try (FileWriter fileWriter = new FileWriter(tempFile)) { + fileWriter.write("Message in File A" + "\n"); + fileWriter.flush(); + fileWriter.close(); + } catch (IOException e) { + throw new RuntimeException("Error writing to File A.", e); + } + Thread.sleep(10000); + + File renameFile = new File(logDirectory.toString(), "testlogA-1234.log"); + tempFile.renameTo(renameFile); + + Thread.sleep(10000); + + File tempFileB = new File(logDirectory.toString(), "testlogA.log"); + try (FileWriter newfileWriter = new FileWriter(tempFileB)) { + newfileWriter.write("Message in renamed file - line 1" + "\n"); + newfileWriter.write("Message in renamed file - line 2" + "\n"); + newfileWriter.write("Message in renamed file - line 3" + "\n"); + newfileWriter.flush(); + newfileWriter.close(); + } catch (IOException e) { + throw new RuntimeException("Error writing to File B.", e); + } + Thread.sleep(10000); + + List logFilePaths = new ArrayList<>(); + String expectedLogPath = logDirectory.toString(); + logFilePaths.add(expectedLogPath + "/testlogA-1234.log"); + logFilePaths.add(expectedLogPath + "/testlogA.log"); + + validateLogs(logStreamName, logFilePaths, false); + collector.stop(); } - void validateLogs(String testLogStreamName, String logFilePath) throws Exception { - var file = new File(logFilePath); + + void validateLogs(String testLogStreamName, List logFilePaths, boolean areResourceFiles) throws Exception { var lines = new HashSet(); - try (InputStream inputStream = getClass().getResourceAsStream(logFilePath); - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = reader.readLine()) != null) { - lines.add(line); + for (String logFilePath : logFilePaths) { + InputStream inputStream; + if (areResourceFiles) { + inputStream = getClass().getResourceAsStream(logFilePath); + } else { + inputStream = new FileInputStream(logFilePath); + } + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String line; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } catch (IOException e) { + throw new RuntimeException("Error reading from the file: " + logFilePath, e); } - } catch (IOException e) { - throw new RuntimeException("Error reading from the file: " + logFilePath, e); } var cwClient = CloudWatchLogsClient.builder() @@ -160,6 +279,11 @@ void validateLogs(String testLogStreamName, String logFilePath) throws Exception .filter(Objects::nonNull) .collect(Collectors.toSet()); + System.out.println("Log lines From Cloudwatch"); + messageToValidate.forEach(System.out::println); + + System.out.println("Expected logs"); + lines.forEach(System.out::println); //Validate body field in JSON-messageToValidate with actual log line from the log file. assertThat(messageToValidate.containsAll(lines)).isTrue(); assertThat(messageToValidate).containsExactlyInAnyOrderElementsOf(lines); diff --git a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml new file mode 100644 index 000000000..66c431d30 --- /dev/null +++ b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml @@ -0,0 +1,19 @@ +receivers: + filelog: + include: [/filerotation/testlogA.log] + start_at: beginning + +exporters: + logging: + verbosity: detailed + awscloudwatchlogs: + log_group_name: "adot-testbed/logs-component-testing/logs" + log_stream_name: ${LOG_STREAM_NAME} + log_retention: 7 + raw_log: true + +service: + pipelines: + logs: + receivers: [filelog] + exporters: [logging,awscloudwatchlogs] diff --git a/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log b/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log new file mode 100644 index 000000000..baf4bf99d --- /dev/null +++ b/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log @@ -0,0 +1 @@ +[otel.javaagent 2023-09-25 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : File rotation First Line From 4e648931dea896d737f2f003663ca72f64a0b528 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Tue, 10 Oct 2023 16:27:09 -0700 Subject: [PATCH 02/11] Rebased the branch --- .../amazon/adot/testbed/LogsTests.java | 14 +++++++++---- .../configurations/config-fileRotation.yaml | 3 +-- .../config-storageExtension.yaml | 21 +++++++++++++++++++ .../resources/logs/filerotationlogs/log4j.log | 1 - .../test/resources/logs/storageExtension.log | 0 5 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml delete mode 100644 adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log create mode 100644 adot-testbed/app/src/test/resources/logs/storageExtension.log diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 370d48386..07f5ef11a 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -3,12 +3,16 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; +import java.io.FileWriter; +import java.io.FileInputStream; +import java.io.InputStreamReader; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.file.Path; +import java.nio.file.Files; import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; @@ -83,17 +87,19 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin // collector.withCommand("sh", "-c", "chmod -R a+rw " + "/logs"); collector.start(); - collector.waitingFor(Wait.forHealthcheck()); +// collector.waitingFor(Wait.forHealthcheck()); return collector; } + LogsTests() throws Exception { + } @Test void testSyslog() throws Exception { String logStreamName = "rfcsyslog-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", "/logs/RFC5424.log", logStreamName); List logFilePaths = new ArrayList<>(); - logFilePaths.add("/logs/testingJSON.log"); + logFilePaths.add("/logs/RFC5424.log"); validateLogs(logStreamName , logFilePaths, true); collector.stop(); } @@ -104,7 +110,7 @@ void testLog4j() throws Exception { collector = createAndStartCollector("/configurations/config-log4j.yaml", "/logs/log4j.log", logStreamName); List logFilePaths = new ArrayList<>(); - logFilePaths.add("/logs/testingJSON.log"); + logFilePaths.add("/logs/log4j.log"); validateLogs(logStreamName , logFilePaths, true); collector.stop(); } @@ -176,7 +182,7 @@ void testCollectorRestartAfterCrash() throws Exception { @Test void testFileRotation() throws Exception { String logStreamName = "fileRotation-logstream-" + uniqueID; - String resourceFilePath = "/logs/filerotationlogs/log4j.log"; // Path to the resource file + String resourceFilePath = "/logs/log4j.log"; // Path to the resource file collector = createAndStartCollector("/configurations/config-fileRotation.yaml", resourceFilePath, logStreamName); Thread.sleep(10000); diff --git a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml index 66c431d30..16078c216 100644 --- a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml +++ b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml @@ -10,10 +10,9 @@ exporters: log_group_name: "adot-testbed/logs-component-testing/logs" log_stream_name: ${LOG_STREAM_NAME} log_retention: 7 - raw_log: true service: pipelines: logs: receivers: [filelog] - exporters: [logging,awscloudwatchlogs] + exporters: [awscloudwatchlogs] diff --git a/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml b/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml new file mode 100644 index 000000000..9ff81c582 --- /dev/null +++ b/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml @@ -0,0 +1,21 @@ +extensions: + file_storage: + directory: /logs/ + +receivers: + filelog: + include: [/logs/storageExtension.log] + storage: file_storage + +exporters: + awscloudwatchlogs: + log_group_name: "adot-testbed/logs-component-testing/logs" + log_stream_name: ${LOG_STREAM_NAME} + log_retention: 7 + +service: + extensions: [ file_storage ] + pipelines: + logs: + receivers: [filelog] + exporters: [awscloudwatchlogs] diff --git a/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log b/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log deleted file mode 100644 index baf4bf99d..000000000 --- a/adot-testbed/app/src/test/resources/logs/filerotationlogs/log4j.log +++ /dev/null @@ -1 +0,0 @@ -[otel.javaagent 2023-09-25 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : File rotation First Line diff --git a/adot-testbed/app/src/test/resources/logs/storageExtension.log b/adot-testbed/app/src/test/resources/logs/storageExtension.log new file mode 100644 index 000000000..e69de29bb From f5e5d3fe7ff7f19c43e24f3e9e8234396152a1d6 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Wed, 11 Oct 2023 11:45:23 -0700 Subject: [PATCH 03/11] Modified Storage extension test case, modifed thread wait time --- .../amazon/adot/testbed/LogsTests.java | 120 ++++++++---------- .../configurations/config-fileRotation.yaml | 2 +- .../config-storageExtension.yaml | 2 +- .../test/resources/logs/storageExtension.log | 0 4 files changed, 58 insertions(+), 66 deletions(-) delete mode 100644 adot-testbed/app/src/test/resources/logs/storageExtension.log diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 07f5ef11a..dba860f27 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -16,7 +16,14 @@ import java.net.URISyntaxException; import java.time.Duration; import java.time.Instant; -import java.util.*; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.List; +import java.util.ArrayList; +import java.util.Objects; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -50,10 +57,10 @@ class LogsTests { : "public.ecr.aws/aws-observability/aws-otel-collector:latest"; private final Logger collectorLogger = LoggerFactory.getLogger("collector"); private static final String uniqueID = UUID.randomUUID().toString(); - private final Path logDirectory = Files.createTempDirectory("filerotationlogs"); + private final Path logDirectory = Files.createTempDirectory("tempLogs"); private GenericContainer collector; - private GenericContainer createAndStartCollector(String configFilePath, String logFilePath, String logStreamName) throws IOException { + private GenericContainer createAndStartCollector(String configFilePath, String logStreamName) throws IOException { // Create an environment variable map Map envVariables = new HashMap<>(); @@ -75,19 +82,13 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin .withEnv(envVariables) .withClasspathResourceMapping("/logs", "/logs", BindMode.READ_WRITE) - .withPrivilegedMode(true) +// .withPrivilegedMode(true) .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.filelog.receiver,+adot.awscloudwatchlogs.exporter,+adot.file_storage.extension"); - //Mount the log file for the file log receiver to parse - collector.withCopyFileToContainer(MountableFile.forClasspathResource(logFilePath), logFilePath); - collector.withFileSystemBind(logDirectory.toString(),"/filerotation", BindMode.READ_WRITE); -// collector.withFileSystemBind(tempFile.getAbsolutePath(), "/filerotation", BindMode.READ_WRITE); -// collector.withFileSystemBind(tempFileB.getAbsolutePath(), "/logs/filerotationlogs/logsB.log", BindMode.READ_WRITE); -// collector.withCopyFileToContainer(MountableFile.forClasspathResource("/logs/"), "/logs/", ); -// collector.withCommand("sh", "-c", "chmod -R a+rw " + "/logs"); + //Mount the Temp directory + collector.withFileSystemBind(logDirectory.toString(),"/tempLogs", BindMode.READ_WRITE); collector.start(); -// collector.waitingFor(Wait.forHealthcheck()); return collector; } @@ -96,7 +97,7 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin @Test void testSyslog() throws Exception { String logStreamName = "rfcsyslog-logstream-" + uniqueID; - collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", "/logs/RFC5424.log", logStreamName); + collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", logStreamName); List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/RFC5424.log"); @@ -107,7 +108,7 @@ void testSyslog() throws Exception { @Test void testLog4j() throws Exception { String logStreamName = "log4j-logstream-" + uniqueID; - collector = createAndStartCollector("/configurations/config-log4j.yaml", "/logs/log4j.log", logStreamName); + collector = createAndStartCollector("/configurations/config-log4j.yaml", logStreamName); List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/log4j.log"); @@ -118,7 +119,7 @@ void testLog4j() throws Exception { @Test void testJson() throws Exception { String logStreamName = "json-logstream-" + uniqueID; - collector = createAndStartCollector("/configurations/config-json.yaml", "/logs/testingJSON.log", logStreamName); + collector = createAndStartCollector("/configurations/config-json.yaml", logStreamName); List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/testingJSON.log"); @@ -127,65 +128,55 @@ void testJson() throws Exception { } @Test - void testCollectorRestartAfterCrash() throws Exception { + void testCollectorRestartStorageExtension() throws Exception { String logStreamName = "storageExtension-logstream-" + uniqueID; - collector = createAndStartCollector("/configurations/config-storageExtension.yaml", "/logs/storageExtension.log", logStreamName); - String resourceFilePath = "/logs/storageExtension.log"; // Path to the resource file + collector = createAndStartCollector("/configurations/config-storageExtension.yaml", logStreamName); + File tempFile = new File(logDirectory.toString(), "storageExtension.log"); + Thread.sleep(5000); + + FileWriter fileWriter = new FileWriter(tempFile); try { - Thread.sleep(10000); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - try (OutputStream outputStream = new FileOutputStream(new File(getClass().getResource(resourceFilePath).toURI()), true); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { - // Append the lines to the file - writer.write("[otel.javaagent 2023-09-28 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- First Entry"); - writer.newLine(); - writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Second Entry"); - writer.newLine(); - writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Third Entry"); - writer.newLine(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException("Error writing to the file: " + resourceFilePath, e); + fileWriter.write("First Message, collector is running" + "\n"); + fileWriter.write("Second Message, collector is running" + "\n"); + fileWriter.write("Third Message, collector is running" + "\n"); + } catch (IOException e) { + throw new RuntimeException("Error writing to File A.", e); } + fileWriter.flush(); List logFilePaths = new ArrayList<>(); - logFilePaths.add(resourceFilePath); - validateLogs(logStreamName , logFilePaths, true); + String expectedLogPath = logDirectory.toString(); + logFilePaths.add(expectedLogPath + "/storageExtension.log"); + + validateLogs(logStreamName , logFilePaths, false); + collector.stop(); - try { - Thread.sleep(10000); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - } - try (OutputStream outputStream = new FileOutputStream(new File(getClass().getResource(resourceFilePath).toURI()), true); - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { - // Append the lines to the file - writer.write("[otel.javaagent 2023-09-28 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fourth Entry"); - writer.newLine(); - writer.write("[otel.javaagent 2023-09-29 16:56:22:242 +0000] [OkHttp ConnectionPool] DEBUG okhttp3.internal.concurrent.TaskRunner - Q10002 run again after 300 s : OkHttp ConnectionPool- Fifth Entry"); - writer.newLine(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException("Error writing to the file: " + resourceFilePath, e); - } - collector.start(); + Thread.sleep(5000); + + // write to the file when collector is stopped try { - Thread.sleep(10000); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); + fileWriter.write("First Message after collector is stopped" + "\n"); + fileWriter.write("Second Message after the collector is stopped" + "\n"); + fileWriter.write("Third Message after the collector is stopped" + "\n"); + fileWriter.flush(); + } catch (IOException e) { + throw new RuntimeException("Error writing to File A.", e); } - validateLogs(logStreamName , logFilePaths, true); + fileWriter.close(); + //restarting the collector + collector.start(); + + validateLogs(logStreamName , logFilePaths, false); collector.stop(); } @Test void testFileRotation() throws Exception { String logStreamName = "fileRotation-logstream-" + uniqueID; - String resourceFilePath = "/logs/log4j.log"; // Path to the resource file - collector = createAndStartCollector("/configurations/config-fileRotation.yaml", resourceFilePath, logStreamName); + collector = createAndStartCollector("/configurations/config-fileRotation.yaml", logStreamName); - Thread.sleep(10000); + Thread.sleep(5000); // Create and write data to File A File tempFile = new File(logDirectory.toString(), "testlogA.log"); @@ -196,13 +187,11 @@ void testFileRotation() throws Exception { } catch (IOException e) { throw new RuntimeException("Error writing to File A.", e); } - Thread.sleep(10000); + Thread.sleep(5000); File renameFile = new File(logDirectory.toString(), "testlogA-1234.log"); tempFile.renameTo(renameFile); - Thread.sleep(10000); - File tempFileB = new File(logDirectory.toString(), "testlogA.log"); try (FileWriter newfileWriter = new FileWriter(tempFileB)) { newfileWriter.write("Message in renamed file - line 1" + "\n"); @@ -213,7 +202,7 @@ void testFileRotation() throws Exception { } catch (IOException e) { throw new RuntimeException("Error writing to File B.", e); } - Thread.sleep(10000); + Thread.sleep(5000); List logFilePaths = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); @@ -231,6 +220,7 @@ void validateLogs(String testLogStreamName, List logFilePaths, boolean a for (String logFilePath : logFilePaths) { InputStream inputStream; + //Check whether the filePath is from resource folder. if (areResourceFiles) { inputStream = getClass().getResourceAsStream(logFilePath); } else { @@ -285,11 +275,13 @@ void validateLogs(String testLogStreamName, List logFilePaths, boolean a .filter(Objects::nonNull) .collect(Collectors.toSet()); - System.out.println("Log lines From Cloudwatch"); + //Print + System.out.println("Actual Logs - Log lines From Cloudwatch"); messageToValidate.forEach(System.out::println); - System.out.println("Expected logs"); + System.out.println("Expected logs- Log lines from log file"); lines.forEach(System.out::println); + //Validate body field in JSON-messageToValidate with actual log line from the log file. assertThat(messageToValidate.containsAll(lines)).isTrue(); assertThat(messageToValidate).containsExactlyInAnyOrderElementsOf(lines); diff --git a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml index 16078c216..d7e5382d5 100644 --- a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml +++ b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml @@ -1,6 +1,6 @@ receivers: filelog: - include: [/filerotation/testlogA.log] + include: [/tempLogs/testlogA.log] start_at: beginning exporters: diff --git a/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml b/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml index 9ff81c582..d56bb3014 100644 --- a/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml +++ b/adot-testbed/app/src/test/resources/configurations/config-storageExtension.yaml @@ -4,7 +4,7 @@ extensions: receivers: filelog: - include: [/logs/storageExtension.log] + include: [/tempLogs/storageExtension.log] storage: file_storage exporters: diff --git a/adot-testbed/app/src/test/resources/logs/storageExtension.log b/adot-testbed/app/src/test/resources/logs/storageExtension.log deleted file mode 100644 index e69de29bb..000000000 From 5e76506ac88e372aa0dfb94e12ee291026ec8f2a Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Wed, 11 Oct 2023 12:53:15 -0700 Subject: [PATCH 04/11] remove privilaged access start, handle exception --- .../software/amazon/adot/testbed/LogsTests.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index dba860f27..02ce1935a 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -57,7 +57,7 @@ class LogsTests { : "public.ecr.aws/aws-observability/aws-otel-collector:latest"; private final Logger collectorLogger = LoggerFactory.getLogger("collector"); private static final String uniqueID = UUID.randomUUID().toString(); - private final Path logDirectory = Files.createTempDirectory("tempLogs"); + private Path logDirectory; private GenericContainer collector; private GenericContainer createAndStartCollector(String configFilePath, String logStreamName) throws IOException { @@ -73,16 +73,17 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin if (System.getenv("AWS_SESSION_TOKEN") != null) { envVariables.put("AWS_SESSION_TOKEN", System.getenv("AWS_SESSION_TOKEN")); } -// envVariables.put("TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED", "true"); - + try { + logDirectory = Files.createTempDirectory("tempLogs"); + } catch (IOException e) { + throw new RuntimeException("Failed to create log directory", e); + } var collector = new GenericContainer<>(TEST_IMAGE) .withCopyFileToContainer(MountableFile.forClasspathResource(configFilePath), "/etc/collector/config.yaml") .withLogConsumer(new Slf4jLogConsumer(collectorLogger)) .waitingFor(Wait.forLogMessage(".*Everything is ready. Begin running and processing data.*", 1)) .withEnv(envVariables) .withClasspathResourceMapping("/logs", "/logs", BindMode.READ_WRITE) - -// .withPrivilegedMode(true) .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.filelog.receiver,+adot.awscloudwatchlogs.exporter,+adot.file_storage.extension"); //Mount the Temp directory @@ -92,8 +93,6 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin return collector; } - LogsTests() throws Exception { - } @Test void testSyslog() throws Exception { String logStreamName = "rfcsyslog-logstream-" + uniqueID; From 497618c1342c23586354b5ae1981b5bdd6177ec9 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Wed, 11 Oct 2023 23:56:47 -0700 Subject: [PATCH 05/11] Modify Usage of Validator - no boolean, tweak configs --- .../software/amazon/adot/testbed/LogsTests.java | 16 ++++++++-------- .../configurations/config-fileRotation.yaml | 3 --- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 02ce1935a..166aafa99 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -100,7 +100,7 @@ void testSyslog() throws Exception { List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/RFC5424.log"); - validateLogs(logStreamName , logFilePaths, true); + validateLogs(logStreamName , logFilePaths); collector.stop(); } @@ -111,7 +111,7 @@ void testLog4j() throws Exception { List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/log4j.log"); - validateLogs(logStreamName , logFilePaths, true); + validateLogs(logStreamName , logFilePaths); collector.stop(); } @@ -122,7 +122,7 @@ void testJson() throws Exception { List logFilePaths = new ArrayList<>(); logFilePaths.add("/logs/testingJSON.log"); - validateLogs(logStreamName , logFilePaths, true); + validateLogs(logStreamName , logFilePaths); collector.stop(); } @@ -147,7 +147,7 @@ void testCollectorRestartStorageExtension() throws Exception { String expectedLogPath = logDirectory.toString(); logFilePaths.add(expectedLogPath + "/storageExtension.log"); - validateLogs(logStreamName , logFilePaths, false); + validateLogs(logStreamName , logFilePaths); collector.stop(); @@ -166,7 +166,7 @@ void testCollectorRestartStorageExtension() throws Exception { //restarting the collector collector.start(); - validateLogs(logStreamName , logFilePaths, false); + validateLogs(logStreamName , logFilePaths); collector.stop(); } @@ -208,19 +208,19 @@ void testFileRotation() throws Exception { logFilePaths.add(expectedLogPath + "/testlogA-1234.log"); logFilePaths.add(expectedLogPath + "/testlogA.log"); - validateLogs(logStreamName, logFilePaths, false); + validateLogs(logStreamName, logFilePaths); collector.stop(); } - void validateLogs(String testLogStreamName, List logFilePaths, boolean areResourceFiles) throws Exception { + void validateLogs(String testLogStreamName, List logFilePaths) throws Exception { var lines = new HashSet(); for (String logFilePath : logFilePaths) { InputStream inputStream; //Check whether the filePath is from resource folder. - if (areResourceFiles) { + if (getClass().getResource(logFilePath) != null) { inputStream = getClass().getResourceAsStream(logFilePath); } else { inputStream = new FileInputStream(logFilePath); diff --git a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml index d7e5382d5..7465e0226 100644 --- a/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml +++ b/adot-testbed/app/src/test/resources/configurations/config-fileRotation.yaml @@ -1,11 +1,8 @@ receivers: filelog: include: [/tempLogs/testlogA.log] - start_at: beginning exporters: - logging: - verbosity: detailed awscloudwatchlogs: log_group_name: "adot-testbed/logs-component-testing/logs" log_stream_name: ${LOG_STREAM_NAME} From 473629e9358736e8d3283da7d4c766052a65e0e2 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Thu, 12 Oct 2023 01:16:11 -0700 Subject: [PATCH 06/11] Remove extra threads --- .../src/test/java/software/amazon/adot/testbed/LogsTests.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 166aafa99..7bbc591b8 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -151,8 +151,6 @@ void testCollectorRestartStorageExtension() throws Exception { collector.stop(); - Thread.sleep(5000); - // write to the file when collector is stopped try { fileWriter.write("First Message after collector is stopped" + "\n"); @@ -201,7 +199,6 @@ void testFileRotation() throws Exception { } catch (IOException e) { throw new RuntimeException("Error writing to File B.", e); } - Thread.sleep(5000); List logFilePaths = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); From f2838134748f2504246ea0a5b4c6e3a4069580e7 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Thu, 12 Oct 2023 12:57:21 -0700 Subject: [PATCH 07/11] Remove Exception block, send input stream to validator --- .../amazon/adot/testbed/LogsTests.java | 131 +++++++++--------- 1 file changed, 62 insertions(+), 69 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 7bbc591b8..7b80ed0f4 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -92,15 +92,17 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin collector.start(); return collector; } - @Test void testSyslog() throws Exception { String logStreamName = "rfcsyslog-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-rfcsyslog.yaml", logStreamName); - List logFilePaths = new ArrayList<>(); - logFilePaths.add("/logs/RFC5424.log"); - validateLogs(logStreamName , logFilePaths); + List inputStreams = new ArrayList<>(); + InputStream inputStream = getClass().getResourceAsStream("/logs/RFC5424.log"); + inputStreams.add(inputStream); + + validateLogs(logStreamName, inputStreams); + collector.stop(); } @@ -109,9 +111,11 @@ void testLog4j() throws Exception { String logStreamName = "log4j-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-log4j.yaml", logStreamName); - List logFilePaths = new ArrayList<>(); - logFilePaths.add("/logs/log4j.log"); - validateLogs(logStreamName , logFilePaths); + List inputStreams = new ArrayList<>(); + InputStream inputStream = getClass().getResourceAsStream("/logs/log4j.log"); + inputStreams.add(inputStream); + + validateLogs(logStreamName, inputStreams); collector.stop(); } @@ -120,9 +124,12 @@ void testJson() throws Exception { String logStreamName = "json-logstream-" + uniqueID; collector = createAndStartCollector("/configurations/config-json.yaml", logStreamName); - List logFilePaths = new ArrayList<>(); - logFilePaths.add("/logs/testingJSON.log"); - validateLogs(logStreamName , logFilePaths); + List inputStreams = new ArrayList<>(); + InputStream inputStream = getClass().getResourceAsStream("/logs/testingJSON.log"); + inputStreams.add(inputStream); + + validateLogs(logStreamName, inputStreams); + collector.stop(); } @@ -134,37 +141,33 @@ void testCollectorRestartStorageExtension() throws Exception { Thread.sleep(5000); FileWriter fileWriter = new FileWriter(tempFile); - try { - fileWriter.write("First Message, collector is running" + "\n"); - fileWriter.write("Second Message, collector is running" + "\n"); - fileWriter.write("Third Message, collector is running" + "\n"); - } catch (IOException e) { - throw new RuntimeException("Error writing to File A.", e); - } + fileWriter.write("First Message, collector is running" + "\n"); + fileWriter.write("Second Message, collector is running" + "\n"); + fileWriter.write("Third Message, collector is running" + "\n"); fileWriter.flush(); - List logFilePaths = new ArrayList<>(); + Thread.sleep(5000); + List inputStreams = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); - logFilePaths.add(expectedLogPath + "/storageExtension.log"); + String logPath = expectedLogPath + "/storageExtension.log"; - validateLogs(logStreamName , logFilePaths); + InputStream inputStream = new FileInputStream(logPath); + inputStreams.add(inputStream); collector.stop(); // write to the file when collector is stopped - try { - fileWriter.write("First Message after collector is stopped" + "\n"); - fileWriter.write("Second Message after the collector is stopped" + "\n"); - fileWriter.write("Third Message after the collector is stopped" + "\n"); - fileWriter.flush(); - } catch (IOException e) { - throw new RuntimeException("Error writing to File A.", e); - } + fileWriter.write("First Message after collector is stopped" + "\n"); + fileWriter.write("Second Message after the collector is stopped" + "\n"); + fileWriter.write("Third Message after the collector is stopped" + "\n"); + fileWriter.flush(); + fileWriter.close(); - //restarting the collector + //restarting the collector collector.start(); - validateLogs(logStreamName , logFilePaths); + validateLogs(logStreamName, inputStreams); + collector.stop(); } @@ -177,59 +180,56 @@ void testFileRotation() throws Exception { // Create and write data to File A File tempFile = new File(logDirectory.toString(), "testlogA.log"); - try (FileWriter fileWriter = new FileWriter(tempFile)) { - fileWriter.write("Message in File A" + "\n"); - fileWriter.flush(); - fileWriter.close(); - } catch (IOException e) { - throw new RuntimeException("Error writing to File A.", e); - } + + FileWriter fileWriter = new FileWriter(tempFile); + fileWriter.write("Message in File A" + "\n"); + fileWriter.flush(); + fileWriter.close(); + Thread.sleep(5000); + //Rename testLogA File renameFile = new File(logDirectory.toString(), "testlogA-1234.log"); tempFile.renameTo(renameFile); + //Create testLogA again to imitate file rotation File tempFileB = new File(logDirectory.toString(), "testlogA.log"); - try (FileWriter newfileWriter = new FileWriter(tempFileB)) { - newfileWriter.write("Message in renamed file - line 1" + "\n"); - newfileWriter.write("Message in renamed file - line 2" + "\n"); - newfileWriter.write("Message in renamed file - line 3" + "\n"); - newfileWriter.flush(); - newfileWriter.close(); - } catch (IOException e) { - throw new RuntimeException("Error writing to File B.", e); - } - List logFilePaths = new ArrayList<>(); + FileWriter newfileWriter = new FileWriter(tempFileB); + newfileWriter.write("Message in renamed file - line 1" + "\n"); + newfileWriter.write("Message in renamed file - line 2" + "\n"); + newfileWriter.write("Message in renamed file - line 3" + "\n"); + newfileWriter.flush(); + newfileWriter.close(); + + + List inputStreams = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); - logFilePaths.add(expectedLogPath + "/testlogA-1234.log"); - logFilePaths.add(expectedLogPath + "/testlogA.log"); - validateLogs(logStreamName, logFilePaths); + String logPath1 = expectedLogPath + "/testlogA-1234.log"; + String logPath2 = expectedLogPath + "/testlogA.log"; + + InputStream inputStream1 = new FileInputStream(logPath1); + InputStream inputStream2 = new FileInputStream(logPath2); + inputStreams.add(inputStream1); + inputStreams.add(inputStream2); + + validateLogs(logStreamName, inputStreams); collector.stop(); } - - void validateLogs(String testLogStreamName, List logFilePaths) throws Exception { + void validateLogs(String testLogStreamName, List inputStreams) throws Exception { var lines = new HashSet(); - for (String logFilePath : logFilePaths) { - InputStream inputStream; - //Check whether the filePath is from resource folder. - if (getClass().getResource(logFilePath) != null) { - inputStream = getClass().getResourceAsStream(logFilePath); - } else { - inputStream = new FileInputStream(logFilePath); - } - + for (InputStream inputStream : inputStreams) { try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String line; while ((line = reader.readLine()) != null) { lines.add(line); } } catch (IOException e) { - throw new RuntimeException("Error reading from the file: " + logFilePath, e); + throw new RuntimeException("Error reading from the file: " + inputStream, e); } } @@ -271,13 +271,6 @@ void validateLogs(String testLogStreamName, List logFilePaths) throws Ex .filter(Objects::nonNull) .collect(Collectors.toSet()); - //Print - System.out.println("Actual Logs - Log lines From Cloudwatch"); - messageToValidate.forEach(System.out::println); - - System.out.println("Expected logs- Log lines from log file"); - lines.forEach(System.out::println); - //Validate body field in JSON-messageToValidate with actual log line from the log file. assertThat(messageToValidate.containsAll(lines)).isTrue(); assertThat(messageToValidate).containsExactlyInAnyOrderElementsOf(lines); From 06ba690462cbd35d63df50beae7f8463413c8512 Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Thu, 12 Oct 2023 13:26:17 -0700 Subject: [PATCH 08/11] Use PrintWriter --- .../amazon/adot/testbed/LogsTests.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 7b80ed0f4..f0294a306 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -3,7 +3,7 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; +import java.io.PrintWriter; import java.io.FileInputStream; import java.io.InputStreamReader; import java.io.FileOutputStream; @@ -140,11 +140,11 @@ void testCollectorRestartStorageExtension() throws Exception { File tempFile = new File(logDirectory.toString(), "storageExtension.log"); Thread.sleep(5000); - FileWriter fileWriter = new FileWriter(tempFile); - fileWriter.write("First Message, collector is running" + "\n"); - fileWriter.write("Second Message, collector is running" + "\n"); - fileWriter.write("Third Message, collector is running" + "\n"); - fileWriter.flush(); + PrintWriter printWriter = new PrintWriter(tempFile); + printWriter.println("First Message, collector is running"); + printWriter.println("Second Message, collector is running"); + printWriter.println("Third Message, collector is running"); + printWriter.flush(); Thread.sleep(5000); List inputStreams = new ArrayList<>(); @@ -157,12 +157,12 @@ void testCollectorRestartStorageExtension() throws Exception { collector.stop(); // write to the file when collector is stopped - fileWriter.write("First Message after collector is stopped" + "\n"); - fileWriter.write("Second Message after the collector is stopped" + "\n"); - fileWriter.write("Third Message after the collector is stopped" + "\n"); - fileWriter.flush(); + printWriter.println("First Message after collector is stopped"); + printWriter.println("Second Message after the collector is stopped"); + printWriter.println("Third Message after the collector is stopped"); + printWriter.flush(); - fileWriter.close(); + printWriter.close(); //restarting the collector collector.start(); @@ -181,10 +181,10 @@ void testFileRotation() throws Exception { // Create and write data to File A File tempFile = new File(logDirectory.toString(), "testlogA.log"); - FileWriter fileWriter = new FileWriter(tempFile); - fileWriter.write("Message in File A" + "\n"); - fileWriter.flush(); - fileWriter.close(); + PrintWriter printWriter = new PrintWriter(tempFile); + printWriter.println("Message in File A"); + printWriter.flush(); + printWriter.close(); Thread.sleep(5000); @@ -195,13 +195,12 @@ void testFileRotation() throws Exception { //Create testLogA again to imitate file rotation File tempFileB = new File(logDirectory.toString(), "testlogA.log"); - FileWriter newfileWriter = new FileWriter(tempFileB); - newfileWriter.write("Message in renamed file - line 1" + "\n"); - newfileWriter.write("Message in renamed file - line 2" + "\n"); - newfileWriter.write("Message in renamed file - line 3" + "\n"); - newfileWriter.flush(); - newfileWriter.close(); - + PrintWriter newprintWriter = new PrintWriter(tempFileB); + newprintWriter.println("Message in renamed file - line 1"); + newprintWriter.println("Message in renamed file - line 2"); + newprintWriter.println("Message in renamed file - line 3"); + newprintWriter.flush(); + newprintWriter.close(); List inputStreams = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); From 8e241fc06fac91295ebc97538a320ceb5414378e Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Thu, 12 Oct 2023 17:00:23 -0700 Subject: [PATCH 09/11] Validate 2 in filerotation test case --- .../java/software/amazon/adot/testbed/LogsTests.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index f0294a306..7ec93afe1 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -186,7 +186,14 @@ void testFileRotation() throws Exception { printWriter.flush(); printWriter.close(); - Thread.sleep(5000); + List inputStreams = new ArrayList<>(); + String expectedLogPath = logDirectory.toString(); + + String logPath = expectedLogPath + "/testlogA.log"; + InputStream inputStream = new FileInputStream(logPath); + inputStreams.add(inputStream); + validateLogs(logStreamName, inputStreams); + inputStreams.remove(inputStream); //Rename testLogA File renameFile = new File(logDirectory.toString(), "testlogA-1234.log"); @@ -202,8 +209,6 @@ void testFileRotation() throws Exception { newprintWriter.flush(); newprintWriter.close(); - List inputStreams = new ArrayList<>(); - String expectedLogPath = logDirectory.toString(); String logPath1 = expectedLogPath + "/testlogA-1234.log"; String logPath2 = expectedLogPath + "/testlogA.log"; From f79acb432664fd02beafdc6a438dd682db5b147e Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Thu, 12 Oct 2023 17:50:45 -0700 Subject: [PATCH 10/11] Addback the validation 2 in storage extension test case --- .../software/amazon/adot/testbed/LogsTests.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index 7ec93afe1..c5d9dfe1e 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -143,7 +143,7 @@ void testCollectorRestartStorageExtension() throws Exception { PrintWriter printWriter = new PrintWriter(tempFile); printWriter.println("First Message, collector is running"); printWriter.println("Second Message, collector is running"); - printWriter.println("Third Message, collector is running"); + printWriter.println("Third Message, collector is running"); printWriter.flush(); Thread.sleep(5000); @@ -154,8 +154,15 @@ void testCollectorRestartStorageExtension() throws Exception { InputStream inputStream = new FileInputStream(logPath); inputStreams.add(inputStream); + validateLogs(logStreamName, inputStreams); + collector.stop(); + // Create a new InputStream for the second call + InputStream secondInputStream = new FileInputStream(logPath); + List secondInputStreams = new ArrayList<>(); + secondInputStreams.add(secondInputStream); + // write to the file when collector is stopped printWriter.println("First Message after collector is stopped"); printWriter.println("Second Message after the collector is stopped"); @@ -163,10 +170,10 @@ void testCollectorRestartStorageExtension() throws Exception { printWriter.flush(); printWriter.close(); - //restarting the collector + // Restart the collector collector.start(); - validateLogs(logStreamName, inputStreams); + validateLogs(logStreamName, secondInputStreams); collector.stop(); } From 25b705aca0ea41a7519bf0fab6e7bf10bd0d06ee Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy Date: Fri, 13 Oct 2023 09:56:12 -0700 Subject: [PATCH 11/11] Rm extra thread, Use right feature gates. --- .../src/test/java/software/amazon/adot/testbed/LogsTests.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java index c5d9dfe1e..c15985481 100644 --- a/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java +++ b/adot-testbed/app/src/test/java/software/amazon/adot/testbed/LogsTests.java @@ -84,7 +84,7 @@ private GenericContainer createAndStartCollector(String configFilePath, Strin .waitingFor(Wait.forLogMessage(".*Everything is ready. Begin running and processing data.*", 1)) .withEnv(envVariables) .withClasspathResourceMapping("/logs", "/logs", BindMode.READ_WRITE) - .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.filelog.receiver,+adot.awscloudwatchlogs.exporter,+adot.file_storage.extension"); + .withCommand("--config", "/etc/collector/config.yaml", "--feature-gates=+adot.receiver.filelog,+adot.exporter.awscloudwatchlogs,+adot.extension.file_storage"); //Mount the Temp directory collector.withFileSystemBind(logDirectory.toString(),"/tempLogs", BindMode.READ_WRITE); @@ -146,7 +146,6 @@ void testCollectorRestartStorageExtension() throws Exception { printWriter.println("Third Message, collector is running"); printWriter.flush(); - Thread.sleep(5000); List inputStreams = new ArrayList<>(); String expectedLogPath = logDirectory.toString(); String logPath = expectedLogPath + "/storageExtension.log";