diff --git a/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature b/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature index 968d5afc6..924c70084 100644 --- a/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature +++ b/core-plugins/src/e2e-test/features/xmlReader/XmlReader.feature @@ -44,7 +44,7 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate OUT record count is equal to IN record count - Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" + Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" @XmlReader_Source_Required @XMLREADER_DELETE_TEST @FILE_SINK_TEST Scenario: To verify data is getting transferred from XmlReader to File sink using pattern and delete action @@ -78,7 +78,7 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra Then Close the pipeline logs Then Validate OUT record count is equal to IN record count Then Validate that file gets successfully deleted from the gcs bucket - Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" + Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" @XMLREADER_TEST @FILE_SINK_TEST Scenario: To verify data is getting transferred from XmlReader to File sink using move action @@ -112,4 +112,4 @@ Feature:File Sink - Verification of XmlReader plugin to File successful data tra Then Close the pipeline logs Then Validate OUT record count is equal to IN record count Then Validate that file gets successfully moved to the target location - Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" + Then Validate output file generated by xmlReader and file sink plugin "fileSinkTargetBucket" is equal to expected output file "outputForXMLTest" diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java index a39278ade..cdf3b8e2e 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/CorePlugin.java @@ -120,6 +120,21 @@ public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedO } } + @Then("Validate output file generated by xmlReader and file sink plugin {string} is equal to expected" + + " output file {string}") + public static void validateOutputFileGeneratedByFileSinkPluginIsEqualToExpectedOutputFileForXml + (String fileSinkBucketPath, String expectedOutputFilePath) { + if (TestSetupHooks.testOnCdap.equals("true")) { + String bucketName = TestSetupHooks.fileSourceBucket; + ValidationHelper.compareDataForXml(fileSinkBucketPath, + PluginPropertyUtils.pluginProp(expectedOutputFilePath), bucketName); + } else { + String outputFileSinkBucketName = PluginPropertyUtils.pluginProp(fileSinkBucketPath); + ValidationHelper.listBucketObjects(outputFileSinkBucketName.substring(5), + PluginPropertyUtils.pluginProp(expectedOutputFilePath)); + } + } + @Then("Validate output Orc file generated by file sink plugin {string} is equal to expected output file {string}") public void validateOutputOrcFileGeneratedByFileSinkPluginIsEqualToExpectedOutputFile(String fileSinkBucketPath, String expectedOutputFilePath) throws IOException { diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 4e378b76c..dab8c670c 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -472,7 +472,7 @@ private static String createGCSBucketWithFile(String filePath) throws IOExceptio } private static String createGCSBucketWithXmlFile(String filePath) throws IOException, URISyntaxException { - String bucketName = StorageClient.createBucket("e2e-test-xml").getName(); + String bucketName = StorageClient.createBucket("e2e-test-" + UUID.randomUUID()).getName(); StorageClient.uploadObject(bucketName, filePath, filePath); return bucketName; } @@ -522,6 +522,8 @@ public static void createBucketWithXmlFile() throws IOException, URISyntaxExcept fileSourceBucket = createGCSBucketWithXmlFile(PluginPropertyUtils.pluginProp("xmlFile")); PluginPropertyUtils.addPluginProp("xmlTestFile", "gs://" + fileSourceBucket + "/" + PluginPropertyUtils.pluginProp("xmlFile")); + PluginPropertyUtils.addPluginProp("bucketName", fileSourceBucket); + PluginPropertyUtils.addPluginProp("targetFolder", "gs://" + fileSourceBucket + "/"); BeforeActions.scenario.write("xml test bucket name - " + fileSourceBucket); } diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java index fb6861f99..09164bf47 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/ValidationHelper.java @@ -97,6 +97,11 @@ public static void compareData(String fileSinkBucketPath, String expectedOutputF validateBothFilesData(outputFileData, expectedOutputFilePath); } + public static void compareDataForXml(String fileSinkBucketPath, String expectedOutputFilePath, String bucketName) { + List outputFileData = readAllFilesFromDirectory(PluginPropertyUtils.pluginProp(fileSinkBucketPath)); + validateBothFilesDataForXml(outputFileData, expectedOutputFilePath, bucketName); + } + // Reading data from all part-r files present in a directory. public static List readAllFilesFromDirectory(String directoryPath) { List outputFileData = new ArrayList<>(); @@ -144,4 +149,40 @@ private static boolean compareFiles(String firstFilePath, String secondFilePath) return Arrays.equals(first, second); } + + private static void validateBothFilesDataForXml(List outputFileData, String expectedOutputFilePath, + String bucketName) { + String[] expectedOutputFileData = fetchObjectData(projectId, expectedOutputFilesBucketName, expectedOutputFilePath); + String xmlExpectedFileBucketName = PluginPropertyUtils.pluginProp("xmlBucketName"); + // Define the bucket names to remove + String expectedFileBucketPath = "gs://" + xmlExpectedFileBucketName + "/"; + String outputFileBucketPath = "gs://" + bucketName + "/"; + + // Normalize expected data by removing the bucket name + List normalizedExpectedData = new ArrayList<>(); + for (String line : expectedOutputFileData) { + normalizedExpectedData.add(line.replace(expectedFileBucketPath, "")); + } + + // Normalize output data by removing the bucket name + List normalizedOutputData = new ArrayList<>(); + for (String line : outputFileData) { + normalizedOutputData.add(line.replace(outputFileBucketPath, "")); + } + + // Compare normalized data + int counter = 0; + for (String expectedLine : normalizedExpectedData) { + Assert.assertTrue( + "Output file and Expected file data should be the same", + normalizedOutputData.contains(expectedLine) + ); + counter++; + } + Assert.assertEquals( + "Number of lines in output file and expected file should match", + normalizedExpectedData.size(), + counter + ); + } } diff --git a/core-plugins/src/e2e-test/resources/pluginParameters.properties b/core-plugins/src/e2e-test/resources/pluginParameters.properties index 1ebbe429a..f2d0553a6 100644 --- a/core-plugins/src/e2e-test/resources/pluginParameters.properties +++ b/core-plugins/src/e2e-test/resources/pluginParameters.properties @@ -304,8 +304,8 @@ xmlFile=testdata/xmldata/testxmlfile.xml node=/students/student outputForXMLTest=e2e-tests/file/expected_outputs/OUTPUT_FOR_XMLREADER_TEST.csv filePattern=^testxmlfile.xml -folder=gs://e2e-test-xml/ -bucketName=e2e-test-xml +targetFolder=dummy +xmlBucketName=e2e-test-xml invalidPattern=abcd invalidNode=dummy xmlFileName=testxmlfile.xml