From 0d3840d180e1b54b477d959816e3a14cb76e098f Mon Sep 17 00:00:00 2001 From: horizonzy Date: Wed, 1 Nov 2023 16:49:12 +0800 Subject: [PATCH 1/5] disable s3 in the test. --- .../ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java index 0b2e01c..f62cb19 100644 --- a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java +++ b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java @@ -76,8 +76,8 @@ public class HoodieWriterTest { @DataProvider(name = "storage") public Object[][] storageType() { return new Object[][]{ - {STORAGE_LOCAL}, - {STORAGE_S3} + {STORAGE_LOCAL} +// , {STORAGE_S3} }; } From b303fe04e3ee266b3b1c2ddcac10509f0688110d Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 2 Nov 2023 16:27:13 +0800 Subject: [PATCH 2/5] address the comment. --- .github/workflows/pr-unit-tests-jdk17.yml | 10 ++++- .../sink/hudi/HoodieTestS3Cloud.java | 38 +++++++++++++++++++ .../lakehouse/sink/hudi/HoodieWriterTest.java | 3 -- 3 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java diff --git a/.github/workflows/pr-unit-tests-jdk17.yml b/.github/workflows/pr-unit-tests-jdk17.yml index 79a353f..d30cfc8 100644 --- a/.github/workflows/pr-unit-tests-jdk17.yml +++ b/.github/workflows/pr-unit-tests-jdk17.yml @@ -45,7 +45,15 @@ jobs: CLOUD_BUCKET_NAME: ${{ secrets.CLOUD_BUCKET_NAME }} AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - run: mvn test -Pcloud + run: mvn test -Pcloud -Dtest="!org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi.HoodieTestS3Cloud" + + - name: cloud test after build + if: github.event_name == 'push' + env: + CLOUD_BUCKET_NAME: ${{ secrets.CLOUD_BUCKET_NAME }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + run: mvn test -Pcloud -Dtest='org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi.HoodieTestS3Cloud' - name: package surefire artifacts if: failure() diff --git a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java new file mode 100644 index 0000000..2bffc9a --- /dev/null +++ b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi; + +import java.net.URI; +import org.apache.pulsar.ecosystem.io.lakehouse.SinkConnectorConfig; +import org.testng.annotations.DataProvider; + +public class HoodieTestS3Cloud extends HoodieWriterTest { + + private static final String STORAGE_S3 = "S3"; + + private URI testPath; + private SinkConnectorConfig sinkConfig; + + @DataProvider(name = "storage") + public Object[][] storageType() { + return new Object[][]{ + {STORAGE_S3} + }; + } +} diff --git a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java index f62cb19..bab0016 100644 --- a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java +++ b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieWriterTest.java @@ -67,8 +67,6 @@ public class HoodieWriterTest { private static final Path PROJECT_DATA = FileSystems.getDefault().getPath("data").toAbsolutePath(); private static final String STORAGE_LOCAL = "LOCAL"; - private static final String STORAGE_S3 = "S3"; - private static final String STORAGE_GCS = "GCS"; private URI testPath; private SinkConnectorConfig sinkConfig; @@ -77,7 +75,6 @@ public class HoodieWriterTest { public Object[][] storageType() { return new Object[][]{ {STORAGE_LOCAL} -// , {STORAGE_S3} }; } From 4db95b5fa6165797a07ec114eb1542f3e20b6453 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 2 Nov 2023 16:30:26 +0800 Subject: [PATCH 3/5] code clean. --- .../ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java index 2bffc9a..f92b77b 100644 --- a/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java +++ b/src/test/java/org/apache/pulsar/ecosystem/io/lakehouse/sink/hudi/HoodieTestS3Cloud.java @@ -18,17 +18,12 @@ */ package org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi; -import java.net.URI; -import org.apache.pulsar.ecosystem.io.lakehouse.SinkConnectorConfig; import org.testng.annotations.DataProvider; public class HoodieTestS3Cloud extends HoodieWriterTest { private static final String STORAGE_S3 = "S3"; - private URI testPath; - private SinkConnectorConfig sinkConfig; - @DataProvider(name = "storage") public Object[][] storageType() { return new Object[][]{ From bd34310ea8bdef511ae8cfa2a0c16eaf67577717 Mon Sep 17 00:00:00 2001 From: horizonzy Date: Thu, 2 Nov 2023 16:33:09 +0800 Subject: [PATCH 4/5] code clean. --- .github/workflows/pr-unit-tests-jdk17.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pr-unit-tests-jdk17.yml b/.github/workflows/pr-unit-tests-jdk17.yml index d30cfc8..a06a279 100644 --- a/.github/workflows/pr-unit-tests-jdk17.yml +++ b/.github/workflows/pr-unit-tests-jdk17.yml @@ -42,9 +42,6 @@ jobs: - name: unit test after build env: - CLOUD_BUCKET_NAME: ${{ secrets.CLOUD_BUCKET_NAME }} - AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} run: mvn test -Pcloud -Dtest="!org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi.HoodieTestS3Cloud" - name: cloud test after build From 3709fd8fd598baa80df2a8844e1b394c5c0ae101 Mon Sep 17 00:00:00 2001 From: Yong Zhang Date: Mon, 6 Nov 2023 09:24:16 +0800 Subject: [PATCH 5/5] Update .github/workflows/pr-unit-tests-jdk17.yml --- .github/workflows/pr-unit-tests-jdk17.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr-unit-tests-jdk17.yml b/.github/workflows/pr-unit-tests-jdk17.yml index a06a279..8af7dc4 100644 --- a/.github/workflows/pr-unit-tests-jdk17.yml +++ b/.github/workflows/pr-unit-tests-jdk17.yml @@ -41,7 +41,6 @@ jobs: run: mvn -ntp -B spotbugs:check - name: unit test after build - env: run: mvn test -Pcloud -Dtest="!org.apache.pulsar.ecosystem.io.lakehouse.sink.hudi.HoodieTestS3Cloud" - name: cloud test after build