From c5ef5c60aba406220b373d8903c21394ad428c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Mathieu?= Date: Thu, 21 Nov 2024 10:37:13 +0100 Subject: [PATCH] feat: storage now have a namespace parameter --- gradle.properties | 2 +- .../java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java | 2 +- .../java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java | 1 + src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java | 4 +++- src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java | 4 +--- .../java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java | 2 +- src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java | 2 +- src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java | 1 + src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java | 5 ----- src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java | 1 + src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java | 3 ++- src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java | 1 + src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java | 1 + .../io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java | 2 +- 14 files changed, 16 insertions(+), 15 deletions(-) diff --git a/gradle.properties b/gradle.properties index d140d9f7..3ccabc36 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,2 @@ version=0.20.0-SNAPSHOT -kestraVersion=[0.18,) +kestraVersion=[0.20,) diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java index 3afe7efa..ceb20770 100644 --- a/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java +++ b/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java @@ -98,7 +98,7 @@ void toCsv() throws Exception { .build(); Download.Output downloadOutput = downloadTask.run(runContext(downloadTask)); - InputStream get = storageInterface.get(null, downloadOutput.getUri()); + InputStream get = storageInterface.get(null, null, downloadOutput.getUri()); // Tests assertThat(extractOutput.getFileCounts().get(0), is(1L)); diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java index 68551969..bf1299a1 100644 --- a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java +++ b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java @@ -47,6 +47,7 @@ void fromJson() throws Exception { ); URI put = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(applicationFile) diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java index 15861d47..82240775 100644 --- a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java +++ b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java @@ -15,7 +15,6 @@ import java.io.FileInputStream; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.NoSuchElementException; import java.util.Objects; import jakarta.inject.Inject; @@ -41,6 +40,7 @@ class LoadTest { @Test void fromCsv() throws Exception { URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(new File(Objects.requireNonNull(LoadTest.class.getClassLoader() @@ -72,6 +72,7 @@ void fromCsv() throws Exception { @Test void fromAvro() throws Exception { URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(new File(Objects.requireNonNull(LoadTest.class.getClassLoader() @@ -101,6 +102,7 @@ void fromAvro() throws Exception { @Test void fromEmpty() throws Exception { URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), IOUtils.toInputStream("", StandardCharsets.UTF_8) diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java index 0ab6fb79..78fa259e 100644 --- a/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java +++ b/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java @@ -8,7 +8,6 @@ import io.micronaut.context.annotation.Value; import io.kestra.core.junit.annotations.KestraTest; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import io.kestra.core.runners.RunContext; @@ -23,7 +22,6 @@ import java.time.LocalTime; import java.util.*; import java.util.concurrent.*; -import java.util.stream.Collectors; import java.util.stream.IntStream; import jakarta.inject.Inject; @@ -123,7 +121,7 @@ void store() throws Exception { .build(); Query.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, ImmutableMap.of())); - String ionResult = CharStreams.toString(new InputStreamReader(storageInterface.get(null, run.getUri()))); + String ionResult = CharStreams.toString(new InputStreamReader(storageInterface.get(null, null, run.getUri()))); assertThat(ionResult, containsString("string:\"hello\"")); assertThat(ionResult, containsString("datetime:2008-12-25T15:30:00.123Z")); diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java index 03a22cbc..15a01dc4 100644 --- a/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java +++ b/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java @@ -22,7 +22,6 @@ import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; -import java.util.Locale; import java.util.Map; import jakarta.inject.Inject; @@ -86,6 +85,7 @@ void run() throws Exception { } URI put = storageInterface.put( + null, null, new URI("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile) diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java index 1aa0345c..061c4131 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java @@ -60,7 +60,7 @@ void run() throws Exception { .from(run.getUri().toString()) .build(); - InputStream get = storageInterface.get(null, download.run(runContext).getUri()); + InputStream get = storageInterface.get(null, null, download.run(runContext).getUri()); assertThat( CharStreams.toString(new InputStreamReader(get)), diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java index 7a9bafd4..d3ac5c27 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java @@ -38,6 +38,7 @@ void run() throws Exception { String out = FriendlyId.createFriendlyId(); URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(new File(Objects.requireNonNull(UploadTest.class.getClassLoader() diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java index 503b4064..e19109d5 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java @@ -5,15 +5,10 @@ import io.kestra.core.runners.RunContextFactory; import io.kestra.core.storages.StorageInterface; import io.kestra.core.utils.TestsUtils; -import io.kestra.plugin.gcp.gcs.models.Blob; import io.micronaut.context.annotation.Value; import io.kestra.core.junit.annotations.KestraTest; import org.junit.jupiter.api.Test; -import java.io.File; -import java.io.FileInputStream; -import java.net.URI; -import java.util.Objects; import jakarta.inject.Inject; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java index cd4b23bd..ddf93d7c 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java @@ -38,6 +38,7 @@ void fromStorage() throws Exception { .toURI()); URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(file) diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java index ea1aab62..17fdfa51 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java @@ -42,6 +42,7 @@ void fromStorage() throws Exception { .toURI()); URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(file) @@ -68,7 +69,7 @@ void fromStorage() throws Exception { Download.Output run = task.run(runContext(task)); assertThat(run.getUri().toString(), endsWith(".yml")); - InputStream get = storageInterface.get(null, run.getUri()); + InputStream get = storageInterface.get(null, null, run.getUri()); assertThat( CharStreams.toString(new InputStreamReader(get)), diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java b/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java index 03271c5e..212f6edb 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java @@ -34,6 +34,7 @@ Upload.Output upload(String out) throws Exception { Upload.Output upload(String out, String resource) throws Exception { URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(new File(Objects.requireNonNull(UploadTest.class.getClassLoader() diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java index eb027482..80b95164 100644 --- a/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java +++ b/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java @@ -104,6 +104,7 @@ void run() throws Exception { static String upload(StorageInterface storageInterface, String bucket, RunContextFactory runContextFactory, String dir) throws Exception { URI source = storageInterface.put( + null, null, new URI("/" + FriendlyId.createFriendlyId()), new FileInputStream(new File(Objects.requireNonNull(ListTest.class.getClassLoader() diff --git a/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java b/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java index 3205b593..c7b71729 100644 --- a/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java +++ b/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java @@ -128,6 +128,6 @@ private URI createTestFile(RunContext runContext) throws Exception { Message.builder().data("Hello World".getBytes()).build()); FileSerde.write(output, Message.builder().attributes(Map.of("key", "value")).build()); - return storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); + return storageInterface.put(null, null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile)); } }