feat: storage now have a namespace parameter

kestra-io · Nov 21, 2024 · c5ef5c6 · c5ef5c6
1 parent 1fe694e
commit c5ef5c6
Show file tree

Hide file tree

Showing 14 changed files with 16 additions and 15 deletions.
diff --git a/gradle.properties b/gradle.properties
@@ -1,2 +1,2 @@
 version=0.20.0-SNAPSHOT
-kestraVersion=[0.18,)
+kestraVersion=[0.20,)
diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/ExtractToGcsTest.java
@@ -98,7 +98,7 @@ void toCsv() throws Exception {
             .build();
 
         Download.Output downloadOutput = downloadTask.run(runContext(downloadTask));
-        InputStream get = storageInterface.get(null, downloadOutput.getUri());
+        InputStream get = storageInterface.get(null, null, downloadOutput.getUri());
 
         // Tests
         assertThat(extractOutput.getFileCounts().get(0), is(1L));

diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadFromGcsTest.java
@@ -47,6 +47,7 @@ void fromJson() throws Exception {
         );
 
         URI put = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(applicationFile)

diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/LoadTest.java
@@ -15,7 +15,6 @@
 import java.io.FileInputStream;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
-import java.util.NoSuchElementException;
 import java.util.Objects;
 import jakarta.inject.Inject;
 
@@ -41,6 +40,7 @@ class LoadTest {
     @Test
     void fromCsv() throws Exception {
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(new File(Objects.requireNonNull(LoadTest.class.getClassLoader()
@@ -72,6 +72,7 @@ void fromCsv() throws Exception {
     @Test
     void fromAvro() throws Exception {
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(new File(Objects.requireNonNull(LoadTest.class.getClassLoader()
@@ -101,6 +102,7 @@ void fromAvro() throws Exception {
     @Test
     void fromEmpty() throws Exception {
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             IOUtils.toInputStream("", StandardCharsets.UTF_8)

diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/QueryTest.java
@@ -8,7 +8,6 @@
 import io.micronaut.context.annotation.Value;
 import io.kestra.core.junit.annotations.KestraTest;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import io.kestra.core.runners.RunContext;
@@ -23,7 +22,6 @@
 import java.time.LocalTime;
 import java.util.*;
 import java.util.concurrent.*;
-import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import jakarta.inject.Inject;
@@ -123,7 +121,7 @@ void store() throws Exception {
             .build();
 
         Query.Output run = task.run(TestsUtils.mockRunContext(runContextFactory, task, ImmutableMap.of()));
-        String ionResult = CharStreams.toString(new InputStreamReader(storageInterface.get(null, run.getUri())));
+        String ionResult = CharStreams.toString(new InputStreamReader(storageInterface.get(null, null, run.getUri())));
 
         assertThat(ionResult, containsString("string:\"hello\""));
         assertThat(ionResult, containsString("datetime:2008-12-25T15:30:00.123Z"));

diff --git a/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java b/src/test/java/io/kestra/plugin/gcp/bigquery/StorageWriteTest.java
@@ -22,7 +22,6 @@
 import java.time.ZonedDateTime;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Locale;
 import java.util.Map;
 import jakarta.inject.Inject;
 
@@ -86,6 +85,7 @@ void run() throws Exception {
         }
 
         URI put = storageInterface.put(
+            null,
             null,
             new URI("/" + IdUtils.create() + ".ion"),
             new FileInputStream(tempFile)

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/ComposeTest.java
@@ -60,7 +60,7 @@ void run() throws Exception {
             .from(run.getUri().toString())
             .build();
 
-        InputStream get = storageInterface.get(null, download.run(runContext).getUri());
+        InputStream get = storageInterface.get(null, null, download.run(runContext).getUri());
 
         assertThat(
             CharStreams.toString(new InputStreamReader(get)),

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/CopyTest.java
@@ -38,6 +38,7 @@ void run() throws Exception {
         String out = FriendlyId.createFriendlyId();
 
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(new File(Objects.requireNonNull(UploadTest.class.getClassLoader()

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteListTest.java
@@ -5,15 +5,10 @@
 import io.kestra.core.runners.RunContextFactory;
 import io.kestra.core.storages.StorageInterface;
 import io.kestra.core.utils.TestsUtils;
-import io.kestra.plugin.gcp.gcs.models.Blob;
 import io.micronaut.context.annotation.Value;
 import io.kestra.core.junit.annotations.KestraTest;
 import org.junit.jupiter.api.Test;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.net.URI;
-import java.util.Objects;
 import jakarta.inject.Inject;
 
 import static org.hamcrest.MatcherAssert.assertThat;

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DeleteTest.java
@@ -38,6 +38,7 @@ void fromStorage() throws Exception {
             .toURI());
 
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(file)

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/DownloadTest.java
@@ -42,6 +42,7 @@ void fromStorage() throws Exception {
             .toURI());
 
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(file)
@@ -68,7 +69,7 @@ void fromStorage() throws Exception {
         Download.Output run = task.run(runContext(task));
         assertThat(run.getUri().toString(), endsWith(".yml"));
 
-        InputStream get = storageInterface.get(null, run.getUri());
+        InputStream get = storageInterface.get(null, null, run.getUri());
 
         assertThat(
             CharStreams.toString(new InputStreamReader(get)),

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java b/src/test/java/io/kestra/plugin/gcp/gcs/GcsTestUtils.java
@@ -34,6 +34,7 @@ Upload.Output upload(String out) throws Exception {
 
     Upload.Output upload(String out, String resource) throws Exception {
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(new File(Objects.requireNonNull(UploadTest.class.getClassLoader()

diff --git a/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java b/src/test/java/io/kestra/plugin/gcp/gcs/ListTest.java
@@ -104,6 +104,7 @@ void run() throws Exception {
 
     static String upload(StorageInterface storageInterface, String bucket, RunContextFactory runContextFactory, String dir) throws Exception {
         URI source = storageInterface.put(
+            null,
             null,
             new URI("/" + FriendlyId.createFriendlyId()),
             new FileInputStream(new File(Objects.requireNonNull(ListTest.class.getClassLoader()

diff --git a/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java b/src/test/java/io/kestra/plugin/gcp/pubsub/PublishThenConsumeTest.java
@@ -128,6 +128,6 @@ private URI createTestFile(RunContext runContext) throws Exception {
             Message.builder().data("Hello World".getBytes()).build());
         FileSerde.write(output,
             Message.builder().attributes(Map.of("key", "value")).build());
-        return storageInterface.put(null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile));
+        return storageInterface.put(null, null, URI.create("/" + IdUtils.create() + ".ion"), new FileInputStream(tempFile));
     }
 }