Merge branch 'dev' of https://github.com/muyangye/streampipes into dev

installer/cli/README.md

@@ -152,20 +152,20 @@ streampipes pull
 
 **Stop** existing StreamPipes containers
 ```bash
-streampipes stop pipeline-elements-all-jvm
+streampipes stop extensions-all-jvm
 ```
 
 **Start** existing StreamPipes containers
 ```bash
-streampipes start pipeline-elements-all-jvm
+streampipes start extensions-all-jvm
 ```
 
 **Restart** existing services
 ```bash
 # restart backend consul container
 streampipes restart backend consul
 # restart existing services by removing and recreating container instance
-streampipes restart --force-create pipeline-elements-all-jvm
+streampipes restart --force-create extensions-all-jvm
 ```
 
 **Clean** your system and remove created StreamPipes Docker volumes, StreamPipes docker network and dangling StreamPipes images of old image layers.
@@ -215,7 +215,7 @@ As of now, this step has to be done **manually**. All environments are located i
 ├── backend               # developing core backend features
 ├── basic                 # wanna run core, UI, connect etc from the IDE?
 ├── full                  # full version containing more pipeline elements
-├── lite                  # few pipeline elements, less memory  
+├── lite                  # few pipeline elements, less memory
 ├── pipeline-element      # developing new pipeline-elements
 └── ui                    # developing UI features
 ```

installer/cli/bin/commands/info

@@ -34,7 +34,7 @@ fi
 
 cat <<EOF
 Version:      $(show_version)
-GitHub:       https://github.com/apache/incubator-streampipes-installer
+GitHub:       https://github.com/apache/streampipes-installer
 Website:      https://streampipes.apache.org
 Bug Report:   [email protected]
 

installer/cli/environments/full

@@ -16,14 +16,12 @@
 [environment:full]
 ui
 backend
-connect-adapters
-connect-adapters-iiot
+extensions-all-jvm
 consul
 couchdb
 jobmanager
 taskmanager
 zookeeper
 kafka
 influxdb
-pipeline-elements-all-jvm
 pipeline-elements-all-flink

installer/scripts/epsg/Readme.md

@@ -67,7 +67,7 @@ Coming soon
 
 The database, in which the epsg data will be imported, is already included as a service (Postgres DB). The Only step the user has to do is to fill the database with the required scripts.
 
-Therefore, the following empty scripts must be replaced in the `incubator-streampipes/installer/scripts/epsg` folder
+Therefore, the following empty scripts must be replaced in the `streampipes/installer/scripts/epsg` folder
 
 * PostgreSQL_Table_Script.sql
 * PostgreSQL_Data_Script.sql
@@ -90,14 +90,14 @@ Unzip the folder and replace files
 * PostgreSQL_Data_Script.sql
 * PostgreSQL_FKey_Script.sql
 
-in the `incubator-streampipes/installer/scripts/epsg` folder.
+in the `streampipes/installer/scripts/epsg` folder.
 
 For indexing the imported data and get better performance, go to 
 <a href="https://github.com/apache/sis/blob/master/core/sis-referencing/src/main/resources/org/apache/sis/referencing/factory/sql/EPSG_Finish.sql" target="_blank">this file</a>
 and replace it with the
 * EPSG_FINISH.sql
 
-in the `incubator-streampipes/installer/scripts/epsg` folder.
+in the `streampipes/installer/scripts/epsg` folder.
 
 # Import into Database
 

pom.xml

@@ -67,8 +67,8 @@
         <httpcore.version>4.4.9</httpcore.version>
         <influxdb.version>2.23</influxdb.version>
         <inlong.version>1.9.0</inlong.version>
-        <jackson.version>2.15.0</jackson.version>
-        <jackson.databind.version>2.15.0</jackson.databind.version>
+        <jackson.version>2.16.0</jackson.version>
+        <jackson.databind.version>2.16.0</jackson.databind.version>
         <jakarta-annotation.version>2.1.1</jakarta-annotation.version>
         <jakarta-activation-api.version>2.0.1</jakarta-activation-api.version>
         <jakarta-inject-api.version>2.0.1</jakarta-inject-api.version>

run-validation-system.sh

@@ -13,4 +13,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-docker-compose up -d && ../incubator-streampipes-extensions/docker-compose up -d
+docker-compose up -d && ../streampipes-extensions/docker-compose up -d

streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java

@@ -22,21 +22,32 @@
 import org.apache.streampipes.client.model.StreamPipesClientConfig;
 import org.apache.streampipes.client.util.StreamPipesApiPath;
 
+import java.util.Map;
+
 
 public class FileApi extends AbstractClientApi implements IFileApi {
 
   public FileApi(StreamPipesClientConfig clientConfig) {
     super(clientConfig);
   }
 
+  public byte[] getFileContent(String filename, boolean isOriginalFileName) {
+    return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename)
+            .withQueryParameters(Map.of("isOriginalFilename", String.valueOf(isOriginalFileName))), null)
+            .executeRequest();
+  }
   @Override
-  public byte[] getFileContent(String filename) {
-    return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename), null).executeRequest();
+  public byte[] getFileContent(String fileId) {
+    return getFileContent(fileId, false);
+  }
+
+  public String getFileContentAsString(String filename, boolean isOriginalFileName) {
+    return new String(getFileContent(filename, isOriginalFileName));
   }
 
   @Override
-  public String getFileContentAsString(String filename) {
-    return new String(getFileContent(filename));
+  public String getFileContentAsString(String fileId) {
+    return getFileContentAsString(fileId, false);
   }
 
   @Override

streampipes-extensions/docker-compose.yml

@@ -33,41 +33,41 @@ x-logging:
 
 services:
 
-  #### apache/incubator-streampipes-extensions
+  #### apache/streampipes-extensions
   connect-worker-main:
     build:
       context: ./streampipes-connect-adapters
       dockerfile: Dockerfile
       args:
         BASE_IMAGE: adoptopenjdk/openjdk8-openj9:alpine
-    image: incubator-streampipes_connect-worker-main:release-validation
+    image: streampipes_connect-worker-main:release-validation
     environment:
       - SP_CONNECT_CONTAINER_HOST=connect-master
       - SP_CONNECT_CONTAINER_WORKER=connect-worker-main
     logging: *default-logging
     networks:
       spnet:
 
-  pipeline-elements-all-jvm:
+  extensions-all-jvm:
     build:
-      context: ./streampipes-pipeline-elements-all-jvm
+      context: ./streampipes-extensions-all-jvm
       dockerfile: Dockerfile
       args:
         BASE_IMAGE: adoptopenjdk/openjdk8-openj9:alpine
-    image: incubator-streampipes_pipeline-elements-all-jvm:release-validation
+    image: streampipes_extensions-all-jvm:release-validation
     volumes:
     - files:/spImages
     logging: *default-logging
     networks:
       spnet:
 
-  pipeline-elements-all-flink:
+  pipeline-elements-experimental-flink:
     build:
-      context: ./streampipes-pipeline-elements-all-flink
+      context: ./streampipes-pipeline-elements-experimental-flink
       dockerfile: Dockerfile
       args:
         BASE_IMAGE: adoptopenjdk/openjdk8-openj9:alpine
-    image: incubator-streampipes_pipeline-elements-all-flink:release-validation
+    image: streampipes_pipeline-elements-experimental-flink:release-validation
     logging: *default-logging
     networks:
       spnet:

streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java

@@ -153,7 +153,7 @@ private List<EventProperty> getAppendProperties(String fileContents,
     CSVParser parser = getCsvParser(fileContents);
     List<EventProperty> propertiesToAppend = new ArrayList<>();
     List<CSVRecord> records = parser.getRecords();
-    if (records.size() > 0) {
+    if (!records.isEmpty()) {
       CSVRecord firstRecord = records.get(0);
       for (String column : columnsToInclude) {
         propertiesToAppend.add(makeEventProperty(column, firstRecord));
@@ -208,7 +208,7 @@ public void onInvocation(ProcessorParams parameters,
     } catch (IOException e) {
       throw new SpRuntimeException(e);
     }
-    if (this.columnMap.size() > 0) {
+    if (!this.columnMap.isEmpty()) {
       this.columnsToAppend = fieldsToAppend
           .stream()
           .map(c -> makeParser(c, this.columnMap.entrySet().stream().findFirst().get().getValue()))

streampipes-maven-plugin/README.md

@@ -37,7 +37,7 @@ By default, the goal runs in the `package` phase.
 ### Command line
 
 ```bash
-# Switch to a directory containing StreamPipes extensions and an Init class, e.g., streampipes-extensions-all-jvm
+# Switch to a directory containing StreamPipes extensions and an Init class, e.g., extensions-all-jvm
 
 mvn streampipes:extract-docs -DinitClass=org.apache.streampipes.extensions.all.jvm.AllExtensionsInit
 ```

streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java

@@ -43,6 +43,20 @@ public static List<FileMetadata> getAllFiles(String filetypes) {
     return filetypes != null ? filterFiletypes(allFiles, filetypes) : allFiles;
   }
 
+  public static File getFileByOriginalName(String originalName) throws IOException {
+    List<FileMetadata> allFiles = getFileMetadataStorage().getAllFileMetadataDescriptions();
+
+    var file = allFiles
+            .stream()
+            .filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName))
+            .findFirst();
+
+    if (file.isEmpty()){
+      throw new IOException("No file with original name '%s' found".formatted(originalName));
+    }
+    return new FileHandler().getFile(file.get().getInternalFilename());
+  }
+
   /**
    * Store a file in the internal file storage.
    * For csv files the bom is removed
@@ -119,7 +133,7 @@ private static FileMetadata makeFileMetadata(String user,
   }
 
   private static String makeInternalFilename(String filetype) {
-    return UUID.randomUUID().toString() + "." + filetype;
+    return UUID.randomUUID() + "." + filetype;
   }
 
   private static List<FileMetadata> filterFiletypes(List<FileMetadata> allFiles, String filetypes) {

streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java

@@ -22,13 +22,19 @@
 import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
 import org.apache.streampipes.rest.security.AuthConstants;
 
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.enums.ParameterIn;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import org.apache.http.HttpStatus;
 import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
 import org.glassfish.jersey.media.multipart.FormDataParam;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Component;
 
 import jakarta.ws.rs.Consumes;
 import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.DefaultValue;
 import jakarta.ws.rs.GET;
 import jakarta.ws.rs.POST;
 import jakarta.ws.rs.Path;
@@ -38,6 +44,7 @@
 import jakarta.ws.rs.core.MediaType;
 import jakarta.ws.rs.core.Response;
 
+import java.io.IOException;
 import java.io.InputStream;
 
 @Path("/v2/files")
@@ -76,8 +83,40 @@ public Response getFileInfo(@QueryParam("filetypes") String filetypes) {
   @GET
   @Path("/{filename}")
   @Produces(MediaType.APPLICATION_OCTET_STREAM)
-  public Response getFile(@PathParam("filename") String filename) {
+  @Operation(
+          summary = "Get file content by file name."
+                  + "If multiple files with the same name exist, only the first is returned."
+                  + "This can only be the case when the original file name is provided.", tags = {"Core", "Files"},
+          responses = {
+              @ApiResponse(
+                      responseCode = "" + HttpStatus.SC_OK,
+                      description = "File could be found and is returned"),
+              @ApiResponse(
+                      responseCode = "" + HttpStatus.SC_NOT_FOUND,
+                      description = "No file with the given file name could be found")
+          }
+  )
+  public Response getFile(
+          @Parameter(
+                  in = ParameterIn.PATH,
+                  description = "The name of the file to be retrieved",
+                  required = true
+          )
+          @PathParam("filename") String filename,
+          @Parameter(
+                  in = ParameterIn.QUERY,
+                  description = "Determines if the provided file name is the original file name "
+                          + "as per upload."
+          )
+          @QueryParam("isOriginalFilename") @DefaultValue("false") boolean isOriginalFilename
+  ) {
+    if (isOriginalFilename) {
+      try {
+        return ok(FileManager.getFileByOriginalName(filename));
+      } catch (IOException e) {
+        return notFound(filename);
+      }
+    }
     return ok(FileManager.getFile(filename));
   }
-
 }

ui/cypress/support/utils/FileManagementUtils.ts

@@ -39,4 +39,18 @@ export class FileManagementUtils {
         cy.dataCy('confirm-delete').click();
         cy.dataCy('delete').should('have.length', 0);
     }
+
+    public static downloadFile(expectedFilename: string) {
+        const downloadsFolder = Cypress.config('downloadsFolder');
+
+        // Go to StreamPipes file management
+        cy.visit('#/files');
+        // Check if file was uploaded
+        cy.dataCy('download').should('have.length', 1);
+
+        // Download file and verify it was downloaded
+        cy.dataCy('download').click();
+        cy.dataCy('download').should('have.length', 1);
+        cy.readFile(downloadsFolder + '/' + expectedFilename);
+    }
 }

ui/cypress/tests/fileManagement/testFileManagement.smoke.spec.ts

@@ -25,6 +25,7 @@ describe('Test File Management', () => {
 
     it('Perform Test', () => {
         FileManagementUtils.addFile('fileTest/test.csv');
+        FileManagementUtils.downloadFile('test.csv');
         FileManagementUtils.deleteFile();
     });
 });