From fbc3a3e8e420c2585059c6310d3193fe7988ded0 Mon Sep 17 00:00:00 2001 From: Tim Bossenmaier Date: Mon, 30 Oct 2023 14:50:24 +0100 Subject: [PATCH 1/5] feat: extend FileApi with the capability to query by the original file name --- .../streampipes/client/api/IFileApi.java | 4 +- .../streampipes/client/api/FileApi.java | 13 +++-- .../adapters/image/ZipFileImageIterator.java | 2 +- .../processor/chunker/ChunkerProcessor.java | 2 +- .../language/LanguageDetectionProcessor.java | 2 +- .../namefinder/NameFinderProcessor.java | 2 +- .../partofspeech/PartOfSpeechProcessor.java | 2 +- .../SentenceDetectionProcessor.java | 2 +- .../tokenizer/TokenizerProcessor.java | 2 +- .../CsvMetadataEnrichmentProcessor.java | 2 +- .../streampipes/manager/file/FileManager.java | 12 +++++ .../rest/impl/PipelineElementFile.java | 50 +++++++++++++++++-- 12 files changed, 77 insertions(+), 18 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java index 97d3da062a..8238fcd906 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java @@ -19,9 +19,9 @@ package org.apache.streampipes.client.api; public interface IFileApi { - byte[] getFileContent(String filename); + byte[] getFileContent(String filename, boolean isOriginalFileName); - String getFileContentAsString(String filename); + String getFileContentAsString(String filename, boolean isOriginalFileName); void writeToFile(String file, String fileLocation); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index 7d83d59f19..deae108280 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -22,6 +22,9 @@ import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.util.StreamPipesApiPath; +import java.util.HashMap; +import java.util.Map; + public class FileApi extends AbstractClientApi implements IFileApi { @@ -30,13 +33,15 @@ public FileApi(StreamPipesClientConfig clientConfig) { } @Override - public byte[] getFileContent(String filename) { - return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename), null).executeRequest(); + public byte[] getFileContent(String filename, boolean isOriginalFileName) { + return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename) + .withQueryParameters(Map.of("isOriginalFilename", String.valueOf(isOriginalFileName))), null) + .executeRequest(); } @Override - public String getFileContentAsString(String filename) { - return new String(getFileContent(filename)); + public String getFileContentAsString(String filename, boolean isOriginalFileName) { + return new String(getFileContent(filename, isOriginalFileName)); } @Override diff --git a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java index 73f1f6bab9..79b0f8114b 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java @@ -87,7 +87,7 @@ private static boolean isImage(String name) { private ZipInputStream fetchZipInputStream(String filename) { StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - byte[] result = client.fileApi().getFileContent(filename); + byte[] result = client.fileApi().getFileContent(filename, false); return new ZipInputStream(new ByteArrayInputStream(result)); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java index be8165468d..7ab5818be0 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java @@ -98,7 +98,7 @@ public void onInvocation(ProcessorParams parameters, this.tags = parameters.extractor().mappingPropertyValue(TAGS_FIELD_KEY); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename, false); InputStream modelIn = new ByteArrayInputStream(fileContent); ChunkerModel model; diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java index 862e615529..8210dd9e49 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java @@ -86,7 +86,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext context) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename,false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java index 0cbdc32d23..62abc061c5 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java @@ -84,7 +84,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(MODEL); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); loadModel(fileContent); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java index 5ef053c8fe..eec1e6be08 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java @@ -86,7 +86,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java index ae997b8cf8..83ae73a1ed 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java @@ -73,7 +73,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java index 2c68a393d6..dc76845fe0 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java @@ -79,7 +79,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java index 3bb61d2654..7059d82ed9 100644 --- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java +++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java @@ -186,7 +186,7 @@ private List getColumnNames(String fileContents, List columnsToI private String getFileContents(IParameterExtractor extractor) { String filename = extractor.selectedFilename(CSV_FILE_KEY); - return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename); + return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename, false); } private StreamPipesClient getStreamPipesClientInstance() { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java index 6ba0f85d11..ccde5c37d7 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java @@ -29,6 +29,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -43,6 +44,17 @@ public static List getAllFiles(String filetypes) { return filetypes != null ? filterFiletypes(allFiles, filetypes) : allFiles; } + public static File getFileByOriginalName(String originalName) throws IOException { + List allFiles = getFileMetadataStorage().getAllFileMetadataDescriptions(); + + var file = allFiles.stream().filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName)).findFirst(); + + if (file.isEmpty()){ + throw new IOException("No file with original name '%s' found".formatted(originalName)); + } + return new FileHandler().getFile(file.get().getInternalFilename()); + } + /** * Store a file in the internal file storage. * For csv files the bom is removed diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java index cf799a441b..6e83f16e4f 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java @@ -17,6 +17,12 @@ */ package org.apache.streampipes.rest.impl; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import jakarta.ws.rs.DefaultValue; +import org.apache.http.HttpStatus; import org.apache.streampipes.manager.file.FileManager; import org.apache.streampipes.model.file.FileMetadata; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; @@ -24,6 +30,8 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; @@ -38,6 +46,8 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import javax.imageio.IIOException; +import java.io.IOException; import java.io.InputStream; @Path("/v2/files") @@ -76,8 +86,40 @@ public Response getFileInfo(@QueryParam("filetypes") String filetypes) { @GET @Path("/{filename}") @Produces(MediaType.APPLICATION_OCTET_STREAM) - public Response getFile(@PathParam("filename") String filename) { - return ok(FileManager.getFile(filename)); - } - + @Operation( + summary = "Get file content by file name." + + "If multiple files with the same name exist, only the first is returned." + + "This can only be the case when the original file name is provided.", tags = {"Core", "Files"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "File could be found and is returned"), + @ApiResponse( + responseCode = "" + HttpStatus.SC_NOT_FOUND, + description = "No file with the given file name could be found") + } + ) + public Response getFile( + @Parameter( + in = ParameterIn.PATH, + description = "The name of the file to be retrieved", + required = true + ) + @PathParam("filename") String filename, + @Parameter( + in = ParameterIn.QUERY, + description = "Determines if the provided file name is the original file name" + + "as per upload." + ) + @QueryParam("isOriginalFilename") @DefaultValue("false") boolean isOriginalFilename + ) { + if (isOriginalFilename) { + try { + return ok(FileManager.getFileByOriginalName(filename)); + } catch (IOException e) { + return notFound(filename); + } + } + return ok(FileManager.getFile(filename)); + } } From 7bac1b7a1872f9d8af9ddec2f85c83f8eecbd0b2 Mon Sep 17 00:00:00 2001 From: Tim Bossenmaier Date: Mon, 30 Oct 2023 15:01:16 +0100 Subject: [PATCH 2/5] style: remove unused import --- .../src/main/java/org/apache/streampipes/client/api/FileApi.java | 1 - 1 file changed, 1 deletion(-) diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index deae108280..ff4364ae8d 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -22,7 +22,6 @@ import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.util.StreamPipesApiPath; -import java.util.HashMap; import java.util.Map; From 464609621aeaad4a080d434a18269a2d8b646958 Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 30 Oct 2023 15:34:33 +0100 Subject: [PATCH 3/5] style: fix checkstyle violations --- .../language/LanguageDetectionProcessor.java | 2 +- .../streampipes/manager/file/FileManager.java | 8 ++-- .../rest/impl/PipelineElementFile.java | 41 +++++++++---------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java index 8210dd9e49..19587b72c0 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java @@ -86,7 +86,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext context) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename,false); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java index ccde5c37d7..fa860de9f4 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java @@ -29,7 +29,6 @@ import java.io.InputStream; import java.util.Arrays; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -47,7 +46,10 @@ public static List getAllFiles(String filetypes) { public static File getFileByOriginalName(String originalName) throws IOException { List allFiles = getFileMetadataStorage().getAllFileMetadataDescriptions(); - var file = allFiles.stream().filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName)).findFirst(); + var file = allFiles + .stream() + .filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName)) + .findFirst(); if (file.isEmpty()){ throw new IOException("No file with original name '%s' found".formatted(originalName)); @@ -131,7 +133,7 @@ private static FileMetadata makeFileMetadata(String user, } private static String makeInternalFilename(String filetype) { - return UUID.randomUUID().toString() + "." + filetype; + return UUID.randomUUID() + "." + filetype; } private static List filterFiletypes(List allFiles, String filetypes) { diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java index 6e83f16e4f..2c74d94043 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java @@ -17,26 +17,24 @@ */ package org.apache.streampipes.rest.impl; -import io.swagger.v3.oas.annotations.Operation; -import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.enums.ParameterIn; -import io.swagger.v3.oas.annotations.responses.ApiResponse; -import jakarta.ws.rs.DefaultValue; -import org.apache.http.HttpStatus; import org.apache.streampipes.manager.file.FileManager; import org.apache.streampipes.model.file.FileMetadata; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; import org.apache.streampipes.rest.security.AuthConstants; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import org.apache.http.HttpStatus; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; @@ -46,7 +44,6 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -import javax.imageio.IIOException; import java.io.IOException; import java.io.InputStream; @@ -87,16 +84,16 @@ public Response getFileInfo(@QueryParam("filetypes") String filetypes) { @Path("/{filename}") @Produces(MediaType.APPLICATION_OCTET_STREAM) @Operation( - summary = "Get file content by file name." + - "If multiple files with the same name exist, only the first is returned." + - "This can only be the case when the original file name is provided.", tags = {"Core", "Files"}, + summary = "Get file content by file name." + + "If multiple files with the same name exist, only the first is returned." + + "This can only be the case when the original file name is provided.", tags = {"Core", "Files"}, responses = { - @ApiResponse( - responseCode = "" + HttpStatus.SC_OK, - description = "File could be found and is returned"), - @ApiResponse( - responseCode = "" + HttpStatus.SC_NOT_FOUND, - description = "No file with the given file name could be found") + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "File could be found and is returned"), + @ApiResponse( + responseCode = "" + HttpStatus.SC_NOT_FOUND, + description = "No file with the given file name could be found") } ) public Response getFile( @@ -108,8 +105,8 @@ public Response getFile( @PathParam("filename") String filename, @Parameter( in = ParameterIn.QUERY, - description = "Determines if the provided file name is the original file name" + - "as per upload." + description = "Determines if the provided file name is the original file name " + + "as per upload." ) @QueryParam("isOriginalFilename") @DefaultValue("false") boolean isOriginalFilename ) { @@ -120,6 +117,6 @@ public Response getFile( return notFound(filename); } } - return ok(FileManager.getFile(filename)); - } + return ok(FileManager.getFile(filename)); + } } From e63c25f52b75a99d6b6a6959012ffc3383eaee9e Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 13 Nov 2023 14:48:43 +0100 Subject: [PATCH 4/5] refactor: simplify file API --- .../java/org/apache/streampipes/client/api/IFileApi.java | 4 ++-- .../java/org/apache/streampipes/client/api/FileApi.java | 8 ++++++++ .../connect/adapters/image/ZipFileImageIterator.java | 2 +- .../jvm/processor/chunker/ChunkerProcessor.java | 2 +- .../processor/language/LanguageDetectionProcessor.java | 2 +- .../jvm/processor/namefinder/NameFinderProcessor.java | 2 +- .../jvm/processor/partofspeech/PartOfSpeechProcessor.java | 2 +- .../sentencedetection/SentenceDetectionProcessor.java | 2 +- .../jvm/processor/tokenizer/TokenizerProcessor.java | 2 +- .../csvmetadata/CsvMetadataEnrichmentProcessor.java | 6 +++--- 10 files changed, 20 insertions(+), 12 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java index 8238fcd906..97d3da062a 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java @@ -19,9 +19,9 @@ package org.apache.streampipes.client.api; public interface IFileApi { - byte[] getFileContent(String filename, boolean isOriginalFileName); + byte[] getFileContent(String filename); - String getFileContentAsString(String filename, boolean isOriginalFileName); + String getFileContentAsString(String filename); void writeToFile(String file, String fileLocation); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index ff4364ae8d..c6738e15e2 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -38,11 +38,19 @@ public byte[] getFileContent(String filename, boolean isOriginalFileName) { .executeRequest(); } + public byte[] getFileContent(String fileId) { + return getFileContent(fileId, false); + } + @Override public String getFileContentAsString(String filename, boolean isOriginalFileName) { return new String(getFileContent(filename, isOriginalFileName)); } + public String getFileContentAsString(String fileId) { + return getFileContentAsString(fileId, false); + } + @Override public void writeToFile(String file, String fileLocation) { new BinaryGetRequest(clientConfig, getBaseResourcePath(file), null) diff --git a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java index 79b0f8114b..73f1f6bab9 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java @@ -87,7 +87,7 @@ private static boolean isImage(String name) { private ZipInputStream fetchZipInputStream(String filename) { StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - byte[] result = client.fileApi().getFileContent(filename, false); + byte[] result = client.fileApi().getFileContent(filename); return new ZipInputStream(new ByteArrayInputStream(result)); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java index 7ab5818be0..be8165468d 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java @@ -98,7 +98,7 @@ public void onInvocation(ProcessorParams parameters, this.tags = parameters.extractor().mappingPropertyValue(TAGS_FIELD_KEY); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); InputStream modelIn = new ByteArrayInputStream(fileContent); ChunkerModel model; diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java index 19587b72c0..862e615529 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java @@ -86,7 +86,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext context) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java index 62abc061c5..0cbdc32d23 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java @@ -84,7 +84,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(MODEL); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); loadModel(fileContent); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java index eec1e6be08..5ef053c8fe 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java @@ -86,7 +86,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java index 83ae73a1ed..ae997b8cf8 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java @@ -73,7 +73,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java index dc76845fe0..2c68a393d6 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java @@ -79,7 +79,7 @@ public void onInvocation(ProcessorParams parameters, SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java index 7059d82ed9..be31a3473d 100644 --- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java +++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java @@ -153,7 +153,7 @@ private List getAppendProperties(String fileContents, CSVParser parser = getCsvParser(fileContents); List propertiesToAppend = new ArrayList<>(); List records = parser.getRecords(); - if (records.size() > 0) { + if (!records.isEmpty()) { CSVRecord firstRecord = records.get(0); for (String column : columnsToInclude) { propertiesToAppend.add(makeEventProperty(column, firstRecord)); @@ -186,7 +186,7 @@ private List getColumnNames(String fileContents, List columnsToI private String getFileContents(IParameterExtractor extractor) { String filename = extractor.selectedFilename(CSV_FILE_KEY); - return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename, false); + return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename); } private StreamPipesClient getStreamPipesClientInstance() { @@ -208,7 +208,7 @@ public void onInvocation(ProcessorParams parameters, } catch (IOException e) { throw new SpRuntimeException(e); } - if (this.columnMap.size() > 0) { + if (!this.columnMap.isEmpty()) { this.columnsToAppend = fieldsToAppend .stream() .map(c -> makeParser(c, this.columnMap.entrySet().stream().findFirst().get().getValue())) From c04d6af4006ec2f1ddc1bb336ee4a96fe7e1b5cd Mon Sep 17 00:00:00 2001 From: bossenti Date: Mon, 13 Nov 2023 15:02:39 +0100 Subject: [PATCH 5/5] style: fix wrong annotations --- .../main/java/org/apache/streampipes/client/api/FileApi.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index c6738e15e2..b6e8dcbf73 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -31,22 +31,21 @@ public FileApi(StreamPipesClientConfig clientConfig) { super(clientConfig); } - @Override public byte[] getFileContent(String filename, boolean isOriginalFileName) { return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename) .withQueryParameters(Map.of("isOriginalFilename", String.valueOf(isOriginalFileName))), null) .executeRequest(); } - + @Override public byte[] getFileContent(String fileId) { return getFileContent(fileId, false); } - @Override public String getFileContentAsString(String filename, boolean isOriginalFileName) { return new String(getFileContent(filename, isOriginalFileName)); } + @Override public String getFileContentAsString(String fileId) { return getFileContentAsString(fileId, false); }