From 556c4d89b55c83c1d36a103dd9ba868dab8ce2fa Mon Sep 17 00:00:00 2001 From: Dominik Riemer Date: Mon, 11 Sep 2023 09:06:17 +0200 Subject: [PATCH] Refactor streampipes-maven-plugin to auto-generate docs (#1907) * Refactor streampipes-maven-plugin to auto-generate docs * Add additional assets to image folder * Assign proper pipeline element type * Add github action to create docs * Upload multiple doc artifacts * Fix copy-paste leftover --- .github/workflows/extensions-docs.yml | 68 ++++++ .../streampipes-extensions-all-jvm/pom.xml | 23 ++ streampipes-maven-plugin/README.md | 88 +++++++ streampipes-maven-plugin/pom.xml | 18 +- .../streampipes/smp/CreateAssetMojo.java | 106 --------- .../smp/ExtractDocumentationMojo.java | 220 ++++++++---------- .../extractor/AssetModelItemExtractor.java | 90 ------- .../smp/extractor/ControllerExtractor.java | 57 ----- .../smp/extractor/ControllerFileFinder.java | 63 ----- .../smp/extractor/DockerImageExtractor.java | 77 ------ .../smp/extractor/ElementFinder.java | 39 ---- .../smp/extractor/ExtensionsFinder.java | 77 ++++++ .../smp/extractor/FieldReplacer.java | 58 ----- .../smp/extractor/LocalesExtractor.java | 40 ++-- .../PipelineElementTypeExtractor.java | 43 ---- .../ResourceDirectoryElementFinder.java | 73 ------ .../AdditionalAssetsResourceGenerator.java | 88 +++++++ .../smp/generator/AssetGenerator.java | 59 ----- .../smp/generator/DataJsonGenerator.java | 75 ------ .../DocumentationResourceGenerator.java | 63 +++++ .../smp/generator/IconResourceGenerator.java | 56 +++++ .../smp/generator/ImagePathReplacer.java | 11 +- .../PipelineElementGridGenerator.java | 157 ------------- .../PipelineElementOverviewGenerator.java | 53 ----- ...tGenerator.java => ResourceGenerator.java} | 30 ++- .../smp/generator/SidebarConfigGenerator.java | 72 +++++- .../streampipes/smp/model/AssetModel.java | 39 +--- .../smp/util/DuplicateRemover.java | 45 ---- .../extractor/TestAssetModelExtractor.java | 60 ----- .../TestPipelineElementOverviewGenerator.java | 44 ---- 30 files changed, 676 insertions(+), 1316 deletions(-) create mode 100644 .github/workflows/extensions-docs.yml create mode 100644 streampipes-maven-plugin/README.md delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java create mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java create mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java create mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java create mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementGridGenerator.java delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/PipelineElementOverviewGenerator.java rename streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/{OutputGenerator.java => ResourceGenerator.java} (55%) delete mode 100644 streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java delete mode 100644 streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java delete mode 100644 streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java diff --git a/.github/workflows/extensions-docs.yml b/.github/workflows/extensions-docs.yml new file mode 100644 index 0000000000..f43f8aef45 --- /dev/null +++ b/.github/workflows/extensions-docs.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "build-extensions-docs-and-share-as-artifact" + +on: + workflow_dispatch: + schedule: + - cron: "0 3 * * 6" # runs every saturday at 03:00:00 + +jobs: + extensions-docs-artifact-building: + runs-on: ubuntu-latest + steps: + - name: clone + uses: actions/checkout@v4 + + - name: create working branch & set GitHub config + run: | + git checkout -b build-extensions-docs-${{ github.run_id }} + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Build with Maven + run: mvn clean package + + - name: Publish Extensions docs as artifact + uses: actions/upload-artifact@v3 + with: + name: streampipes-extensions-docs + path: | + streampipes-extensions/streampipes-extensions-all-jvm/target/docs/pe/ + retention-days: 5 + + - name: Publish Extensions image assets as artifact + uses: actions/upload-artifact@v3 + with: + name: streampipes-extensions-docs-image-assets + path: | + streampipes-extensions/streampipes-extensions-all-jvm/target/docs/img/ + retention-days: 5 + + - name: Publish Extensions sidebar + uses: actions/upload-artifact@v3 + with: + name: streampipes-extensions-docs-sidebar + path: | + streampipes-extensions/streampipes-extensions-all-jvm/target/docs/sidebars.json + retention-days: 5 diff --git a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml index 3a057237cc..3b03c5940b 100644 --- a/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml +++ b/streampipes-extensions/streampipes-extensions-all-jvm/pom.xml @@ -78,6 +78,29 @@ + + org.apache.streampipes + streampipes-maven-plugin + 0.93.0-SNAPSHOT + + + org.apache.rocketmq + rocketmq-client-java + 5.0.2 + + + + org.apache.streampipes.extensions.all.jvm.AllExtensionsInit + + + + package + + extract-docs + + + + org.apache.maven.plugins maven-checkstyle-plugin diff --git a/streampipes-maven-plugin/README.md b/streampipes-maven-plugin/README.md new file mode 100644 index 0000000000..666f46c81d --- /dev/null +++ b/streampipes-maven-plugin/README.md @@ -0,0 +1,88 @@ + + +# Apache StreamPipes Maven Plugin + +## Description + +The Apache StreamPipes is an internal developer plugin which eases the generation of documentation for +adapters and pipeline elements. + +The plugin generates documentation files which can be integrated into the UI. + +## Usage + +The streampipes-maven-plugin can either be started from the command line or embedded into a project's pom. + +### Prerequisites + +The plugin must be started from a module which contains an `Init` class which inherits `ExtensionsModelSubmitter`. +By default, the goal runs in the `package` phase. + +### Command line + +```bash +# Switch to a directory containing StreamPipes extensions and an Init class, e.g., streampipes-extensions-all-jvm + +mvn streampipes:extract-docs -DinitClass=org.apache.streampipes.extensions.all.jvm.AllExtensionsInit +``` + +### Inclusion in pom file + +```xml + + + org.apache.streampipes + streampipes-maven-plugin + 0.93.0 + + + org.apache.rocketmq + rocketmq-client-java + 5.0.2 + + + + org.apache.streampipes.extensions.all.jvm.AllExtensionsInit + + + + package + + extract-docs + + + + +``` + +* Replace the `version` with the current development version. +* Replace the `initClass` with the module's init class. +* The dependency to RocketMQ is only needed for modules containing the RocketMQ sink and can be omitted otherwise. + +## Output + +The plugin creates a new folder `docs` in the module's `target` directory. + +The folder includes: + +* An `img` folder which has a subdirectory for each extension (named by the `appId`) containing the icon. +* A `pe` folder which has a subdirectory for each extension (named by the `appId`) containing the `documentation.md` + file, which has been rewritten to match the requirements of the Docusaurus Markdown parser. +* An updated `sidebar.json` file containing the sidebar, which is downloaded from the `streampipes-website` repo on + branch `dev` and updated with the current set of extensions. diff --git a/streampipes-maven-plugin/pom.xml b/streampipes-maven-plugin/pom.xml index 17d70dae08..e994f01928 100644 --- a/streampipes-maven-plugin/pom.xml +++ b/streampipes-maven-plugin/pom.xml @@ -118,27 +118,19 @@ org.codehaus.plexus plexus-utils - - org.jboss.forge.roaster - roaster-api - - - org.jboss.forge.roaster - roaster-jdt - runtime - com.google.code.gson gson - - com.j2html - j2html - org.yaml snakeyaml + + org.apache.streampipes + streampipes-service-extensions + 0.93.0-SNAPSHOT + diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java deleted file mode 100644 index d5567453d8..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/CreateAssetMojo.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp; - - -import org.apache.streampipes.smp.extractor.ControllerFileFinder; -import org.apache.streampipes.smp.generator.AssetGenerator; -import org.apache.streampipes.smp.model.AssetModel; -import org.apache.streampipes.smp.util.DuplicateRemover; - -import org.apache.maven.execution.MavenSession; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.maven.plugins.annotations.Component; -import org.apache.maven.plugins.annotations.LifecyclePhase; -import org.apache.maven.plugins.annotations.Mojo; -import org.apache.maven.plugins.annotations.Parameter; -import org.apache.maven.project.MavenProject; -import org.codehaus.plexus.components.interactivity.Prompter; -import org.codehaus.plexus.components.interactivity.PrompterException; - -import java.io.File; -import java.util.List; - -/** - * Goal which creates an asset directory. - */ -@Mojo(name = "create-assets", defaultPhase = LifecyclePhase.NONE) -public class CreateAssetMojo - extends AbstractMojo { - - private static final String APP_ID_PROPERTY = "appId"; - private static final String GENERATE_ALL_PROPERTY = "all"; - @Parameter(defaultValue = "${session}") - private MavenSession session; - @Parameter(defaultValue = "${project}", readonly = true) - private MavenProject project; - @Component - private Prompter prompter; - - public void execute() throws MojoExecutionException { - - String appId = this.session.getExecutionProperties().getProperty(APP_ID_PROPERTY); - String addAll = this.session.getExecutionProperties().getProperty(GENERATE_ALL_PROPERTY); - File baseDir = this.project.getBasedir(); - - if (appId != null) { - getLog().info("Generating asset directory for element " + appId); - AssetModel assetModel = new AssetModel(appId, "Pipeline Element Name", "Pipeline Element " - + "Description"); - new AssetGenerator(baseDir.getAbsolutePath(), assetModel); - } - - if (addAll != null) { - List sourceRoots = this.project.getCompileSourceRoots(); - getLog().info("Generating asset directories for source root " + sourceRoots.get(0)); - - getLog().info("Finding controllers..."); - List allAssetModels = new ControllerFileFinder(getLog(), baseDir.getAbsolutePath(), - sourceRoots.get(0), - "**/*Controller.java").makeAssetModels(); - - getLog().info("Checking for already existing asset directories..."); - - List filteredModels = - new DuplicateRemover(baseDir.getAbsolutePath(), allAssetModels).removeAlreadyExisting(); - - try { - String proceed = prompter.prompt(makeProceedText(filteredModels)); - if (proceed.equals("Y")) { - filteredModels.forEach(am -> new AssetGenerator(baseDir.getAbsolutePath(), am) - .genreateAssetDirectoryAndContents()); - } - } catch (PrompterException e) { - e.printStackTrace(); - } - } - } - - private String makeProceedText(List assetModels) { - StringBuilder builder = new StringBuilder(); - builder.append("The following asset directories will be created: "); - - assetModels.forEach(f -> builder.append(f.toString())); - - builder.append("Proceed? [Y][N]"); - return builder.toString(); - } - -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java index 1942bec18b..a5b68bfaae 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/ExtractDocumentationMojo.java @@ -18,49 +18,49 @@ package org.apache.streampipes.smp; -import org.apache.streampipes.smp.extractor.ControllerFileFinder; -import org.apache.streampipes.smp.extractor.ResourceDirectoryElementFinder; -import org.apache.streampipes.smp.generator.DataJsonGenerator; -import org.apache.streampipes.smp.generator.ImagePathReplacer; -import org.apache.streampipes.smp.generator.MarkdownHeaderGenerator; -import org.apache.streampipes.smp.generator.MarkdownTitleRemover; -import org.apache.streampipes.smp.generator.PipelineElementOverviewGenerator; +import org.apache.streampipes.smp.extractor.ExtensionsFinder; +import org.apache.streampipes.smp.extractor.LocalesExtractor; +import org.apache.streampipes.smp.generator.AdditionalAssetsResourceGenerator; +import org.apache.streampipes.smp.generator.DocumentationResourceGenerator; +import org.apache.streampipes.smp.generator.IconResourceGenerator; import org.apache.streampipes.smp.generator.SidebarConfigGenerator; import org.apache.streampipes.smp.model.AssetModel; -import org.apache.streampipes.smp.util.DirectoryManager; -import org.apache.streampipes.smp.util.Utils; +import com.google.common.base.Charsets; import org.apache.commons.io.FileUtils; +import org.apache.maven.artifact.DependencyResolutionRequiredException; import org.apache.maven.execution.MavenSession; import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; import org.apache.maven.plugin.MojoFailureException; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; import org.apache.maven.plugins.annotations.Parameter; +import org.apache.maven.plugins.annotations.ResolutionScope; import org.apache.maven.project.MavenProject; import java.io.File; import java.io.IOException; -import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; /** * goal which extracts pipeline element documentations for the StreamPipes website + documentation */ -@Mojo(name = "extract-docs", defaultPhase = LifecyclePhase.NONE, aggregator = true) +@Mojo( + name = "extract-docs", + defaultPhase = LifecyclePhase.PACKAGE, + requiresDependencyResolution = ResolutionScope.TEST, + requiresDependencyCollection = ResolutionScope.TEST) public class ExtractDocumentationMojo extends AbstractMojo { private static final String DOCS_ROOT_FOLDER = "docs"; - private static final String DOCS_FOLDER = "docs-documentation"; - private static final String DOCS_WEBSITE_FOLDER = "docs-website"; + private static final String DOCS_PE_FOLDER = "pe"; private static final String IMG_FOLDER = "img"; private static final String SP_IGNORE_FILENAME = ".spignore"; @@ -70,127 +70,97 @@ public class ExtractDocumentationMojo extends AbstractMojo { @Parameter(defaultValue = "${project}", readonly = true) private MavenProject project; + @Parameter(name = "initClass", required = true) + private String initClass; + @Override - public void execute() throws MojoExecutionException, MojoFailureException { - String targetDir = this.session.getExecutionRootDirectory() + File.separator + "target"; - String spIgnoreFile = + public void execute() throws MojoFailureException { + var log = getLog(); + var targetDir = this.session.getExecutionRootDirectory() + File.separator + "target"; + var spIgnoreFile = this.session.getExecutionRootDirectory() + File.separator + SP_IGNORE_FILENAME; - Path docsBasePath = Paths.get(targetDir, DOCS_ROOT_FOLDER); - - List projects = session.getProjects(); - List documentedPipelineElements = new ArrayList<>(); - List pipelineElementsToExclude = new ArrayList<>(); - - if (Files.exists(Paths.get(spIgnoreFile))) { - try { - pipelineElementsToExclude = Files.readAllLines(Paths.get(spIgnoreFile)); - } catch (IOException e) { - e.printStackTrace(); - } - } + var docsBasePath = Paths.get(targetDir, DOCS_ROOT_FOLDER); - for (MavenProject currentModule : projects) { - File baseDir = currentModule.getBasedir(); - List sourceRoots = currentModule.getCompileSourceRoots(); - List allAssetModels = new ArrayList<>(); - //String targetDir = this.project.getModel().getBuild().getDirectory(); - - //String dockerImageName = - // new DockerImageExtractor(baseDir.getAbsolutePath()).extractImageName(); - - if (currentModule.getName().equals("streampipes-connect-adapters")) { - System.out.println("Opening adapters"); - allAssetModels = new ResourceDirectoryElementFinder(sourceRoots.get(0), getLog(), baseDir.getAbsolutePath()) - .makeAssetModels(); - allAssetModels.forEach(am -> System.out.println(am.getAppId())); - } else { - allAssetModels = new ControllerFileFinder(getLog(), - baseDir.getAbsolutePath(), sourceRoots.get(0), - "**/*Controller.java").makeAssetModels(); - } - - allAssetModels.forEach(am -> { - am.setBaseDir(baseDir.getAbsolutePath()); - am.setModuleName(currentModule.getName()); - //am.setContainerName(dockerImageName); - }); - - List finalPipelineElementsToExclude = pipelineElementsToExclude; - documentedPipelineElements.addAll( - allAssetModels - .stream() - .filter(am -> finalPipelineElementsToExclude.stream().noneMatch(pe -> pe.equals(am.getAppId()))) - .collect(Collectors.toList())); - } try { - Collections.sort(documentedPipelineElements); - documentedPipelineElements.forEach(am -> System.out.println( - am.getAppId() + ", " + am.getPipelineElementName() + ", " + am.getPipelineElementDescription())); - - for (AssetModel assetModel : documentedPipelineElements) { - Path docsPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, DOCS_FOLDER, - "pe"); - Path docsWebsitePath = Paths.get(targetDir, DOCS_ROOT_FOLDER, DOCS_WEBSITE_FOLDER, - assetModel.getAppId()); - Path imgPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, IMG_FOLDER, - assetModel.getAppId()); - DirectoryManager.createIfNotExists(docsPath); - DirectoryManager.createIfNotExists(imgPath); - - Boolean iconExists = Files.exists(Utils.makeResourcePath(assetModel.getBaseDir(), - assetModel.getAppId()).resolve("icon.png")); - - if (iconExists) { - Files.copy(Utils.makeResourcePath(assetModel.getBaseDir(), - assetModel.getAppId()).resolve("icon.png") - , imgPath.resolve("icon.png")); - } else { - ClassLoader classLoader = this.getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream("placeholder-icon.png"); - Files.copy(inputStream, imgPath.resolve("icon.png")); + List pipelineElementsToExclude = new ArrayList<>(); + + var path = Paths.get(spIgnoreFile); + if (Files.exists(path)) { + try { + pipelineElementsToExclude = Files.readAllLines(path); + } catch (IOException e) { + log.error(String.format("Could not open .spignore file at path %s", path)); } - - String originalDocumentationFileContents = - FileUtils.readFileToString(Utils.makeResourcePath(assetModel.getBaseDir(), - assetModel.getAppId()).resolve("documentation.md").toFile()); - - // modify docs for documentation page - String documentationFileContents = - new MarkdownTitleRemover(originalDocumentationFileContents).removeTitle(); - - documentationFileContents = - new MarkdownHeaderGenerator(assetModel, documentationFileContents).createHeaders(); - - documentationFileContents = new ImagePathReplacer(documentationFileContents, - assetModel.getAppId()).replaceContentForDocs(); - - FileUtils.writeStringToFile(docsPath.resolve(assetModel.getAppId() + ".md").toFile(), - documentationFileContents); - } - Boolean existsOverviewFile = Files.exists(docsBasePath.resolve("pipeline-elements.md")); - if (!existsOverviewFile) { - AssetModel assetModel = new AssetModel("pipeline-elements", "Overview", ""); - String header = new MarkdownHeaderGenerator(assetModel, "").createHeaders(); - FileUtils.writeStringToFile(docsBasePath.resolve("pipeline-elements.md").toFile(), header); + var loader = getClassLoader(project); + var finalPipelineElementsToExclude = pipelineElementsToExclude; + var localesExtractor = new LocalesExtractor(log, loader); + var extensionsElements = new ExtensionsFinder(loader, initClass) + .findExtensions() + .stream() + .filter(am -> !finalPipelineElementsToExclude.contains(am.getAppId())) + .peek(localesExtractor::applyLocales) + .sorted(AssetModel::compareTo) + .toList(); + + log.info(String.format("Will generate documentation for %s resources", extensionsElements.size())); + + for (AssetModel extensionsElement : extensionsElements) { + try { + var imgPath = Paths.get(targetDir, DOCS_ROOT_FOLDER, IMG_FOLDER, extensionsElement.getAppId()); + new IconResourceGenerator(loader, extensionsElement, imgPath).generate(); + + var docsPath = Paths.get( + targetDir, + DOCS_ROOT_FOLDER, + DOCS_PE_FOLDER, + extensionsElement.getAppId()); + new DocumentationResourceGenerator(loader, extensionsElement, docsPath).generate(); + + log.info( + String.format( + "Generated documentation for %s (%s, %s)", + extensionsElement.getAppId(), + extensionsElement.getPipelineElementName(), + extensionsElement.getPipelineElementDescription())); + + new AdditionalAssetsResourceGenerator(log, loader, extensionsElement, imgPath).generate(); + } catch (IOException e) { + log.warn( + String.format( + "Could not generate documentation for %s (%s, %s)", + extensionsElement.getAppId(), + extensionsElement.getPipelineElementName(), + extensionsElement.getPipelineElementDescription()), e); + } } - String pipelineElementOverviewContent = - new PipelineElementOverviewGenerator(documentedPipelineElements).generate(); - FileUtils.writeStringToFile(docsBasePath.resolve("pipeline-elements.md").toFile(), - pipelineElementOverviewContent, true); - - + var sidebarConfig = new SidebarConfigGenerator(log, extensionsElements).generate(); + log.info("Writing sidebar config"); FileUtils.writeStringToFile(docsBasePath.resolve("sidebars.json").toFile(), - new SidebarConfigGenerator(documentedPipelineElements).generate()); + sidebarConfig, Charsets.UTF_8); - FileUtils.writeStringToFile(docsBasePath.resolve("_data.json").toFile(), - new DataJsonGenerator(documentedPipelineElements).generate()); + } catch (IOException | DependencyResolutionRequiredException | ClassNotFoundException + | IllegalAccessException | InstantiationException e) { + throw new MojoFailureException( + "Could not generate documentation - please check the initClass for available extensions", e); + } + } + private ClassLoader getClassLoader(MavenProject project) + throws DependencyResolutionRequiredException, MalformedURLException { + List runtimeUrls = new ArrayList<>(); + List runtimeClasspathElements = project.getCompileClasspathElements(); - } catch (IOException e) { - e.printStackTrace(); + for (String element : runtimeClasspathElements) { + runtimeUrls.add(new File(element).toURI().toURL()); } + + URL[] array = new URL[runtimeUrls.size()]; + array = runtimeUrls.toArray(array); + + return new URLClassLoader(array, + Thread.currentThread().getContextClassLoader()); } } diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java deleted file mode 100644 index 74c6873dd5..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/AssetModelItemExtractor.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.constants.PeType; -import org.apache.streampipes.smp.model.AssetModel; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class AssetModelItemExtractor { - - private static final String REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\.create\\" - + "((\".*?\"),.*?(\".*?\"),.*?(\".*?\")\\)"; - - private static final String ONLY_APP_ID_REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\" - + ".create\\" - + "((\".*?\")\\)"; - - private static final Pattern pattern = Pattern.compile(REGEX); - private static final Pattern onlyAppIdPattern = Pattern.compile(ONLY_APP_ID_REGEX); - - private String declareModelContent; - - public AssetModelItemExtractor(String declareModelContent) { - this.declareModelContent = declareModelContent - .replaceAll("\n", ""); - - } - - - public AssetModel extractAssetItem() throws IllegalArgumentException { - AssetModel assetModel = new AssetModel(); - Matcher matcher = pattern.matcher(this.declareModelContent); - Boolean matches = false; - -// while (matcher.find()) { -// matches = true; -// PeType peType = new PipelineElementTypeExtractor(matcher.group(1)).extractType(); -// String appId = removeQuotes(matcher.group(2)); -// String pipelineElementName = removeQuotes(matcher.group(3)); -// String pipelineElementDescription = removeQuotes(matcher.group(4)); -// -// assetModel.setPeType(peType); -// assetModel.setAppId(appId); -// assetModel.setPipelineElementName(pipelineElementName); -// assetModel.setPipelineElementDescription(pipelineElementDescription); -// -// } - - if (matches) { - return assetModel; - } else { - Matcher appIdMatcher = onlyAppIdPattern.matcher(this.declareModelContent); - while (appIdMatcher.find()) { - matches = true; - PeType peType = new PipelineElementTypeExtractor(appIdMatcher.group(1)).extractType(); - assetModel.setAppId(removeQuotes(appIdMatcher.group(2))); - assetModel.setPeType(peType); - } - if (!matches) { - throw new IllegalArgumentException("Could not extract appId"); - } else { - return assetModel; - } - } - } - - private String removeQuotes(String text) { - return text - .replaceAll("\" \\+ \"", "") - .replaceAll("\"", ""); - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java deleted file mode 100644 index 7f2c0436a2..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerExtractor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.model.AssetModel; - -import org.jboss.forge.roaster.Roaster; -import org.jboss.forge.roaster.model.source.JavaClassSource; - -import java.nio.file.Files; -import java.nio.file.Paths; - -public class ControllerExtractor { - - private static String declareModelMethod = "declareModel"; - private String filename; - private String baseDir; - - public ControllerExtractor(String baseDir, String filename) { - this.filename = filename; - this.baseDir = baseDir; - } - - public AssetModel extractControllerDetails() throws Exception { - String fileContents = new String(Files.readAllBytes(Paths.get(filename))); - JavaClassSource clazz = Roaster.parse(JavaClassSource.class, fileContents); - if (clazz.hasMethodSignature(declareModelMethod)) { - String declareMethodContent = (clazz.getMethod(declareModelMethod).getBody()); - declareMethodContent = - new FieldReplacer(clazz, declareMethodContent).replaceDeclareModelContent(); - AssetModel assetModel = new AssetModelItemExtractor(declareMethodContent).extractAssetItem(); - if (assetModel.getPipelineElementName() == null) { - assetModel = new LocalesExtractor(baseDir, assetModel).extract(); - } - return assetModel; - - } else { - throw new Exception("Not a valid controller class or could not read id elements"); - } - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java deleted file mode 100644 index d6d5dae104..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ControllerFileFinder.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.model.AssetModel; - -import org.apache.maven.plugin.logging.Log; -import org.apache.maven.shared.model.fileset.FileSet; -import org.apache.maven.shared.model.fileset.util.FileSetManager; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -public class ControllerFileFinder extends ElementFinder { - - private String filterPattern; - - public ControllerFileFinder(Log log, String baseDir, String sourceRoot, String filterPattern) { - super(sourceRoot, log, baseDir); - this.filterPattern = filterPattern; - } - - public String[] findFiles() { - FileSet fileSet = new FileSet(); - fileSet.setDirectory(sourceRoot); - fileSet.addInclude(filterPattern); - FileSetManager fileSetManager = new FileSetManager(); - - return fileSetManager.getIncludedFiles(fileSet); - } - - @Override - public List makeAssetModels() { - List allAssetModels = new ArrayList<>(); - for (String file : findFiles()) { - try { - allAssetModels.add( - new ControllerExtractor(baseDir, sourceRoot + File.separator + file).extractControllerDetails()); - } catch (Exception e) { - log.error(e.getMessage()); - log.info("Could not parse file " + file + ", skipping..."); - } - } - return allAssetModels; - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java deleted file mode 100644 index a62d707954..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/DockerImageExtractor.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.yaml.snakeyaml.Yaml; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; - -public class DockerImageExtractor { - - private static final String Deployment = "deployment"; - private static final String DockerCompose = "docker-compose.yml"; - private static final String Image = "image"; - private static final String Services = "services"; - private static final String Colon = ":"; - private static final String ImagePrefix = "${SP_PE_DOCKER_REGISTRY}/streampipes" - + "/streampipes" - + "-pipeline-elements/"; - - private String baseDir; - - public DockerImageExtractor(String baseDir) { - this.baseDir = baseDir; - } - - public String extractImageName() { - Path dockerComposePath = Paths.get(baseDir, Deployment, DockerCompose); - - Yaml yaml = new Yaml(); - try { - Map fileContents = - yaml.load(new String(Files.readAllBytes(dockerComposePath))); - - return extractNameFromYaml(fileContents); - - } catch (IOException e) { - e.printStackTrace(); - return ""; - } - } - - public String extractNameFromYaml(Map fileContents) { - Map services = (Map) fileContents.get(Services); - AtomicReference imageName = new AtomicReference<>(""); - if (services.size() > 0) { - services.forEach((key, value) -> imageName.set(parseValue(((Map) value).get(Image).toString()))); - } - - return imageName.get(); - } - - private String parseValue(String imageName) { - return imageName.replace(ImagePrefix, "").split(Colon)[0]; - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java deleted file mode 100644 index 55818c8d15..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ElementFinder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.model.AssetModel; - -import org.apache.maven.plugin.logging.Log; - -import java.util.List; - -public abstract class ElementFinder { - - protected String sourceRoot; - protected Log log; - protected String baseDir; - - public ElementFinder(String sourceRoot, Log log, String baseDir) { - this.sourceRoot = sourceRoot; - this.log = log; - this.baseDir = baseDir; - } - - public abstract List makeAssetModels(); -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java new file mode 100644 index 0000000000..b36abb767b --- /dev/null +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ExtensionsFinder.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.smp.extractor; + +import org.apache.streampipes.extensions.api.pe.IStreamPipesPipelineElement; +import org.apache.streampipes.extensions.api.pe.config.IDataProcessorConfiguration; +import org.apache.streampipes.extensions.api.pe.config.IDataSinkConfiguration; +import org.apache.streampipes.extensions.api.pe.config.IPipelineElementConfiguration; +import org.apache.streampipes.extensions.management.model.SpServiceDefinition; +import org.apache.streampipes.service.extensions.ExtensionsModelSubmitter; +import org.apache.streampipes.smp.constants.PeType; +import org.apache.streampipes.smp.model.AssetModel; + +import org.apache.maven.artifact.DependencyResolutionRequiredException; + +import java.net.MalformedURLException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class ExtensionsFinder { + + private final ClassLoader loader; + private final String initClass; + + public ExtensionsFinder(ClassLoader loader, + String initClass) { + this.loader = loader; + this.initClass = initClass; + } + + public List findExtensions() + throws MalformedURLException, DependencyResolutionRequiredException, ClassNotFoundException, + InstantiationException, IllegalAccessException { + var extensions = new ArrayList(); + var serviceDef = ((ExtensionsModelSubmitter) loader.loadClass(initClass).newInstance()).provideServiceDefinition(); + + extensions.addAll(findAdapters(serviceDef)); + extensions.addAll(findPipelineElements(serviceDef, IDataProcessorConfiguration.class, PeType.PROCESSOR)); + extensions.addAll(findPipelineElements(serviceDef, IDataSinkConfiguration.class, PeType.SINK)); + + return extensions; + } + + private List findPipelineElements(SpServiceDefinition serviceDef, + Class> configType, + PeType peType) { + return serviceDef.getDeclarers() + .stream() + .map(IStreamPipesPipelineElement::declareConfig) + .filter(configType::isInstance) + .map(config -> new AssetModel(config.getDescription().getAppId(), peType)).toList(); + } + + private Collection findAdapters(SpServiceDefinition serviceDef) { + return serviceDef.getAdapters().stream().map(adapter -> { + var config = adapter.declareConfig(); + return new AssetModel(config.getAdapterDescription().getAppId(), PeType.ADAPTER); + }).toList(); + } +} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java deleted file mode 100644 index a661820320..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/FieldReplacer.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.jboss.forge.roaster.model.source.JavaClassSource; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class FieldReplacer { - - private static final String CHECK_FIELD_REGEX = "(DataSinkBuilder|ProcessingElementBuilder)\\" - + ".create\\" - + "((.*?),(.*?)\\)"; - private static final Pattern checkFieldPattern = Pattern.compile(CHECK_FIELD_REGEX); - private String declareModelContent; - private JavaClassSource source; - - - public FieldReplacer(JavaClassSource source, String declareModelContent) { - this.source = source; - this.declareModelContent = declareModelContent; - } - - public String replaceDeclareModelContent() { - Matcher matcher = checkFieldPattern.matcher(this.declareModelContent); - while (matcher.find()) { - String match = matcher.group(2); - if (!match.startsWith("\"")) { - declareModelContent = declareModelContent.replaceFirst(match, - getFieldValue(match)); - } - } - return declareModelContent; - } - - private String getFieldValue(String fieldName) { - return "\"" - + source.getField(fieldName).getStringInitializer() - + "\""; - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java index ec50554f75..7688c2e64f 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/LocalesExtractor.java @@ -19,41 +19,47 @@ package org.apache.streampipes.smp.extractor; import org.apache.streampipes.smp.model.AssetModel; -import org.apache.streampipes.smp.util.Utils; -import java.io.FileReader; +import org.apache.maven.plugin.logging.Log; + import java.io.IOException; +import java.io.InputStream; import java.util.Properties; public class LocalesExtractor { + public static final String LOCALES_FILE_EN = "strings.en"; + private static final String TITLE = ".title"; private static final String DESCRIPTION = ".description"; - private AssetModel assetModel; - private String baseDir; - public LocalesExtractor(String baseDir, AssetModel assetModel) { - this.baseDir = baseDir; - this.assetModel = assetModel; - } + private final Log log; + private final ClassLoader loader; - public AssetModel extract() throws IOException { - String localesPath = Utils.makeLocalesPath(baseDir, assetModel.getAppId()); - Properties props = loadProperties(localesPath); - - assetModel.setPipelineElementName(extractKey(props, assetModel, TITLE)); - assetModel.setPipelineElementDescription(extractKey(props, assetModel, DESCRIPTION)); + public LocalesExtractor(Log log, + ClassLoader loader) { + this.log = log; + this.loader = loader; + } - return assetModel; + public void applyLocales(AssetModel assetModel) { + var localeFile = loader.getResourceAsStream(assetModel.getAppId() + "/" + LOCALES_FILE_EN); + try { + var props = loadProperties(localeFile); + assetModel.setPipelineElementName(extractKey(props, assetModel, TITLE)); + assetModel.setPipelineElementDescription(extractKey(props, assetModel, DESCRIPTION)); + } catch (IOException e) { + log.warn(String.format("Could not load properties file %s", assetModel.getAppId()), e); + } } private String extractKey(Properties props, AssetModel assetModel, String appendix) { return props.getProperty(assetModel.getAppId() + appendix); } - private Properties loadProperties(String localesPath) throws IOException { + private Properties loadProperties(InputStream stream) throws IOException { Properties props = new Properties(); - props.load(new FileReader(localesPath)); + props.load(stream); return props; } } diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java deleted file mode 100644 index df5322fba8..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/PipelineElementTypeExtractor.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.constants.PeType; - -public class PipelineElementTypeExtractor { - - private static final String ProcessorString = "ProcessingElementBuilder"; - private static final String SinkString = "DataSinkBuilder"; - - private String typeString; - - public PipelineElementTypeExtractor(String typeString) { - this.typeString = typeString; - } - - public PeType extractType() throws IllegalArgumentException { - if (typeString.equals(ProcessorString)) { - return PeType.PROCESSOR; - } else if (typeString.equals(SinkString)) { - return PeType.SINK; - } else { - throw new IllegalArgumentException("Unknown pipeline element type " + typeString); - } - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java deleted file mode 100644 index 1151b78f5a..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/extractor/ResourceDirectoryElementFinder.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.constants.PeType; -import org.apache.streampipes.smp.model.AssetModel; -import org.apache.streampipes.smp.util.Utils; - -import org.apache.maven.plugin.logging.Log; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -public class ResourceDirectoryElementFinder extends ElementFinder { - - - public ResourceDirectoryElementFinder(String sourceRoot, Log log, String baseDir) { - super(sourceRoot, log, baseDir); - System.out.println(sourceRoot); - System.out.println(baseDir); - } - - @Override - public List makeAssetModels() { - List adapterModels = new ArrayList<>(); - Path resourceDirPath = Utils.makeResourcePath(baseDir); - File[] resourceDirs = resourceDirPath.toFile().listFiles(File::isDirectory); - if (resourceDirs != null) { - List appDirs = Arrays.stream(resourceDirs) - .filter(rd -> rd.getName().startsWith("org.apache.streampipes")) - .collect(Collectors.toList()); - - appDirs.forEach(ap -> { - try { - adapterModels.add(extractModel(baseDir, ap)); - } catch (IOException e) { - e.printStackTrace(); - } - }); - } - - return adapterModels; - } - - private AssetModel extractModel(String sourceRoot, File resourceDir) throws IOException { - AssetModel model = new AssetModel(); - - model.setPeType(PeType.ADAPTER); - model.setAppId(resourceDir.getName()); - return new LocalesExtractor(sourceRoot, model).extract(); - } - -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java new file mode 100644 index 0000000000..0122c4233c --- /dev/null +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AdditionalAssetsResourceGenerator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.smp.generator; + +import org.apache.streampipes.smp.extractor.LocalesExtractor; +import org.apache.streampipes.smp.model.AssetModel; + +import org.apache.maven.plugin.logging.Log; + +import java.io.IOException; +import java.io.InputStream; +import java.net.JarURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; + +public class AdditionalAssetsResourceGenerator extends ResourceGenerator { + + private static final String Slash = "/"; + + private final Log log; + + public AdditionalAssetsResourceGenerator(Log log, + ClassLoader loader, + AssetModel extensionsElement, + Path targetPath) { + super(loader, extensionsElement, targetPath); + this.log = log; + } + + @Override + public void generate() throws IOException { + var resourceName = extensionsElement.getAppId() + Slash; + var resources = loader.getResources(resourceName); + while (resources.hasMoreElements()) { + URL resourceUrl = resources.nextElement(); + if (resourceUrl.getProtocol().equals("jar")) { + JarURLConnection jarConnection = (JarURLConnection) resourceUrl.openConnection(); + try (JarFile jarFile = jarConnection.getJarFile()) { + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + if (!entry.isDirectory() && entry.getName().startsWith(resourceName)) { + var filename = entry.getName(); + if (!isDefaultResource(filename)) { + try (InputStream inputStream = jarFile.getInputStream(entry)) { + Files.createDirectories(targetPath); + filename = filename.substring(filename.lastIndexOf(Slash) + 1); + Files.copy(inputStream, targetPath.resolve(filename), StandardCopyOption.REPLACE_EXISTING); + log.info(String.format("Additional resource file %s copied to %s", filename, targetPath)); + } + } + } + } + } + } + } + } + + private boolean isDefaultResource(String resourceName) { + var defaultResources = List.of( + IconResourceGenerator.ICON_NAME, + DocumentationResourceGenerator.DOCUMENTATION_FILE_NAME, + LocalesExtractor.LOCALES_FILE_EN); + return defaultResources.stream().anyMatch(resourceName::contains); + } +} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java deleted file mode 100644 index 7f6bd8c1b3..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/AssetGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.generator; - -import org.apache.streampipes.smp.model.AssetModel; -import org.apache.streampipes.smp.parser.DocumentationParser; -import org.apache.streampipes.smp.util.Utils; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; - -public class AssetGenerator { - - private static final String DOCUMENTATION_FILE = "documentation.md"; - private AssetModel assetModel; - private String baseDir; - - public AssetGenerator(String baseDir, AssetModel assetModel) { - this.assetModel = assetModel; - this.baseDir = baseDir; - } - - public void genreateAssetDirectoryAndContents() { - ClassLoader classLoader = this.getClass().getClassLoader(); - String resourcePath = Utils.makePath(baseDir, assetModel.getAppId()); - - Boolean dir = new File(resourcePath).mkdirs(); - InputStream inputStream = classLoader.getResourceAsStream(DOCUMENTATION_FILE); - try { - String content = - new DocumentationParser(assetModel) - .parseAndStoreDocumentation(IOUtils.toString(inputStream)); - FileUtils.writeStringToFile(new File(resourcePath + File.separator + DOCUMENTATION_FILE), - content); - } catch (IOException e) { - e.printStackTrace(); - } - } -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java deleted file mode 100644 index f992398413..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DataJsonGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.generator; - -import org.apache.streampipes.smp.constants.PeType; -import org.apache.streampipes.smp.model.AssetModel; - -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; - -import java.util.List; - -public class DataJsonGenerator extends OutputGenerator { - - public DataJsonGenerator(List assetModels) { - super(assetModels); - } - - @Override - public String generate() { - JsonObject jsonObject = new JsonObject(); - jsonObject.add("pipelineElements", makePipelineElementList()); - - return jsonObject.toString(); - } - - private JsonArray makePipelineElementList() { - JsonArray elementList = new JsonArray(); - - assetModels.forEach(am -> { - elementList.add(makePipelineElementEntry(am)); - }); - - return elementList; - } - - private JsonObject makePipelineElementEntry(AssetModel am) { - JsonObject entry = new JsonObject(); - entry.addProperty("appId", am.getAppId()); - entry.addProperty("type", am.getPeType().getFriendly()); - entry.addProperty("name", am.getPipelineElementName()); - entry.addProperty("description", am.getPipelineElementDescription()); - entry.addProperty("dockerHubLink", makeDockerHubLink(am.getPeType(), am.getModuleName())); - entry.addProperty("githubLink", makeGithubLink(am.getModuleName())); - - return entry; - } - - private String makeDockerHubLink(PeType peType, String moduleName) { - String dockerHubRepoName = peType == PeType.ADAPTER ? "connect-worker" : moduleName.replace("streampipes-", ""); - return "https://hub.docker.com/r/apachestreampipes/" + dockerHubRepoName; - } - - private String makeGithubLink(String moduleName) { - return "https://github.com/apache/incubator-streampipes-extensions/tree/dev/" + moduleName; - } - - -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java new file mode 100644 index 0000000000..7b83aa767a --- /dev/null +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/DocumentationResourceGenerator.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.smp.generator; + +import org.apache.streampipes.smp.model.AssetModel; +import org.apache.streampipes.smp.util.DirectoryManager; + +import com.google.common.base.Charsets; +import org.apache.commons.io.FileUtils; + +import java.io.IOException; +import java.nio.file.Path; + +public class DocumentationResourceGenerator extends ResourceGenerator { + + public static final String DOCUMENTATION_FILE_NAME = "documentation.md"; + + public DocumentationResourceGenerator(ClassLoader loader, + AssetModel extensionsElement, + Path targetPath) { + super(loader, extensionsElement, targetPath); + } + + @Override + public void generate() throws IOException { + DirectoryManager.createIfNotExists(targetPath); + try (var inputStream = getResourceInputStream(DOCUMENTATION_FILE_NAME)) { + if (inputStream != null) { + var originalDocumentationFileContents = new String(inputStream.readAllBytes()); + // modify docs for documentation page + String documentationFileContents = + new MarkdownTitleRemover(originalDocumentationFileContents).removeTitle(); + + documentationFileContents = + new MarkdownHeaderGenerator(extensionsElement, documentationFileContents).createHeaders(); + + documentationFileContents = new ImagePathReplacer(documentationFileContents, + extensionsElement.getAppId()).replaceContentForDocs(); + + FileUtils.writeStringToFile( + targetPath.resolve(extensionsElement.getAppId() + ".md").toFile(), + documentationFileContents, + Charsets.UTF_8); + } + } + } +} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java new file mode 100644 index 0000000000..b64581d42e --- /dev/null +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/IconResourceGenerator.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.smp.generator; + +import org.apache.streampipes.smp.model.AssetModel; +import org.apache.streampipes.smp.util.DirectoryManager; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +public class IconResourceGenerator extends ResourceGenerator { + + public static final String ICON_NAME = "icon.png"; + private static final String PlaceholderIconName = "placeholder-icon.png"; + + public IconResourceGenerator(ClassLoader loader, + AssetModel extensionsElement, + Path targetPath) { + super(loader, extensionsElement, targetPath); + } + + @Override + public void generate() throws IOException { + DirectoryManager.createIfNotExists(targetPath); + try (var inputStream = getResourceInputStream(ICON_NAME)) { + if (inputStream != null) { + Files.copy(inputStream, targetPath.resolve(ICON_NAME), StandardCopyOption.REPLACE_EXISTING); + } else { + ClassLoader classLoader = this.getClass().getClassLoader(); + try (var placeHolderIconStream = classLoader.getResourceAsStream(PlaceholderIconName)) { + if (placeHolderIconStream != null) { + Files.copy(placeHolderIconStream, targetPath.resolve(ICON_NAME), StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + } +} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java index cda528f0c5..85bd0bc636 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ImagePathReplacer.java @@ -26,11 +26,10 @@ public class ImagePathReplacer { private static final String IMAGE_REGEX = "( assetModels) { - super(assetModels); - } - - public String generate() { - - ContainerTag gridContent = div().withClass(PeGridConst.PE_GRID_CONTAINER); - for (AssetModel am : assetModels) { - gridContent.with(makeGrid(am)); - } - String markdownContent = gridContent.render(); - return markdownContent; - //return new MarkdownHeaderGenerator(assetModel, markdownContent).createHeaders(); - } - - private DomContent makeGrid(AssetModel assetModel) { - return new PipelineElementGridGenerator(assetModel).makeGrid(); - } - - -} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java similarity index 55% rename from streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java rename to streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java index 75186a33d2..9816ed785d 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/OutputGenerator.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/ResourceGenerator.java @@ -20,17 +20,27 @@ import org.apache.streampipes.smp.model.AssetModel; -import java.util.List; - -public abstract class OutputGenerator { - - protected List assetModels; - - public OutputGenerator(List assetModels) { - this.assetModels = assetModels; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +public abstract class ResourceGenerator { + + protected final ClassLoader loader; + protected final AssetModel extensionsElement; + protected final Path targetPath; + + public ResourceGenerator(ClassLoader loader, + AssetModel extensionsElement, + Path targetPath) { + this.loader = loader; + this.extensionsElement = extensionsElement; + this.targetPath = targetPath; } - public abstract String generate(); - + protected InputStream getResourceInputStream(String resourceName) { + return loader.getResourceAsStream(extensionsElement.getAppId() + "/" + resourceName); + } + public abstract void generate() throws IOException; } diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java index 3c492e66de..5f21291e46 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/generator/SidebarConfigGenerator.java @@ -18,29 +18,81 @@ package org.apache.streampipes.smp.generator; +import org.apache.streampipes.smp.constants.PeType; import org.apache.streampipes.smp.model.AssetModel; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import org.apache.http.client.fluent.Request; +import org.apache.maven.plugin.logging.Log; +import java.io.IOException; import java.util.List; -public class SidebarConfigGenerator extends OutputGenerator { +public class SidebarConfigGenerator { + private static final String ExistingSidebarsUrl = + "https://raw.githubusercontent.com/apache/streampipes-website/dev/website-v2/sidebars.json"; + private static final String DocumentationSection = "documentation"; + private static final String PipelineElementSection = "\uD83D\uDCDA Pipeline Elements"; - public SidebarConfigGenerator(List assetModels) { - super(assetModels); + private final List assetModels; + private final Log log; + + public SidebarConfigGenerator(Log log, + List assetModels) { + this.log = log; + this.assetModels = assetModels; } - public String generate() { + public String generate() throws IOException { + log.info("Downloading existing sidebar from Git repo"); + var existingSidebar = loadExistingSidebar(); + log.info("Sidebar download successful"); + var existingSidebarJson = JsonParser.parseString(existingSidebar); + JsonArray pipelineElements = new JsonArray(); assetModels.forEach(pe -> pipelineElements.add("pe/" + pe.getAppId())); - JsonObject pipelineElementObject = new JsonObject(); - pipelineElementObject.add("Pipeline Elements", pipelineElements); - JsonObject jsonObject = new JsonObject(); - jsonObject.add("pipeline-elements", pipelineElementObject); + pipelineElements.add(makeItems(PeType.ADAPTER, "Adapters")); + pipelineElements.add(makeItems(PeType.PROCESSOR, "Data Processors")); + pipelineElements.add(makeItems(PeType.SINK, "Data Sinks")); + + var section = existingSidebarJson + .getAsJsonObject() + .get(DocumentationSection) + .getAsJsonObject(); + + section.add(PipelineElementSection, pipelineElements); + + return existingSidebarJson.toString(); + } + + private JsonObject makeItems(PeType type, + String typeName) { + var obj = new JsonObject(); + var peJsonArray = new JsonArray(); + obj.add("type", new JsonPrimitive("category")); + obj.add("label", new JsonPrimitive(typeName)); + + assetModels + .stream() + .filter(am -> am.getPeType() == type) + .forEach(am -> { + peJsonArray.add(new JsonPrimitive("pe/" + am.getAppId())); + }); + + obj.add("items", peJsonArray); + return obj; + } - return jsonObject.toString(); + private String loadExistingSidebar() throws IOException { + return Request + .Get(ExistingSidebarsUrl) + .execute() + .returnContent() + .asString(); } -} \ No newline at end of file +} diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java index 60447217d0..ecb77de77b 100644 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java +++ b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/model/AssetModel.java @@ -26,14 +26,16 @@ public class AssetModel implements Comparable { private String pipelineElementName; private String pipelineElementDescription; private PeType peType = PeType.PROCESSOR; - private String moduleName = ""; - private String containerName = ""; - private String baseDir; public AssetModel() { } + public AssetModel(String appId, PeType peType) { + this.appId = appId; + this.peType = peType; + } + public AssetModel(String appId, String pipelineElementName, String pipelineElementDescription) { this.appId = appId; this.pipelineElementName = pipelineElementName; @@ -44,9 +46,6 @@ public String getAppId() { return appId; } - public void setAppId(String appId) { - this.appId = appId; - } public String getPipelineElementName() { return pipelineElementName; @@ -68,34 +67,6 @@ public PeType getPeType() { return peType; } - public void setPeType(PeType peType) { - this.peType = peType; - } - - public String getModuleName() { - return moduleName; - } - - public void setModuleName(String moduleName) { - this.moduleName = moduleName; - } - - public String getContainerName() { - return containerName; - } - - public void setContainerName(String containerName) { - this.containerName = containerName; - } - - public String getBaseDir() { - return baseDir; - } - - public void setBaseDir(String baseDir) { - this.baseDir = baseDir; - } - @Override public String toString() { return pipelineElementName + " (" + peType.toString() + ", " + appId + "): " diff --git a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java b/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java deleted file mode 100644 index 5dc380eff6..0000000000 --- a/streampipes-maven-plugin/src/main/java/org/apache/streampipes/smp/util/DuplicateRemover.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.util; - -import org.apache.streampipes.smp.model.AssetModel; - -import java.io.File; -import java.util.List; -import java.util.stream.Collectors; - -public class DuplicateRemover { - - private List assetModels; - private String sourceDir; - - public DuplicateRemover(String sourceDir, List assetModels) { - this.assetModels = assetModels; - this.sourceDir = sourceDir; - } - - public List removeAlreadyExisting() { - - return assetModels - .stream() - .filter(am -> !(new File(Utils.makePath(sourceDir, am.getAppId() + File.separator - + "documentation.md")).exists())) - .collect(Collectors.toList()); - } -} diff --git a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java b/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java deleted file mode 100644 index 0dccd4ed18..0000000000 --- a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/extractor/TestAssetModelExtractor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.extractor; - -import org.apache.streampipes.smp.constants.PeType; -import org.apache.streampipes.smp.model.AssetModel; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; - -public class TestAssetModelExtractor { - - private String getGetDeclareModelContent4 = - "return ProcessingElementBuilder.create(\"org.apache.streampipes.processors.filters.jvm.numericalfilter\")\n" - + " .category(DataProcessorType.FILTER)\n" - + " .withAssets(Assets.DOCUMENTATION, Assets.ICON)\n" - + " .withLocales(Locales.EN)\n" - + " .requiredStream(StreamRequirementsBuilder\n" - + " .create()\n" - + " .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(),\n" - + " Labels.withId(NUMBER_MAPPING),\n" - + " PropertyScope.NONE).build())\n" - + " .outputStrategy(OutputStrategies.keep())\n" - + " .requiredSingleValueSelection(Labels.withId(OPERATION), Options.from(\"<\", \"<=\", \">\",\n" - + " \">=\", \"==\", \"!=\"))\n" - + " .requiredFloatParameter(Labels.withId(VALUE), NUMBER_MAPPING)\n" - + " .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())\n" - + " .supportedFormats(SupportedFormats.jsonFormat())\n" - + " .build();"; - - - @Test - public void testAssetExtraction4() { - AssetModel model = new AssetModelItemExtractor(getGetDeclareModelContent4).extractAssetItem(); - - assertEquals("org.apache.streampipes.processors.filters.jvm.numericalfilter", model.getAppId()); - assertEquals(PeType.PROCESSOR, model.getPeType()); - assertNull(model.getPipelineElementName()); - assertNull(model.getPipelineElementDescription()); - - } -} diff --git a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java b/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java deleted file mode 100644 index 87e60b975d..0000000000 --- a/streampipes-maven-plugin/src/test/java/org/apache/streampipes/smp/generator/TestPipelineElementOverviewGenerator.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.streampipes.smp.generator; - -import org.apache.streampipes.smp.constants.PeType; -import org.apache.streampipes.smp.model.AssetModel; - -import org.junit.Test; - -import java.util.Arrays; - -import static org.junit.Assert.assertTrue; - -public class TestPipelineElementOverviewGenerator { - - private AssetModel am = new AssetModel("a", "B", "C"); - - @Test - public void testHtmlGenerator() { - am.setPeType(PeType.PROCESSOR); - String html = - new PipelineElementOverviewGenerator(Arrays.asList(am)).generate(); - - System.out.println(html); - - assertTrue(true); - } -}