Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Warn user about duplicate processor names - #41 implementation #42

Merged
merged 2 commits into from
Mar 1, 2018
Merged

Warn user about duplicate processor names - #41 implementation #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f9c9fd8
Implementing duplicate processor names check
Feb 26, 2018
db353df
Implementing duplicate processor names check - adding positive test
Feb 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions src/main/java/com/github/hermannpencole/nifi/config/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@

import com.github.hermannpencole.nifi.config.model.ConfigException;
import com.github.hermannpencole.nifi.config.service.*;
import com.github.hermannpencole.nifi.swagger.ApiClient;
import com.github.hermannpencole.nifi.swagger.ApiException;
import com.github.hermannpencole.nifi.swagger.Configuration;
import com.github.hermannpencole.nifi.swagger.client.model.PositionDTO;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
Expand Down Expand Up @@ -79,6 +77,7 @@ public static void main(String[] args) throws Exception {
options.addOption("keepTemplate", false, "Keep template after installation (default false)");
options.addOption("placeWidth", true, "Width of place for installing group (default 1935 : 430 * (4 + 1/2) = 4 pro line)");
options.addOption("startPosition", true, "Starting position for the place for installing group, format x,y (default : 0,0)");
options.addOption("failOnDuplicateNames", false, "Fail if template contains duplicate processor names in extractConfig mode");

// parse the command line arguments
CommandLine cmd = commandLineParser.parse(options, args);
Expand Down Expand Up @@ -139,7 +138,7 @@ public static void main(String[] args) throws Exception {
} else if ("extractConfig".equals(cmd.getOptionValue("m"))) {
//Get an instance of the bean from the context
ExtractProcessorService processorService = injector.getInstance(ExtractProcessorService.class);
processorService.extractByBranch(branchList, fileConfiguration);
processorService.extractByBranch(branchList, fileConfiguration, cmd.hasOption("failOnDuplicateNames"));
LOG.info("The group configuration {} is extrated on file {}", branch, fileConfiguration);
} else if ("deployTemplate".equals(cmd.getOptionValue("m"))) {
TemplateService templateService = injector.getInstance(TemplateService.class);
Expand Down
39 changes: 34 additions & 5 deletions src/main/java/com/github/hermannpencole/nifi/config/service/ExtractProcessorService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.github.hermannpencole.nifi.swagger.client.ControllerApi;
import com.github.hermannpencole.nifi.swagger.client.FlowApi;
import com.github.hermannpencole.nifi.swagger.client.model.*;
import com.google.common.base.Joiner;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.slf4j.Logger;
Expand All @@ -14,10 +15,8 @@
import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;

/**
* Class that offer service for nifi processor
Expand Down Expand Up @@ -49,7 +48,7 @@ public class ExtractProcessorService {
* @throws IOException
* @throws ApiException
*/
public void extractByBranch(List<String> branch, String fileConfiguration) throws IOException, ApiException {
public void extractByBranch(List<String> branch, String fileConfiguration, boolean failOnDuplicateNames) throws IOException, ApiException {
File file = new File(fileConfiguration);

ProcessGroupFlowEntity componentSearch = processGroupService.changeDirectory(branch)
Expand All @@ -67,6 +66,8 @@ public void extractByBranch(List<String> branch, String fileConfiguration) throw
result.getControllerServicesDTO().add(extractController(controllerServiceEntity));
}

checkDuplicateProcessorNames(result.getProcessors(), failOnDuplicateNames);

//convert to json
Gson gson = new GsonBuilder().setPrettyPrinting().create();
LOG.debug("saving in file {}", fileConfiguration);
Expand All @@ -77,6 +78,34 @@ public void extractByBranch(List<String> branch, String fileConfiguration) throw
}
}

private void checkDuplicateProcessorNames(List<ProcessorDTO> processors, boolean failOnDuplicateNames) {
//warn or fail on duplicate processor names
if (processors == null || processors.isEmpty()) {
return;
}

Map<String, Integer> duplicateProcessorNames = detectDuplicateProcessorNames(processors);
if (!duplicateProcessorNames.isEmpty()) {
String messageFormatted = "Duplicate processor names detected: "
+ Joiner.on(", ").withKeyValueSeparator(" used times: ").join(duplicateProcessorNames);

if (failOnDuplicateNames) {
throw new ConfigException(messageFormatted);
} else {
LOG.warn(messageFormatted);
}
}
}

private Map<String, Integer> detectDuplicateProcessorNames(List<ProcessorDTO> processorList) {
Map<String, Integer> processorNameCountMap = new HashMap<>();
processorList.forEach(proc -> processorNameCountMap.merge(proc.getName(), 1, Integer::sum));

return processorNameCountMap.entrySet().stream()
.filter(entry -> entry.getValue() > 1)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}

/**
* extract from component
*
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/github/hermannpencole/nifi/config/MainTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ protected void configure() {
Mockito.when(Guice.createInjector((AbstractModule)anyObject())).thenReturn(injector);

Main.main(new String[]{"-nifi","http://localhost:8080/nifi-api","-conf","adr","-m","extractConfig", "-accessFromTicket"});
verify(extractProcessorServiceMock).extractByBranch(Arrays.asList("root"), "adr");
verify(extractProcessorServiceMock).extractByBranch(Arrays.asList("root"), "adr", false);
}

@Test
Expand Down
40 changes: 36 additions & 4 deletions src/test/java/com/github/hermannpencole/nifi/config/service/ExtractProcessorServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public void extractNotExitingBranchTest() throws ApiException, IOException, URIS
List<String> branch = Arrays.asList("root", "elt1");
File temp = File.createTempFile("tempfile", ".tmp");
when(processGroupServiceMock.changeDirectory(branch)).thenReturn(Optional.empty());
extractService.extractByBranch(branch, temp.getAbsolutePath());
extractService.extractByBranch(branch, temp.getAbsolutePath(), false);
}

@Test(expected = FileNotFoundException.class)
Expand All @@ -52,7 +52,7 @@ public void extractErrorFileBranchTest() throws ApiException, IOException, URISy
when(flowapiMock.getControllerServicesFromGroup("idComponent")).thenReturn(new ControllerServicesEntity());

File temp = File.createTempFile("tempfile", ".tmp");
extractService.extractByBranch(branch, temp.getParent());
extractService.extractByBranch(branch, temp.getParent(), false);
}

@Test
Expand All @@ -65,7 +65,7 @@ public void extractEmptyBranchTest() throws ApiException, IOException, URISyntax
when(processGroupServiceMock.changeDirectory(branch)).thenReturn(Optional.of(response));
when(flowapiMock.getControllerServicesFromGroup("idComponent")).thenReturn(new ControllerServicesEntity());

extractService.extractByBranch(branch, temp.getAbsolutePath());
extractService.extractByBranch(branch, temp.getAbsolutePath(), false);

//evaluate response
Gson gson = new Gson();
Expand Down Expand Up @@ -98,7 +98,7 @@ public void extractBranchTest() throws ApiException, IOException, URISyntaxExcep
ProcessGroupFlowEntity subGroupResponse = TestUtils.createProcessGroupFlowEntity("idSubGroup", "nameSubGroup");
when(flowapiMock.getFlow(subGroupResponse.getProcessGroupFlow().getId())).thenReturn(subGroupResponse);

extractService.extractByBranch(branch, temp.getAbsolutePath());
extractService.extractByBranch(branch, temp.getAbsolutePath(), false);
Gson gson = new Gson();
try (Reader reader = new InputStreamReader(new FileInputStream(temp), "UTF-8")) {
GroupProcessorsEntity result = gson.fromJson(reader, GroupProcessorsEntity.class);
Expand All @@ -112,7 +112,39 @@ public void extractBranchTest() throws ApiException, IOException, URISyntaxExcep
}
}

@Test(expected = ConfigException.class)
public void extractDuplicateProcessorNamesTest() throws ApiException, IOException {
List<String> branch = Arrays.asList("root", "elt1");
File temp = File.createTempFile("tempfile", ".tmp");

ProcessGroupFlowEntity response = TestUtils.createProcessGroupFlowEntity("idComponent", "nameComponent");
response.getProcessGroupFlow().getFlow()
.getProcessors().add(TestUtils.createProcessorEntity("idProc1","nameProcA") );
response.getProcessGroupFlow().getFlow()
.getProcessors().add(TestUtils.createProcessorEntity("idProc2","nameProcA") );
response.getProcessGroupFlow().getFlow()
.getProcessors().add(TestUtils.createProcessorEntity("idProc2","nameProcB") );

when(processGroupServiceMock.changeDirectory(branch)).thenReturn(Optional.of(response));
when(flowapiMock.getControllerServicesFromGroup("idComponent")).thenReturn(new ControllerServicesEntity());

extractService.extractByBranch(branch, temp.getAbsolutePath(), true);
}

@Test
public void extractNonDuplicateProcessorNamesTest() throws ApiException, IOException {
List<String> branch = Arrays.asList("root", "elt1");
File temp = File.createTempFile("tempfile", ".tmp");

ProcessGroupFlowEntity response = TestUtils.createProcessGroupFlowEntity("idComponent", "nameComponent");
response.getProcessGroupFlow().getFlow()
.getProcessors().add(TestUtils.createProcessorEntity("idProc1","nameProcA") );
response.getProcessGroupFlow().getFlow()
.getProcessors().add(TestUtils.createProcessorEntity("idProc2","nameProcB") );

when(processGroupServiceMock.changeDirectory(branch)).thenReturn(Optional.of(response));
when(flowapiMock.getControllerServicesFromGroup("idComponent")).thenReturn(new ControllerServicesEntity());

extractService.extractByBranch(branch, temp.getAbsolutePath(), true);
}
}