From 00200e35b9cd382ef58e8e0cdb18fa3ebc183d8d Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:17:44 +0530 Subject: [PATCH 01/17] add: get relationship kind by id --- .../org/finos/waltz/data/rel/RelationshipKindDao.java | 8 ++++++++ .../relationship_kind/RelationshipKindService.java | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/waltz-data/src/main/java/org/finos/waltz/data/rel/RelationshipKindDao.java b/waltz-data/src/main/java/org/finos/waltz/data/rel/RelationshipKindDao.java index c9bcb39382..0ed215165f 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/rel/RelationshipKindDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/rel/RelationshipKindDao.java @@ -95,6 +95,14 @@ public Set findAll() { .fetchSet(TO_DOMAIN_MAPPER); } + public RelationshipKind getById(long id) { + return dsl + .select() + .from(RELATIONSHIP_KIND) + .where(RELATIONSHIP_KIND.ID.eq(id)) + .fetchOne(TO_DOMAIN_MAPPER); + } + public Set findRelationshipKindsBetweenEntites(EntityReference parent, EntityReference target){ diff --git a/waltz-service/src/main/java/org/finos/waltz/service/relationship_kind/RelationshipKindService.java b/waltz-service/src/main/java/org/finos/waltz/service/relationship_kind/RelationshipKindService.java index 0d3e51e3d4..bce6644c4c 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/relationship_kind/RelationshipKindService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/relationship_kind/RelationshipKindService.java @@ -34,6 +34,10 @@ public Collection findAll() { return relationshipKindDao.findAll(); } + public RelationshipKind getById(long id) { + return relationshipKindDao.getById(id); + } + public boolean create(RelationshipKind relationshipKind) { return relationshipKindDao.create(relationshipKind); From c17709c5196c79f2979a11dc0080bc9a29528451 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:19:43 +0530 Subject: [PATCH 02/17] add: get entity relationships by relationship kind / code --- .../EntityRelationshipDao.java | 26 +++++++++++++++++++ .../EntityRelationshipService.java | 4 +++ 2 files changed, 30 insertions(+) diff --git a/waltz-data/src/main/java/org/finos/waltz/data/entity_relationship/EntityRelationshipDao.java b/waltz-data/src/main/java/org/finos/waltz/data/entity_relationship/EntityRelationshipDao.java index 91dbbf5281..d80444e903 100644 --- a/waltz-data/src/main/java/org/finos/waltz/data/entity_relationship/EntityRelationshipDao.java +++ b/waltz-data/src/main/java/org/finos/waltz/data/entity_relationship/EntityRelationshipDao.java @@ -281,6 +281,32 @@ private Condition mkExactRefMatchCondition(EntityReference ref) { return matchesA.or(matchesB); } + public Collection getEntityRelationshipsByKind(org.finos.waltz.model.rel.RelationshipKind relationshipKind) { + return dsl + .select(ENTITY_RELATIONSHIP.fields()) + .from(ENTITY_RELATIONSHIP) + .where(ENTITY_RELATIONSHIP.RELATIONSHIP.eq(relationshipKind.code())) + .fetch(r -> { + EntityRelationshipRecord record = r.into(ENTITY_RELATIONSHIP); + return ImmutableEntityRelationship.builder() + .id(record.getId()) + .a(ImmutableEntityReference.builder() + .kind(EntityKind.valueOf(record.getKindA())) + .id(record.getIdA()) + .build()) + .b(ImmutableEntityReference.builder() + .kind(EntityKind.valueOf(record.getKindB())) + .id(record.getIdB()) + .build()) + .provenance(record.getProvenance()) + .relationship(record.getRelationship()) + .description(record.getDescription()) + .lastUpdatedBy(record.getLastUpdatedBy()) + .lastUpdatedAt(toLocalDateTime(record.getLastUpdatedAt())) + .build(); + }); + } + private Condition mkExactKeyMatchCondition(EntityRelationshipKey key) { return ENTITY_RELATIONSHIP.ID_A.eq(key.a().id()) diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/EntityRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/EntityRelationshipService.java index 59fe5cdbfb..41574650b8 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/EntityRelationshipService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/EntityRelationshipService.java @@ -118,4 +118,8 @@ public int deleteForGenericEntitySelector(IdSelectionOptions selectionOptions) { public void migrateEntityRelationships(EntityReference sourceReference, EntityReference targetReference, String userId) { entityRelationshipDao.migrateEntityRelationships(sourceReference, targetReference, userId); } + + public Collection getEntityRelationshipsByKind(org.finos.waltz.model.rel.RelationshipKind relationshipKind) { + return entityRelationshipDao.getEntityRelationshipsByKind(relationshipKind); + } } From 9678a929f4dfa0491526c125ac447585f368ea96 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:21:52 +0530 Subject: [PATCH 03/17] add: models for bulk relationship upload --- .../BulkUploadRelationshipApplyResult.java | 15 ++++++ .../BulkUploadRelationshipItem.java | 25 ++++++++++ .../BulkUploadRelationshipParsedResult.java | 50 +++++++++++++++++++ .../BulkUploadRelationshipValidatedItem.java | 28 +++++++++++ ...ulkUploadRelationshipValidationResult.java | 16 ++++++ .../entity_relationship/UploadOperation.java | 7 +++ .../entity_relationship/ValidationError.java | 9 ++++ 7 files changed, 150 insertions(+) create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipApplyResult.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipItem.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipParsedResult.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidatedItem.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidationResult.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/UploadOperation.java create mode 100644 waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/ValidationError.java diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipApplyResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipApplyResult.java new file mode 100644 index 0000000000..721e105c74 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipApplyResult.java @@ -0,0 +1,15 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.immutables.value.Value; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkUploadRelationshipApplyResult.class) +public interface BulkUploadRelationshipApplyResult { + + Long recordsAdded(); + + Long recordsUpdated(); + + Long skippedRows(); +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipItem.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipItem.java new file mode 100644 index 0000000000..1bda22936f --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipItem.java @@ -0,0 +1,25 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +@Value.Immutable +@JsonDeserialize(as = ImmutableBulkUploadRelationshipItem.class) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonFormat(with = JsonFormat.Feature.ACCEPT_CASE_INSENSITIVE_PROPERTIES) +public interface BulkUploadRelationshipItem { + + @JsonAlias({"source_external_id", "source_id"}) + String sourceExternalId(); + + @JsonAlias({"target_external_id", "target_id"}) + String targetExternalId(); + + @Nullable + String description(); + +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipParsedResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipParsedResult.java new file mode 100644 index 0000000000..7a6756982f --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipParsedResult.java @@ -0,0 +1,50 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +import java.util.List; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkUploadRelationshipParsedResult.class) +public interface BulkUploadRelationshipParsedResult { + + @Value.Immutable + interface BulkUploadRelationshipParseError { + String message(); + + @Nullable + Integer line(); + + @Nullable + Integer column(); + } + + String input(); + + List parsedItems(); + + @Nullable + BulkUploadRelationshipParseError error(); + + static BulkUploadRelationshipParsedResult mkResult(List items, + String input) { + if(items.isEmpty()) { + return ImmutableBulkUploadRelationshipParsedResult + .builder() + .input(input) + .error(ImmutableBulkUploadRelationshipParseError + .builder() + .message("Cannot parse input.") + .build()) + .build(); + } + + return ImmutableBulkUploadRelationshipParsedResult + .builder() + .parsedItems(items) + .input(input) + .build(); + } +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidatedItem.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidatedItem.java new file mode 100644 index 0000000000..199083843e --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidatedItem.java @@ -0,0 +1,28 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +import java.util.Set; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkUploadRelationshipValidatedItem.class) +public interface BulkUploadRelationshipValidatedItem { + BulkUploadRelationshipItem parsedItem(); + + @Nullable + EntityReference sourceEntityRef(); + + @Nullable + EntityReference targetEntityRef(); + + @Nullable + String description(); + + @Nullable + Set error(); + + UploadOperation uploadOperation(); +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidationResult.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidationResult.java new file mode 100644 index 0000000000..90a276b8ce --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/BulkUploadRelationshipValidationResult.java @@ -0,0 +1,16 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import org.finos.waltz.model.Nullable; +import org.immutables.value.Value; + +import java.util.List; + +@Value.Immutable +@JsonSerialize(as = ImmutableBulkUploadRelationshipValidationResult.class) +public interface BulkUploadRelationshipValidationResult { + List validatedItems(); + + @Nullable + BulkUploadRelationshipParsedResult.BulkUploadRelationshipParseError parseError(); +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/UploadOperation.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/UploadOperation.java new file mode 100644 index 0000000000..64553e7fb2 --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/UploadOperation.java @@ -0,0 +1,7 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +public enum UploadOperation { + ADD, + UPDATE, + NONE +} diff --git a/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/ValidationError.java b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/ValidationError.java new file mode 100644 index 0000000000..6ada02a2cd --- /dev/null +++ b/waltz-model/src/main/java/org/finos/waltz/model/bulk_upload/entity_relationship/ValidationError.java @@ -0,0 +1,9 @@ +package org.finos.waltz.model.bulk_upload.entity_relationship; + +public enum ValidationError { + SOURCE_INVALID, + SOURCE_NOT_FOUND, + TARGET_INVALID, + TARGET_NOT_FOUND + +} From 20d0976667dbfaa7be2c050206fbf81a28d5f832 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:23:43 +0530 Subject: [PATCH 04/17] add: Item parser for relationship items --- .../BulkUploadRelationshipItemParser.java | 124 ++++++++++++++++++ .../BulkUploadRelationshipItemParserTest.java | 64 +++++++++ 2 files changed, 188 insertions(+) create mode 100644 waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParser.java create mode 100644 waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParser.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParser.java new file mode 100644 index 0000000000..386bcd661f --- /dev/null +++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParser.java @@ -0,0 +1,124 @@ +package org.finos.waltz.service.entity_relationship; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import org.finos.waltz.common.StreamUtilities; +import org.finos.waltz.common.StringUtilities; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipItem; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipParsedResult; +import org.finos.waltz.model.bulk_upload.entity_relationship.ImmutableBulkUploadRelationshipParsedResult; +import org.finos.waltz.model.bulk_upload.entity_relationship.ImmutableBulkUploadRelationshipParseError; + +import java.io.IOException; +import java.util.List; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.finos.waltz.common.StringUtilities.isEmpty; + +public class BulkUploadRelationshipItemParser { + + public enum InputFormat { + CSV, + TSV, + JSON + } + + public BulkUploadRelationshipParsedResult parse(String input, InputFormat format) { + if (isEmpty(input)) { + return handleEmptyInput(input); + } + try { + switch (format) { + case CSV: + return parseCSV(clean(input)); + case TSV: + return parseTSV(clean(input)); + case JSON: + return parseJSON(clean(input)); + default: + throw new IllegalArgumentException(format("Unknown format: %s", format)); + } + } catch (IOException e) { + return ImmutableBulkUploadRelationshipParsedResult + .builder() + .input(input) + .error(ImmutableBulkUploadRelationshipParseError + .builder() + .message(e.getMessage()) + .build()) + .build(); + } + } + + private BulkUploadRelationshipParsedResult parseJSON(String input) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + MappingIterator items = mapper + .readerFor(BulkUploadRelationshipItem.class) + .readValues(input); + + return BulkUploadRelationshipParsedResult.mkResult( + items.readAll(), + input); + } + + private BulkUploadRelationshipParsedResult parseTSV(String input) throws IOException { + List items = attemptToParseDelimited(input, configTSVSchema()); + return BulkUploadRelationshipParsedResult.mkResult(items, input); + } + + private BulkUploadRelationshipParsedResult parseCSV(String input) throws IOException { + List items = attemptToParseDelimited(input, configCSVSchema()); + return BulkUploadRelationshipParsedResult.mkResult(items, input); + } + + private List attemptToParseDelimited(String input, CsvSchema bootstrapSchema) throws IOException { + CsvMapper mapper = new CsvMapper(); + mapper.enable(CsvParser.Feature.TRIM_SPACES); + mapper.enable(CsvParser.Feature.SKIP_EMPTY_LINES); + + MappingIterator items = mapper + .readerFor(BulkUploadRelationshipItem.class) + .with(bootstrapSchema) + .readValues(input); + + return items.readAll(); + } + + private BulkUploadRelationshipParsedResult handleEmptyInput(String input) { + return ImmutableBulkUploadRelationshipParsedResult + .builder() + .error(ImmutableBulkUploadRelationshipParseError + .builder() + .message("Cannot parse input.") + .column(0) + .line(0) + .build()) + .input(input) + .build(); + } + + private CsvSchema configCSVSchema() { + return CsvSchema + .emptySchema() + .withHeader(); + } + + private CsvSchema configTSVSchema() { + return CsvSchema + .emptySchema() + .withHeader() + .withColumnSeparator('\t'); + } + + private String clean(String input) { + return StreamUtilities + .lines(input) + .filter(StringUtilities::isDefined) + .filter(line -> !line.startsWith("#")) + .collect(Collectors.joining("\n")); + } +} diff --git a/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java b/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java new file mode 100644 index 0000000000..292d8e5a0d --- /dev/null +++ b/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java @@ -0,0 +1,64 @@ +package org.finos.waltz.service.entity_relationship; + +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipItem; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipParsedResult; +import org.finos.waltz.model.bulk_upload.entity_relationship.ImmutableBulkUploadRelationshipItem; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import static org.finos.waltz.common.IOUtilities.readAsString; +import static org.junit.jupiter.api.Assertions.*; + +public class BulkUploadRelationshipItemParserTest { + + private final BulkUploadRelationshipItemParser parser = new BulkUploadRelationshipItemParser(); + + + private String readTestFile(String fileName) { + return readAsString(BulkUploadRelationshipItemParserTest.class.getResourceAsStream(fileName)); + } + + private List getParsedItems() { + List parsedItems = new ArrayList(); + parsedItems.add(ImmutableBulkUploadRelationshipItem + .builder() + .sourceExternalId("CT_001") + .targetExternalId("10235-1") + .comment("Comment") + .build()); + parsedItems.add(ImmutableBulkUploadRelationshipItem + .builder() + .sourceExternalId("CT_001") + .targetExternalId("109235-1") + .build()); + + return parsedItems; + } + + + @Test + void simpleTSV() { + /* + Test to check whether the parser is parsing items + */ + BulkUploadRelationshipParsedResult result = parser.parse(readTestFile("test-relationship-item.tsv"), BulkUploadRelationshipItemParser.InputFormat.TSV); + assertEquals(null, result.error()); + assertEquals(2, result.parsedItems().size()); + + Set sourceExternalIds = SetUtilities.map(result.parsedItems(), BulkUploadRelationshipItem::sourceExternalId); + assertEquals(sourceExternalIds, SetUtilities.asSet("CT_001", "CT_002")); + } + + @Test + void errorTSV() { + /* + Test to check whether the parser throws format exception + */ + BulkUploadRelationshipParsedResult result = parser.parse(readTestFile("test-relationship-error-item.tsv"), BulkUploadRelationshipItemParser.InputFormat.TSV); + assertNotNull(result.error()); + } +} From 899d5764e60d2792f2e61a3ec65fd935412351d9 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:24:35 +0530 Subject: [PATCH 05/17] add: bulk upload relationship service and endpoint --- .../BulkUploadRelationshipService.java | 309 ++++++++++++++++++ .../api/EntityRelationshipEndpoint.java | 40 ++- 2 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java new file mode 100644 index 0000000000..9568be5f78 --- /dev/null +++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java @@ -0,0 +1,309 @@ +package org.finos.waltz.service.entity_relationship; + +import org.finos.waltz.common.DateTimeUtilities; +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.common.StringUtilities; +import org.finos.waltz.model.*; +import org.finos.waltz.model.bulk_upload.entity_relationship.*; +import org.finos.waltz.model.bulk_upload.measurable_rating.ChangeOperation; +import org.finos.waltz.model.entity_relationship.EntityRelationship; +import org.finos.waltz.model.entity_relationship.ImmutableEntityRelationship; +import org.finos.waltz.model.exceptions.NotAuthorizedException; +import org.finos.waltz.model.rel.RelationshipKind; +import org.finos.waltz.model.user.SystemRole; +import org.finos.waltz.schema.Tables; +import org.finos.waltz.schema.tables.records.ChangeLogRecord; +import org.finos.waltz.schema.tables.records.EntityRelationshipRecord; +import org.finos.waltz.service.DIConfiguration; +import org.finos.waltz.service.relationship_kind.RelationshipKindService; +import org.finos.waltz.service.user.UserRoleService; +import org.jooq.DSLContext; +import org.jooq.UpdateConditionStep; +import org.jooq.impl.DSL; +import org.jooq.lambda.tuple.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import org.springframework.stereotype.Service; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static java.util.stream.Collectors.toSet; +import static org.finos.waltz.common.StringUtilities.isEmpty; +import static org.finos.waltz.data.JooqUtilities.loadExternalIdToEntityRefMap; +import static org.finos.waltz.data.JooqUtilities.summarizeResults; +import static org.finos.waltz.model.EntityReference.mkRef; +import static org.jooq.lambda.tuple.Tuple.tuple; + +@Service +public class BulkUploadRelationshipService { + + private static final Logger LOG = LoggerFactory.getLogger(BulkUploadRelationshipService.class); + + private final RelationshipKindService relationshipKindService; + + private final org.finos.waltz.schema.tables.EntityRelationship ENTITY_RELATIONSHIP = Tables.ENTITY_RELATIONSHIP; + + private static final String PROVENANCE = "bulkUploadRelationships"; + + private final EntityRelationshipService entityRelationshipService; + + private final UserRoleService userRoleService; + + private final DSLContext dsl; + + + public BulkUploadRelationshipService(RelationshipKindService relationshipKindService, + EntityRelationshipService entityRelationshipService, + UserRoleService userRoleService, + DSLContext dsl) { + this.relationshipKindService = relationshipKindService; + this.entityRelationshipService = entityRelationshipService; + this.userRoleService = userRoleService; + this.dsl = dsl; + } + + public BulkUploadRelationshipValidationResult bulkPreview(String input, Long relationshipKindId) { + + RelationshipKind relationshipKind = relationshipKindService.getById(relationshipKindId); + BulkUploadRelationshipParsedResult parsedResult = new BulkUploadRelationshipItemParser().parse(input, BulkUploadRelationshipItemParser.InputFormat.TSV); + Map sourceEntityRefMap = new HashMap<>(); + Map targetEntityRefMap = new HashMap<>(); + + if (parsedResult.error() != null) { + return ImmutableBulkUploadRelationshipValidationResult + .builder() + .parseError(parsedResult.error()) + .build(); + } else { + // load source entity ref map + if (relationshipKind.categoryA() != null) { + sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryA())); + } else { + sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA()); + } + + // load target entity ref map + if (relationshipKind.categoryB() != null) { + targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryB())); + } else { + targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB()); + } + + // enrich each row to a tuple + final Map finalSourceEntityRefMap = sourceEntityRefMap; + final Map finalTargetEntityRefMap = targetEntityRefMap; + final List>> listOfRows = parsedResult.parsedItems().stream() + .map(r -> tuple( + r, + finalSourceEntityRefMap.get(r.sourceExternalId()), + finalTargetEntityRefMap.get(r.targetExternalId()) + )) + .map(t -> { + Set validationErrors = new HashSet<>(); + if (t.v2 == null) { + validationErrors.add(ValidationError.SOURCE_NOT_FOUND); + } + if (t.v3 == null) { + validationErrors.add(ValidationError.TARGET_NOT_FOUND); + } + return t.concat(validationErrors); + }) + .collect(Collectors.toList()); + + // load all entity relationships for the given kind + final Collection relationshipsForRelationshipKind = entityRelationshipService.getEntityRelationshipsByKind(relationshipKind); + List filteredRelationshipsForKind = relationshipsForRelationshipKind + .stream() + .filter(r -> r.relationship().equals(relationshipKind.code()) && r.a().kind().equals(relationshipKind.kindA()) && r.b().kind().equals(relationshipKind.kindB())) + .collect(Collectors.toList()); + + List requiredRelationship = listOfRows.stream() + .filter(t -> t.v4.isEmpty()) + .map(t -> ImmutableEntityRelationship + .builder() + .a(t.v2) + .b(t.v3) + .relationship(relationshipKind.code()) + .description(t.v1.description()) + .lastUpdatedBy("abc.xyz") + .provenance("dummy") + .lastUpdatedAt(LocalDateTime.now()) + .build()) + .collect(Collectors.toList()); + + // create a diff b/w required and existing relationships + DiffResult diffResult = DiffResult + .mkDiff( + filteredRelationshipsForKind, + requiredRelationship, + r -> tuple(r.a(), r.b()), + (a, b) -> StringUtilities.safeEq(b.description(), a.description()) + ); + + Set> toAdd = SetUtilities.map(diffResult.otherOnly(), d -> tuple(d.a(), d.b())); + Set> toUpdate = SetUtilities.map(diffResult.differingIntersection(), d -> tuple(d.a(), d.b())); + + List validatedItemsList = listOfRows + .stream() + .map(t -> { + if (toAdd.contains(tuple(t.v2, t.v3))) { + return t.concat(UploadOperation.ADD); + } else if (toUpdate.contains(tuple(t.v2, t.v3))) { + return t.concat(UploadOperation.UPDATE); + } + return t.concat(UploadOperation.NONE); + }) + .map(t -> ImmutableBulkUploadRelationshipValidatedItem + .builder() + .parsedItem(t.v1) + .sourceEntityRef(t.v2) + .targetEntityRef(t.v3) + .description(t.v1.description()) + .error(t.v4) + .uploadOperation(t.v5) + .build()) + .collect(Collectors.toList()); + return ImmutableBulkUploadRelationshipValidationResult + .builder() + .validatedItems(validatedItemsList) + .build(); + } + } + + public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidationResult preview, Long relationshipKindId, String user) { + verifyUserHasPermissions(user); + + if (preview.parseError() != null) { + throw new IllegalStateException("Cannot apply changes with formatting errors"); + } + + RelationshipKind relationshipKind = relationshipKindService.getById(relationshipKindId); + + Timestamp now = DateTimeUtilities.nowUtcTimestamp(); + + final Set toInsert = preview + .validatedItems() + .stream() + .filter(d -> d.uploadOperation() == UploadOperation.ADD && d.error().isEmpty()) + .map(d -> { + EntityRelationshipRecord r = new EntityRelationshipRecord(); + r.setIdA(d.sourceEntityRef().id()); + r.setIdB(d.targetEntityRef().id()); + r.setKindA(d.sourceEntityRef().kind().name()); + r.setKindB(d.targetEntityRef().kind().name()); + r.setRelationship(relationshipKind.code()); + r.setDescription(d.description()); + r.setProvenance(PROVENANCE); + r.setLastUpdatedAt(now); + r.setLastUpdatedBy(user); + return r; + + }) + .collect(Collectors.toSet()); + + final Set> toUpdate = preview + .validatedItems() + .stream() + .filter(d -> d.uploadOperation() == UploadOperation.UPDATE && d.error().isEmpty()) + .map(d -> DSL + .update(ENTITY_RELATIONSHIP) + .set(ENTITY_RELATIONSHIP.DESCRIPTION, d.description()) + .set(ENTITY_RELATIONSHIP.LAST_UPDATED_AT, now) + .set(ENTITY_RELATIONSHIP.LAST_UPDATED_BY, user) + .where(ENTITY_RELATIONSHIP.RELATIONSHIP.eq(relationshipKind.code())) + .and(ENTITY_RELATIONSHIP.KIND_A.eq(relationshipKind.kindA().name())) + .and(ENTITY_RELATIONSHIP.KIND_B.eq(relationshipKind.kindB().name())) + .and(ENTITY_RELATIONSHIP.ID_A.eq(d.sourceEntityRef().id())) + .and(ENTITY_RELATIONSHIP.ID_B.eq(d.targetEntityRef().id()))) + .collect(Collectors.toSet()); + + final Set auditLogs = preview + .validatedItems() + .stream() + .filter(d -> d.uploadOperation() != UploadOperation.NONE) + .map(d -> { + ChangeLogRecord r = new ChangeLogRecord(); + r.setMessage(mkChangeMessage(d.sourceEntityRef(), + d.targetEntityRef(), + relationshipKind, + d.uploadOperation())); + r.setOperation(toChangeLogOperation(d.uploadOperation()).name()); + r.setParentKind(relationshipKind.code()); + r.setParentId(relationshipKindId); + r.setCreatedAt(now); + r.setUserId(user); + r.setSeverity(Severity.INFORMATION.name()); + return r; + }) + .collect(Collectors.toSet()); + + final long skippedRows = preview + .validatedItems() + .stream() + .filter(d -> d.uploadOperation() == UploadOperation.NONE || !d.error().isEmpty()) + .count(); + + return dsl + .transactionResult(ctx -> { + DSLContext tx = ctx.dsl(); + long insertCount = summarizeResults(tx.batchInsert(toInsert).execute()); + long updateCount = summarizeResults(tx.batch(toUpdate).execute()); + + long changeLogCount = summarizeResults(tx.batchInsert(auditLogs).execute()); + + LOG.info( + "Batch Relationships: {} adds, {} updates, {} changelogs.", + insertCount, + updateCount, + changeLogCount + ); + + return ImmutableBulkUploadRelationshipApplyResult + .builder() + .recordsAdded(insertCount) + .recordsUpdated(updateCount) + .skippedRows(skippedRows) + .build(); + }); + } + + private String mkChangeMessage(EntityReference sourceRef, EntityReference targetRef, RelationshipKind relationshipKind, UploadOperation uploadOperation) { + return format( + "Bulk Relationships Update - Operation %s, Relationship b/w %s -> %s, Relationship Kind %s. %s", + uploadOperation, + sourceRef.name(), + targetRef.name(), + relationshipKind.id(), + relationshipKind.code() + ); + } + + private Operation toChangeLogOperation(UploadOperation uploadOperation) { + switch (uploadOperation) { + case ADD: + return Operation.ADD; + case UPDATE: + return Operation.UPDATE; + default: + return Operation.UNKNOWN; + } + } + + private void verifyUserHasPermissions(String userId) { + if (!userRoleService.hasRole(userId, SystemRole.ADMIN.name())) { + throw new NotAuthorizedException(); + } + } + +// public static void main(String[] args) { +// AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); +// DSLContext dsl = ctx.getBean(DSLContext.class); +// System.out.println(dsl.select().from(Tables.MEASURABLE).fetch()); +// } +} diff --git a/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/EntityRelationshipEndpoint.java b/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/EntityRelationshipEndpoint.java index c9c7359025..142a1846d4 100644 --- a/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/EntityRelationshipEndpoint.java +++ b/waltz-web/src/main/java/org/finos/waltz/web/endpoints/api/EntityRelationshipEndpoint.java @@ -18,7 +18,14 @@ package org.finos.waltz.web.endpoints.api; +import org.finos.waltz.common.EnumUtilities; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.bulk_upload.BulkUpdateMode; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipApplyResult; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipValidationResult; +import org.finos.waltz.service.entity_relationship.BulkUploadRelationshipService; import org.finos.waltz.service.entity_relationship.EntityRelationshipService; +import org.finos.waltz.service.measurable_rating.BulkMeasurableItemParser; import org.finos.waltz.web.DatumRoute; import org.finos.waltz.web.ListRoute; import org.finos.waltz.web.endpoints.Endpoint; @@ -29,10 +36,12 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import spark.Request; +import spark.Response; import java.util.List; import java.util.stream.Collectors; +import static org.finos.waltz.model.EntityReference.mkRef; import static org.finos.waltz.web.WebUtilities.*; import static org.finos.waltz.web.endpoints.EndpointUtilities.*; import static org.finos.waltz.common.Checks.checkNotNull; @@ -45,11 +54,16 @@ public class EntityRelationshipEndpoint implements Endpoint { private final EntityRelationshipService entityRelationshipService; + private final BulkUploadRelationshipService bulkUploadRelationshipService; + @Autowired - public EntityRelationshipEndpoint(EntityRelationshipService entityRelationshipService) { + public EntityRelationshipEndpoint(EntityRelationshipService entityRelationshipService, + BulkUploadRelationshipService bulkUploadRelationshipService) { checkNotNull(entityRelationshipService, "entityRelationshipService cannot be null"); + checkNotNull(bulkUploadRelationshipService, "bulkUploadRelationshipService cannot be null"); this.entityRelationshipService = entityRelationshipService; + this.bulkUploadRelationshipService = bulkUploadRelationshipService; } @@ -57,8 +71,13 @@ public EntityRelationshipEndpoint(EntityRelationshipService entityRelationshipSe public void register() { String getByIdPath = mkPath(BASE_URL, "id", ":id"); String findForEntityPath = mkPath(BASE_URL, "entity", ":kind", ":id"); + String bulkPreview = mkPath(BASE_URL, "bulk", "preview", ":id"); + String bulkApply = mkPath(BASE_URL, "bulk", "apply", ":id"); String exactMatchPath = mkPath(BASE_URL, "relationship", ":aKind", ":aId", ":bKind", ":bId", ":relationshipKind"); + registerPreviewBulkUploadRelationship(bulkPreview); + registerApplyBulkUploadRelationship(bulkApply); + DatumRoute removeRelationshipRoute = (req, resp) -> { EntityRelationshipKey entityRelationshipKey = ImmutableEntityRelationshipKey .builder() @@ -125,4 +144,23 @@ private List parseRelationshipKindParams(Request req) { .filter(rk -> rk != null) .collect(Collectors.toList()); } + + private void registerPreviewBulkUploadRelationship(String path) { + postForDatum(path, (req, resp) -> { + Long relationshipKindId = Long.parseLong(req.params("id")); + String body = req.body(); + return bulkUploadRelationshipService.bulkPreview(body, relationshipKindId); + }); + } + + private void registerApplyBulkUploadRelationship(String path) { + postForDatum(path, (req, resp) -> { + Long relationshipKindId = Long.parseLong(req.params("id")); + String body = req.body(); + String user = getUsername(req); + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(body, relationshipKindId); + BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, relationshipKindId, user); + return applyResult; + }); + } } From ade24b4998f266b800eabdb61f99419523a29bce Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 15:25:47 +0530 Subject: [PATCH 06/17] UI component for bulk relationships upload --- .../entity-relationship-store.js | 19 +- .../system/relationship-kinds-view.html | 10 + .../client/system/relationship-kinds-view.js | 17 +- .../BulkRelationshipUpload.svelte | 321 ++++++++++++++++++ 4 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte diff --git a/waltz-ng/client/svelte-stores/entity-relationship-store.js b/waltz-ng/client/svelte-stores/entity-relationship-store.js index 806cbd1b8f..46bb5010d8 100644 --- a/waltz-ng/client/svelte-stores/entity-relationship-store.js +++ b/waltz-ng/client/svelte-stores/entity-relationship-store.js @@ -80,12 +80,29 @@ export function mkEntityRelationshipStore() { }; + const bulkUploadRelationshipsPreview = (relationshipKindId, data) => remote + .execute( + "POST", + `api/entity-relationship/bulk/preview/${relationshipKindId}`, + data + ); + + const bulkUploadRelationshipsApply = (relationshipKindId, data) => remote + .execute( + "POST", + `api/entity-relationship/bulk/apply/${relationshipKindId}`, + data + ); + + return { getById, findBetweenEntities, findForEntity, remove, - create + create, + bulkUploadRelationshipsPreview, + bulkUploadRelationshipsApply }; } diff --git a/waltz-ng/client/system/relationship-kinds-view.html b/waltz-ng/client/system/relationship-kinds-view.html index 2866b6afae..507fd1bb11 100644 --- a/waltz-ng/client/system/relationship-kinds-view.html +++ b/waltz-ng/client/system/relationship-kinds-view.html @@ -165,6 +165,11 @@

class="clickable"> Remove + | + + Bulk Upload Relationships + @@ -175,6 +180,11 @@

+
+ + +
+

Update relationship kind:

diff --git a/waltz-ng/client/system/relationship-kinds-view.js b/waltz-ng/client/system/relationship-kinds-view.js index 360cc4481e..fc11b65e1e 100644 --- a/waltz-ng/client/system/relationship-kinds-view.js +++ b/waltz-ng/client/system/relationship-kinds-view.js @@ -22,6 +22,7 @@ import * as _ from "lodash"; import {displayError} from "../common/error-utils"; import {entity} from "../common/services/enums/entity"; import toasts from "../svelte-stores/toast-store"; +import BulkRelationshipUpload from "./svelte/bulk-relationships/BulkRelationshipUpload.svelte"; const initialState = { relationshipKinds: [], @@ -40,8 +41,10 @@ const initialState = { }, visibility: { create: false, - edit: false - } + update: false, + bulk: false + }, + BulkRelationshipUpload }; @@ -51,7 +54,7 @@ const columnDefs = [ displayName: "Name", width: "25%", },{ - field: "name", + field: "reverseName", displayName: "Reverse Name", width: "25%", }, { @@ -127,12 +130,14 @@ function controller(serviceBroker, $q) { vm.form.position = vm.selectedRelationshipKind.position; vm.visibility.update = true; + vm.visibility.bulk = false; }; vm.createRelationshipKind = () => { vm.resetForm(); vm.visibility.create = true; vm.visibility.update = false; + vm.visibility.bulk = false; vm.selectedRelationshipKind = null; }; @@ -171,6 +176,7 @@ function controller(serviceBroker, $q) { vm.onDismiss = () => { vm.visibility.create = false; vm.visibility.update = false; + vm.visibility.bulk = false; vm.resetForm(); }; @@ -225,6 +231,11 @@ function controller(serviceBroker, $q) { }); } } + + vm.bulkUploadRelationships = () => { + vm.visibility.bulk = true; + vm.visibility.update = false; + } } controller.$inject = [ diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte new file mode 100644 index 0000000000..999ac03fc2 --- /dev/null +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -0,0 +1,321 @@ + + +
+ + +
+ Bulk Upload Relationships +
+
+ {#if mode === MODES.EDIT} +
+ The bulk relationship editor can be used to upload multiple changes to the selected relationship. +
+
+ Help +
+

The bulk upload format should look like, column order is not important but the headers are:

+
+
Source External Id
+
This uniquely identifies the source (from) entity for that relationship.
+ +
Target External Id
+
This uniquely identifies the target (to) entity for that relationship.
+ +
Description
+
This optionally adds a comment to the change.
+
+ For Example: +
+ 
+sourceExternalId	 targetExternalId	 description
+12312	22312	This is a description
+#2234	357325	This line will be ignored
+66723	172452-1313-2424	This is another example
+12233	8242	This is another example
+
+
+ + +
+ {/if} + {#if mode === MODES.LOADING} + + {/if} + {#if mode === MODES.PREVIEW} + {#if previewResponse.parseError} +
+ There was a problem generating the preview: + 
{previewResponse.parseError.message}
+
+ + {/if} + {#if !previewResponse.parseError} +
+ The table below shows the preview of what will happen if you apply these changes: +
+
+
+ + + + + + + + + + + + {#each previewResponse.validatedItems as obj } + + + + + + + + {/each} + +
Source EntityTarget EntityDescriptionOperationError
{obj.sourceEntityRef ? obj.sourceEntityRef.name : obj.parsedItem.sourceExternalId}{obj.targetEntityRef? obj.targetEntityRef.name : obj.parsedItem.targetExternalId}{obj.description}{obj.uploadOperation}{obj.error.length != 0? obj.error : ""}
+
+
+ + + + +
+ {/if} + {/if} + {#if mode === MODES.RESULT} +
+ The table below shows the result of your applied changes: +
+ + + + + + + + + + + + + + + +
Added Records 0}> + {applyResponse.recordsAdded} +
Updated Records 0}> + {applyResponse.recordsUpdated} +
Skipped Records 0}> + {applyResponse.skippedRows} +
+ + {/if} +
+ + + +
+ + \ No newline at end of file From fdc4a021a763181aa1f02e6d6c9db3ba6c0d5340 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Wed, 16 Oct 2024 16:27:29 +0530 Subject: [PATCH 07/17] add: test items; update: test --- .../BulkUploadRelationshipItemParserTest.java | 21 ++++++++++++------- .../test-relationship-error-item.tsv | 2 ++ .../test-relationship-item.tsv | 4 ++++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-error-item.tsv create mode 100644 waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-item.tsv diff --git a/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java b/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java index 292d8e5a0d..7faaae2552 100644 --- a/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java +++ b/waltz-service/src/test/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipItemParserTest.java @@ -26,16 +26,23 @@ private List getParsedItems() { List parsedItems = new ArrayList(); parsedItems.add(ImmutableBulkUploadRelationshipItem .builder() - .sourceExternalId("CT_001") - .targetExternalId("10235-1") - .comment("Comment") + .sourceExternalId("sourceExtA") + .targetExternalId("tarExtB") + .description("desc changes") .build()); parsedItems.add(ImmutableBulkUploadRelationshipItem .builder() - .sourceExternalId("CT_001") - .targetExternalId("109235-1") + .sourceExternalId("sourceExtB") + .targetExternalId("tarExtB") + .description("desc chnages 2") .build()); - + parsedItems.add(ImmutableBulkUploadRelationshipItem + .builder() + .sourceExternalId("sourceExtC") + .targetExternalId("tarExtC") + .description("desc changes") + .build()); + return parsedItems; } @@ -50,7 +57,7 @@ void simpleTSV() { assertEquals(2, result.parsedItems().size()); Set sourceExternalIds = SetUtilities.map(result.parsedItems(), BulkUploadRelationshipItem::sourceExternalId); - assertEquals(sourceExternalIds, SetUtilities.asSet("CT_001", "CT_002")); + assertEquals(sourceExternalIds, SetUtilities.asSet("sourceExtA", "sourceExtB", "sourceExtC")); } @Test diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-error-item.tsv b/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-error-item.tsv new file mode 100644 index 0000000000..25cef7dd87 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-error-item.tsv @@ -0,0 +1,2 @@ +sourceExt targetExt desc +randomExt invalidData corrupt corrupt \ No newline at end of file diff --git a/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-item.tsv b/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-item.tsv new file mode 100644 index 0000000000..1c4cc47e43 --- /dev/null +++ b/waltz-service/src/test/resources/org/finos/waltz/service/entity_relationship/test-relationship-item.tsv @@ -0,0 +1,4 @@ +source_external_id target_external_id description +sourceExtA tarExtA desc changes +sourceExtB tarExtB desc chnages 2 +sourceExtC tarExtC desc changes \ No newline at end of file From 20363c2edfd6e17efed9f322c55c5b7fda508962 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Thu, 17 Oct 2024 17:34:37 +0530 Subject: [PATCH 08/17] add: tests; change: indentation; wip: tests --- .../BulkUploadRelationshipServiceTest.java | 329 ++++++++++++++++++ .../BulkUploadRelationshipService.java | 30 +- 2 files changed, 341 insertions(+), 18 deletions(-) create mode 100644 waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java diff --git a/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java new file mode 100644 index 0000000000..fa6fcfb922 --- /dev/null +++ b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java @@ -0,0 +1,329 @@ +package org.finos.waltz.integration_test.inmem.service; + +import com.sun.org.slf4j.internal.Logger; +import com.sun.org.slf4j.internal.LoggerFactory; +import org.finos.waltz.common.SetUtilities; +import org.finos.waltz.common.exception.InsufficientPrivelegeException; +import org.finos.waltz.integration_test.inmem.BaseInMemoryIntegrationTest; +import org.finos.waltz.model.EntityKind; +import org.finos.waltz.model.EntityReference; +import org.finos.waltz.model.app_group.AppGroupDetail; +import org.finos.waltz.model.application.Application; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipApplyResult; +import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipValidationResult; +import org.finos.waltz.model.entity_relationship.EntityRelationship; +import org.finos.waltz.model.entity_relationship.ImmutableEntityRelationship; +import org.finos.waltz.model.measurable.Measurable; +import org.finos.waltz.model.rel.ImmutableRelationshipKind; +import org.finos.waltz.model.rel.RelationshipKind; +import org.finos.waltz.model.user.SystemRole; +import org.finos.waltz.service.app_group.AppGroupService; +import org.finos.waltz.service.application.ApplicationService; +import org.finos.waltz.service.entity_relationship.BulkUploadRelationshipService; +import org.finos.waltz.service.entity_relationship.EntityRelationshipService; +import org.finos.waltz.service.measurable.MeasurableService; +import org.finos.waltz.service.relationship_kind.RelationshipKindService; +import org.finos.waltz.test_common.helpers.*; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static java.lang.String.format; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +// do not run tests individually - run this test class instead +public class BulkUploadRelationshipServiceTest extends BaseInMemoryIntegrationTest { + + @Autowired + UserHelper userHelper; + + @Autowired + RelationshipKindService relationshipKindService; + + @Autowired + EntityRelationshipService entityRelationshipService; + + @Autowired + BulkUploadRelationshipService bulkUploadRelationshipService; + + @Autowired + AppHelper appHelper; + + @Autowired + ActorHelper actorHelper; + + @Autowired + ApplicationService applicationService; + + @Autowired + AppGroupHelper appGroupHelper; + + @Autowired + AppGroupService appGroupService; + + @Autowired + MeasurableHelper measurableHelper; + + @Autowired + MeasurableService measurableService; + + private static final Logger LOG = LoggerFactory.getLogger(BulkUploadRelationshipServiceTest.class); + + @Test + public void testBulkUploadWithTargetCategory() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + List appList = new ArrayList(); + + appHelper.createNewApp("tApp0", 65L, "t-0"); + appHelper.createNewApp("tApp1", 63L, "t-1"); + appHelper.createNewApp("tApp2", 66L, "t-2"); + appHelper.createNewApp("tApp3", 64L, "t-3"); + + List allApps = applicationService.findAll(); + + long cat1 = measurableHelper.createMeasurableCategory("testCat0"); + long msblId1 = measurableHelper.createMeasurable("M_0", "measurable0", cat1); + Measurable measurable1 = measurableService.getById(msblId1); + + appList = allApps.stream().map( + t -> EntityReference.mkRef(t.kind(), + t.id().get(), + t.name(), + t.description(), + t.externalId().orElse("not"))) + .collect(Collectors.toList()); + + relationshipKindService.create(ImmutableRelationshipKind + .builder() + .kindA(EntityKind.APPLICATION) + .kindB(EntityKind.MEASURABLE) + .categoryB(measurable1.categoryId()) + .code("RELATES_TO") + .name("Relates to") + .reverseName("Is related by") + .description("test rk") + .position(1) + .build()); + + RelationshipKind relationshipKind = relationshipKindService.getById(28L); + + // inserting relationships in the db to test against + for(int i = 0; i < appList.size(); i++) { + EntityReference t = appList.get(i); + entityRelationshipService.createRelationship( + ImmutableEntityRelationship + .builder() + .relationship(relationshipKind.code()) + .a(t) + .b(EntityReference.mkRef(EntityKind.MEASURABLE, + msblId1, measurable1.name(), measurable1.description(), + measurable1.externalId().orElse("not"))) + .lastUpdatedBy("test_user") + .lastUpdatedAt(LocalDateTime.now()) + .provenance("testProv") + .description("testDesc") + .build()); + } + + appHelper.createNewApp("tApp5", 65L, "t-4"); + appHelper.createNewApp("tApp6", 63L, "t-5"); + appHelper.createNewApp("tApp7", 66L, "t-6"); + appHelper.createNewApp("tApp8", 64L, "t-7"); + + allApps = applicationService.findAll(); + appList.clear(); + + appList = allApps.stream().map( + t -> EntityReference.mkRef(t.kind(), + t.id().get(), + t.name(), + t.description(), + t.externalId().orElse("not"))) + .collect(Collectors.toList()); + + appList = appList.subList(4, appList.size()); + + final EntityReference measurable1ref = EntityReference.mkRef( + EntityKind.MEASURABLE, + measurable1.id().get(), + measurable1.name(), + measurable1.description(), + measurable1.externalId().get()); + + String testInput = mkGoodTestInput(appList, measurable1ref, "testDesc"); + testInput = testInput.concat(mkGoodTestRow("t-0", "M_0", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("t-1", "M_0", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("t-0222", "M_0", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("t-1222", "M_0", "changed desc")); + + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 28L); + BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 28L, "test_user"); + + // expecting 4 relationships to be added + assertEquals(4l, applyResult.recordsAdded().longValue()); + + // expecting 2 records to be updated + assertEquals(2l, applyResult.recordsUpdated().longValue()); + + // expecting 2 rows to be skipped as these 2 are rows with errors + assertEquals(2l, applyResult.skippedRows().longValue()); + } + + @Test + public void testBulkUploadWithSourceCategory() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + List appList = new ArrayList(); + + List allApps = applicationService.findAll(); + + Measurable measurable1 = measurableService.getById(1l); + + appList = allApps.stream().map( + t -> EntityReference.mkRef(t.kind(), + t.id().get(), + t.name(), + t.description(), + t.externalId().orElse("not"))) + .collect(Collectors.toList()); + + appList = appList.subList(0, appList.size() - 4); + + relationshipKindService.create(ImmutableRelationshipKind + .builder() + .kindB(EntityKind.APPLICATION) + .kindA(EntityKind.MEASURABLE) + .categoryA(measurable1.categoryId()) + .code("RELATES_TO") + .name("Relates to") + .reverseName("Is related by") + .description("test rk") + .position(1) + .build()); + + RelationshipKind relationshipKind = relationshipKindService.getById(29L); + + // inserting relationships in the db to test against + for(int i = 0; i < appList.size(); i++) { + EntityReference t = appList.get(i); + entityRelationshipService.createRelationship( + ImmutableEntityRelationship + .builder() + .relationship(relationshipKind.code()) + .b(t) + .a(EntityReference.mkRef(EntityKind.MEASURABLE, + measurable1.id().get(), measurable1.name(), measurable1.description(), + measurable1.externalId().orElse("not"))) + .lastUpdatedBy("test_user") + .lastUpdatedAt(LocalDateTime.now()) + .provenance("testProv") + .description("testDesc") + .build()); + } + + allApps = applicationService.findAll(); + appList.clear(); + + appList = allApps.stream().map( + t -> EntityReference.mkRef(t.kind(), + t.id().get(), + t.name(), + t.description(), + t.externalId().orElse("not"))) + .collect(Collectors.toList()); + + appList = appList.subList(4, appList.size()); + + final EntityReference measurable1ref = EntityReference.mkRef( + EntityKind.MEASURABLE, + measurable1.id().get(), + measurable1.name(), + measurable1.description(), + measurable1.externalId().get()); + + String testInput = mkGoodTestInput(measurable1ref, appList, "testDesc"); + testInput = testInput.concat(mkGoodTestRow("M_0", "t-0", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("M_0", "t-1", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("M_12", "t-0", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("M_1222", "t-0", "changed desc")); + + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 29L); + BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 29L, "test_user"); + + // expecting 4 relationships to be added + assertEquals(4l, applyResult.recordsAdded().longValue()); + + // expecting 2 records to be updated + assertEquals(2l, applyResult.recordsUpdated().longValue()); + + // expecting 2 rows to be skipped as these 2 are rows with errors + assertEquals(2l, applyResult.skippedRows().longValue()); + } + + @Test + public void testErrorInput() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + String testInput = mkGoodTestRow("asfsfee", "", ""); + testInput = testInput.concat(mkGoodTestRow("jde", "ere", "eerer")); + + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 13L); + + assertNotNull(previewResult.parseError()); + BulkUploadRelationshipApplyResult applyResult = null; + try { + applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 13L, "test_user"); + } catch (Exception e) { + assertNotNull(e); + } + } + + // helpers + private String mkGoodTestInput(List srcItems, EntityReference targItem, String description) { + return format("source_external_id\ttarget_external_id\tdescription\n" + mkGoodTestRows(srcItems, targItem, description)); + } + + private String mkGoodTestInput(EntityReference srcItem, List targItems, String description) { + return format("source_external_id\ttarget_external_id\tdescription\n" + mkGoodTestRows(srcItem, targItems, description)); + } + + private String mkGoodTestRow(String srcExtId, String tarExtId, String description) { + return format("%s\t%s\t%s\n", + srcExtId, + tarExtId, + description); + } + + private String mkGoodTestRows(List src, EntityReference tar, String description) { + String result = ""; + for (int i = 0; i < src.size(); i++) { + result += format("%s\t%s\t%s\n", + src.get(i).externalId().orElse("not"), + tar.externalId().orElse("not"), + description); + } + return result; + } + + private String mkGoodTestRows(EntityReference src, List tar, String description) { + String result = ""; + for (int i = 0; i < tar.size(); i++) { + result += format("%s\t%s\t%s\n", + src.externalId().orElse("not"), + tar.get(i).externalId().orElse("not"), + description); + } + return result; + } + + +} diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java index 9568be5f78..b82f473a50 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java @@ -5,7 +5,6 @@ import org.finos.waltz.common.StringUtilities; import org.finos.waltz.model.*; import org.finos.waltz.model.bulk_upload.entity_relationship.*; -import org.finos.waltz.model.bulk_upload.measurable_rating.ChangeOperation; import org.finos.waltz.model.entity_relationship.EntityRelationship; import org.finos.waltz.model.entity_relationship.ImmutableEntityRelationship; import org.finos.waltz.model.exceptions.NotAuthorizedException; @@ -14,7 +13,6 @@ import org.finos.waltz.schema.Tables; import org.finos.waltz.schema.tables.records.ChangeLogRecord; import org.finos.waltz.schema.tables.records.EntityRelationshipRecord; -import org.finos.waltz.service.DIConfiguration; import org.finos.waltz.service.relationship_kind.RelationshipKindService; import org.finos.waltz.service.user.UserRoleService; import org.jooq.DSLContext; @@ -23,18 +21,14 @@ import org.jooq.lambda.tuple.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; import org.springframework.stereotype.Service; import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.lang.String.format; -import static java.util.stream.Collectors.toSet; -import static org.finos.waltz.common.StringUtilities.isEmpty; import static org.finos.waltz.data.JooqUtilities.loadExternalIdToEntityRefMap; import static org.finos.waltz.data.JooqUtilities.summarizeResults; import static org.finos.waltz.model.EntityReference.mkRef; @@ -98,7 +92,9 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel // enrich each row to a tuple final Map finalSourceEntityRefMap = sourceEntityRefMap; final Map finalTargetEntityRefMap = targetEntityRefMap; - final List>> listOfRows = parsedResult.parsedItems().stream() + final List>> listOfRows = parsedResult + .parsedItems() + .stream() .map(r -> tuple( r, finalSourceEntityRefMap.get(r.sourceExternalId()), @@ -120,10 +116,14 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel final Collection relationshipsForRelationshipKind = entityRelationshipService.getEntityRelationshipsByKind(relationshipKind); List filteredRelationshipsForKind = relationshipsForRelationshipKind .stream() - .filter(r -> r.relationship().equals(relationshipKind.code()) && r.a().kind().equals(relationshipKind.kindA()) && r.b().kind().equals(relationshipKind.kindB())) + .filter(r -> r.relationship() + .equals(relationshipKind.code()) + && r.a().kind().equals(relationshipKind.kindA()) + && r.b().kind().equals(relationshipKind.kindB())) .collect(Collectors.toList()); - List requiredRelationship = listOfRows.stream() + List requiredRelationship = listOfRows + .stream() .filter(t -> t.v4.isEmpty()) .map(t -> ImmutableEntityRelationship .builder() @@ -176,7 +176,9 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel } } - public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidationResult preview, Long relationshipKindId, String user) { + public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidationResult preview, + Long relationshipKindId, String user) { + verifyUserHasPermissions(user); if (preview.parseError() != null) { @@ -203,7 +205,6 @@ public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidat r.setLastUpdatedAt(now); r.setLastUpdatedBy(user); return r; - }) .collect(Collectors.toSet()); @@ -254,7 +255,6 @@ public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidat DSLContext tx = ctx.dsl(); long insertCount = summarizeResults(tx.batchInsert(toInsert).execute()); long updateCount = summarizeResults(tx.batch(toUpdate).execute()); - long changeLogCount = summarizeResults(tx.batchInsert(auditLogs).execute()); LOG.info( @@ -300,10 +300,4 @@ private void verifyUserHasPermissions(String userId) { throw new NotAuthorizedException(); } } - -// public static void main(String[] args) { -// AnnotationConfigApplicationContext ctx = new AnnotationConfigApplicationContext(DIConfiguration.class); -// DSLContext dsl = ctx.getBean(DSLContext.class); -// System.out.println(dsl.select().from(Tables.MEASURABLE).fetch()); -// } } From 9b9afead87974866ea58e1b3a561d66f9ae22e1d Mon Sep 17 00:00:00 2001 From: jainshrk Date: Thu, 17 Oct 2024 17:47:41 +0530 Subject: [PATCH 09/17] remove dummy response --- .../BulkRelationshipUpload.svelte | 84 ------------------- 1 file changed, 84 deletions(-) diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte index 999ac03fc2..efe039928a 100644 --- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -8,90 +8,6 @@ export let selectedRelationshipKind; console.log(selectedRelationshipKind) - const dummy_response = { - "validatedItems": [ - { - "parsedItem": { - "sourceExternalId": "INV71008", - "targetExternalId": "829-1", - "comment": "Comment" - }, - "sourceEntityRef": { - "description": "The objective of project \"Banking Repricing\" is to introduce new competitive package-based current account products and increase the general revenue of current accounts.
\nFor this, a fundamental change of the underlying
\nIT platform is necessary to enable a highly customizable pricing and billing of services.
\n2018:
\n- Design future state of accounts and pricing of banking products in general (Business)
\n- Evaluate business requirements and onboard first product on pricing & billing platform", - "kind": "CHANGE_INITIATIVE", - "id": 8743518, - "name": "Banking Repricing (replaced by INV75686)", - "externalId": "INV71008", - "entityLifecycleStatus": "ACTIVE" - }, - "targetEntityRef": { - "description": "Payments (ZV - Zahlungsverkehr) and Current Accounts (KK - Kontokorrent ); Real-time processing system for account administration, transaction processing, booking, output management and settlement of German current accounts for PBC (incl. Norisbank, dbEurope), CIB and PWM customers. \r\nDVAG : a module for the provisioning of DVAG business. \r\nSystem: Mainframe, Midrange, eBranch. Apps usage type: account administration, transaction booking, account settlement, inhouse clearing, reporting. Zahlungsverkehr (payments) part is done within 22954-1 Domestic Payments DE.", - "kind": "APPLICATION", - "id": 19229, - "name": "ZVKK Current Account and Payment Mgmt", - "externalId": "829-1", - "entityLifecycleStatus": "ACTIVE" - }, - "comment": "Comment", - "error": ["SOURCE_NOT_FOUND"], - "uploadOperation": "NONE" - }, - { - "parsedItem": { - "sourceExternalId": "INV71008", - "targetExternalId": "829-1", - "comment": "Comment" - }, - "sourceEntityRef": { - "description": "The objective of project \"Banking Repricing\" is to introduce new competitive package-based current account products and increase the general revenue of current accounts.
\nFor this, a fundamental change of the underlying
\nIT platform is necessary to enable a highly customizable pricing and billing of services.
\n2018:
\n- Design future state of accounts and pricing of banking products in general (Business)
\n- Evaluate business requirements and onboard first product on pricing & billing platform", - "kind": "CHANGE_INITIATIVE", - "id": 8743518, - "name": "Banking Repricing (replaced by INV75686)", - "externalId": "INV71008", - "entityLifecycleStatus": "ACTIVE" - }, - "targetEntityRef": { - "description": "Payments (ZV - Zahlungsverkehr) and Current Accounts (KK - Kontokorrent ); Real-time processing system for account administration, transaction processing, booking, output management and settlement of German current accounts for PBC (incl. Norisbank, dbEurope), CIB and PWM customers. \r\nDVAG : a module for the provisioning of DVAG business. \r\nSystem: Mainframe, Midrange, eBranch. Apps usage type: account administration, transaction booking, account settlement, inhouse clearing, reporting. Zahlungsverkehr (payments) part is done within 22954-1 Domestic Payments DE.", - "kind": "APPLICATION", - "id": 19229, - "name": "ZVKK Current Account and Payment Mgmt", - "externalId": "829-1", - "entityLifecycleStatus": "ACTIVE" - }, - "comment": "Comment", - "error": [], - "uploadOperation": "NONE" - }, - { - "parsedItem": { - "sourceExternalId": "PBS0202290", - "targetExternalId": "109235-1", - "comment": "Comment" - }, - "sourceEntityRef": { - "description": "SatellitenmodellDSL-MA vor Ort im VC zur Verbesserung der Reichweite und Heben von Potentialen\r\nStärkung RepräsentanzmodellUmwandlung von DSL Repräsentanzen und BHW Repräsentanzen in einheitliches Modell\r\nStarpool vor OrtÜbernahme von Tätigkeiten durch Starpool\r\nAusbau DVAG-Kooperation", - "kind": "CHANGE_INITIATIVE", - "id": 601, - "name": "Dummy E_Ausbau Vertriebsoberfläche DSL", - "externalId": "PBS0202290", - "entityLifecycleStatus": "ACTIVE" - }, - "targetEntityRef": { - "description": "Architecture tool to aggregate information from different source. To provide reporting and visualization across CT for applications, tech and data for both senior management and the architecture community.", - "kind": "APPLICATION", - "id": 20506, - "name": "Waltz", - "externalId": "109235-1", - "entityLifecycleStatus": "ACTIVE" - }, - "comment": "Comment", - "error": [], - "uploadOperation": "ADD" - } - ], - "parseError": null - } - const MODES = { EDIT: "EDIT", PREVIEW: "PREVIEW", From a86b59be53f8ec51b8843a07e6efd3b7bd954fe4 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Thu, 17 Oct 2024 17:53:28 +0530 Subject: [PATCH 10/17] indentation changes --- .../BulkRelationshipUpload.svelte | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte index efe039928a..f0204751dd 100644 --- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -74,12 +74,6 @@ // set mode to given mode mode = givenMode; } - - async function dummyonBulkPreview() { - mode = MODES.LOADING; - previewResponse = dummy_response; - mode = MODES.PREVIEW; - }
@@ -116,14 +110,15 @@ sourceExternalId targetExternalId description 66723 172452-1313-2424 This is another example 12233 8242 This is another example -
+ + type="submit" + disabled={!rawText}>Submit
{/if} {#if mode === MODES.LOADING} @@ -135,7 +130,8 @@ sourceExternalId targetExternalId description There was a problem generating the preview: 
{previewResponse.parseError.message}
- {/if} @@ -145,7 +141,9 @@ sourceExternalId targetExternalId description
- +
@@ -181,7 +179,9 @@ sourceExternalId targetExternalId description disabled={ previewResponse.parseError } type="submit">Apply - + {/if} @@ -212,7 +212,9 @@ sourceExternalId targetExternalId description
Source Entity
- + {/if}
From 6b9a58c14ad902611ccba9c492b5028e8c8f8de4 Mon Sep 17 00:00:00 2001 From: jainshrk Date: Fri, 18 Oct 2024 08:47:16 +0530 Subject: [PATCH 11/17] code cleanup --- .../BulkRelationshipUpload.svelte | 1 - .../BulkUploadRelationshipService.java | 218 +++++++++--------- 2 files changed, 108 insertions(+), 111 deletions(-) diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte index f0204751dd..1c2b68a201 100644 --- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -6,7 +6,6 @@ import _ from "lodash"; export let selectedRelationshipKind; - console.log(selectedRelationshipKind) const MODES = { EDIT: "EDIT", diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java index b82f473a50..c68b8666d1 100644 --- a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java +++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java @@ -41,8 +41,6 @@ public class BulkUploadRelationshipService { private final RelationshipKindService relationshipKindService; - private final org.finos.waltz.schema.tables.EntityRelationship ENTITY_RELATIONSHIP = Tables.ENTITY_RELATIONSHIP; - private static final String PROVENANCE = "bulkUploadRelationships"; private final EntityRelationshipService entityRelationshipService; @@ -66,114 +64,114 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel RelationshipKind relationshipKind = relationshipKindService.getById(relationshipKindId); BulkUploadRelationshipParsedResult parsedResult = new BulkUploadRelationshipItemParser().parse(input, BulkUploadRelationshipItemParser.InputFormat.TSV); - Map sourceEntityRefMap = new HashMap<>(); - Map targetEntityRefMap = new HashMap<>(); + Map sourceEntityRefMap; + Map targetEntityRefMap; if (parsedResult.error() != null) { return ImmutableBulkUploadRelationshipValidationResult .builder() .parseError(parsedResult.error()) .build(); + } + + // load source entity ref map + if (relationshipKind.categoryA() != null) { + sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryA())); } else { - // load source entity ref map - if (relationshipKind.categoryA() != null) { - sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryA())); - } else { - sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA()); - } - - // load target entity ref map - if (relationshipKind.categoryB() != null) { - targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryB())); - } else { - targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB()); - } - - // enrich each row to a tuple - final Map finalSourceEntityRefMap = sourceEntityRefMap; - final Map finalTargetEntityRefMap = targetEntityRefMap; - final List>> listOfRows = parsedResult - .parsedItems() - .stream() - .map(r -> tuple( - r, - finalSourceEntityRefMap.get(r.sourceExternalId()), - finalTargetEntityRefMap.get(r.targetExternalId()) - )) - .map(t -> { - Set validationErrors = new HashSet<>(); - if (t.v2 == null) { - validationErrors.add(ValidationError.SOURCE_NOT_FOUND); - } - if (t.v3 == null) { - validationErrors.add(ValidationError.TARGET_NOT_FOUND); - } - return t.concat(validationErrors); - }) - .collect(Collectors.toList()); - - // load all entity relationships for the given kind - final Collection relationshipsForRelationshipKind = entityRelationshipService.getEntityRelationshipsByKind(relationshipKind); - List filteredRelationshipsForKind = relationshipsForRelationshipKind - .stream() - .filter(r -> r.relationship() - .equals(relationshipKind.code()) - && r.a().kind().equals(relationshipKind.kindA()) - && r.b().kind().equals(relationshipKind.kindB())) - .collect(Collectors.toList()); - - List requiredRelationship = listOfRows - .stream() - .filter(t -> t.v4.isEmpty()) - .map(t -> ImmutableEntityRelationship - .builder() - .a(t.v2) - .b(t.v3) - .relationship(relationshipKind.code()) - .description(t.v1.description()) - .lastUpdatedBy("abc.xyz") - .provenance("dummy") - .lastUpdatedAt(LocalDateTime.now()) - .build()) - .collect(Collectors.toList()); - - // create a diff b/w required and existing relationships - DiffResult diffResult = DiffResult - .mkDiff( - filteredRelationshipsForKind, - requiredRelationship, - r -> tuple(r.a(), r.b()), - (a, b) -> StringUtilities.safeEq(b.description(), a.description()) - ); + sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA()); + } - Set> toAdd = SetUtilities.map(diffResult.otherOnly(), d -> tuple(d.a(), d.b())); - Set> toUpdate = SetUtilities.map(diffResult.differingIntersection(), d -> tuple(d.a(), d.b())); - - List validatedItemsList = listOfRows - .stream() - .map(t -> { - if (toAdd.contains(tuple(t.v2, t.v3))) { - return t.concat(UploadOperation.ADD); - } else if (toUpdate.contains(tuple(t.v2, t.v3))) { - return t.concat(UploadOperation.UPDATE); - } - return t.concat(UploadOperation.NONE); - }) - .map(t -> ImmutableBulkUploadRelationshipValidatedItem - .builder() - .parsedItem(t.v1) - .sourceEntityRef(t.v2) - .targetEntityRef(t.v3) - .description(t.v1.description()) - .error(t.v4) - .uploadOperation(t.v5) - .build()) - .collect(Collectors.toList()); - return ImmutableBulkUploadRelationshipValidationResult - .builder() - .validatedItems(validatedItemsList) - .build(); + // load target entity ref map + if (relationshipKind.categoryB() != null) { + targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryB())); + } else { + targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB()); } + + // enrich each row to a tuple + final Map finalSourceEntityRefMap = sourceEntityRefMap; + final Map finalTargetEntityRefMap = targetEntityRefMap; + final List>> listOfRows = parsedResult + .parsedItems() + .stream() + .map(r -> tuple( + r, + finalSourceEntityRefMap.get(r.sourceExternalId()), + finalTargetEntityRefMap.get(r.targetExternalId()) + )) + .map(t -> { + Set validationErrors = new HashSet<>(); + if (t.v2 == null) { + validationErrors.add(ValidationError.SOURCE_NOT_FOUND); + } + if (t.v3 == null) { + validationErrors.add(ValidationError.TARGET_NOT_FOUND); + } + return t.concat(validationErrors); + }) + .collect(Collectors.toList()); + + // load all entity relationships for the given kind + final Collection relationshipsForRelationshipKind = entityRelationshipService.getEntityRelationshipsByKind(relationshipKind); + List filteredRelationshipsForKind = relationshipsForRelationshipKind + .stream() + .filter(r -> r.relationship() + .equals(relationshipKind.code()) + && r.a().kind().equals(relationshipKind.kindA()) + && r.b().kind().equals(relationshipKind.kindB())) + .collect(Collectors.toList()); + + List requiredRelationship = listOfRows + .stream() + .filter(t -> t.v4.isEmpty()) + .map(t -> ImmutableEntityRelationship + .builder() + .a(t.v2) + .b(t.v3) + .relationship(relationshipKind.code()) + .description(t.v1.description()) + .lastUpdatedBy("abc.xyz") + .provenance("dummy") + .lastUpdatedAt(LocalDateTime.now()) + .build()) + .collect(Collectors.toList()); + + // create a diff b/w required and existing relationships + DiffResult diffResult = DiffResult + .mkDiff( + filteredRelationshipsForKind, + requiredRelationship, + r -> tuple(r.a(), r.b()), + (a, b) -> StringUtilities.safeEq(b.description(), a.description()) + ); + + Set> toAdd = SetUtilities.map(diffResult.otherOnly(), d -> tuple(d.a(), d.b())); + Set> toUpdate = SetUtilities.map(diffResult.differingIntersection(), d -> tuple(d.a(), d.b())); + + List validatedItemsList = listOfRows + .stream() + .map(t -> { + if (toAdd.contains(tuple(t.v2, t.v3))) { + return t.concat(UploadOperation.ADD); + } else if (toUpdate.contains(tuple(t.v2, t.v3))) { + return t.concat(UploadOperation.UPDATE); + } + return t.concat(UploadOperation.NONE); + }) + .map(t -> ImmutableBulkUploadRelationshipValidatedItem + .builder() + .parsedItem(t.v1) + .sourceEntityRef(t.v2) + .targetEntityRef(t.v3) + .description(t.v1.description()) + .error(t.v4) + .uploadOperation(t.v5) + .build()) + .collect(Collectors.toList()); + return ImmutableBulkUploadRelationshipValidationResult + .builder() + .validatedItems(validatedItemsList) + .build(); } public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidationResult preview, @@ -213,15 +211,15 @@ public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidat .stream() .filter(d -> d.uploadOperation() == UploadOperation.UPDATE && d.error().isEmpty()) .map(d -> DSL - .update(ENTITY_RELATIONSHIP) - .set(ENTITY_RELATIONSHIP.DESCRIPTION, d.description()) - .set(ENTITY_RELATIONSHIP.LAST_UPDATED_AT, now) - .set(ENTITY_RELATIONSHIP.LAST_UPDATED_BY, user) - .where(ENTITY_RELATIONSHIP.RELATIONSHIP.eq(relationshipKind.code())) - .and(ENTITY_RELATIONSHIP.KIND_A.eq(relationshipKind.kindA().name())) - .and(ENTITY_RELATIONSHIP.KIND_B.eq(relationshipKind.kindB().name())) - .and(ENTITY_RELATIONSHIP.ID_A.eq(d.sourceEntityRef().id())) - .and(ENTITY_RELATIONSHIP.ID_B.eq(d.targetEntityRef().id()))) + .update(Tables.ENTITY_RELATIONSHIP) + .set(Tables.ENTITY_RELATIONSHIP.DESCRIPTION, d.description()) + .set(Tables.ENTITY_RELATIONSHIP.LAST_UPDATED_AT, now) + .set(Tables.ENTITY_RELATIONSHIP.LAST_UPDATED_BY, user) + .where(Tables.ENTITY_RELATIONSHIP.RELATIONSHIP.eq(relationshipKind.code())) + .and(Tables.ENTITY_RELATIONSHIP.KIND_A.eq(relationshipKind.kindA().name())) + .and(Tables.ENTITY_RELATIONSHIP.KIND_B.eq(relationshipKind.kindB().name())) + .and(Tables.ENTITY_RELATIONSHIP.ID_A.eq(d.sourceEntityRef().id())) + .and(Tables.ENTITY_RELATIONSHIP.ID_B.eq(d.targetEntityRef().id()))) .collect(Collectors.toSet()); final Set auditLogs = preview From 8abacfaacaf7389cbcc19fa257f839aa9a14779b Mon Sep 17 00:00:00 2001 From: jainshrk Date: Fri, 18 Oct 2024 14:06:29 +0530 Subject: [PATCH 12/17] add / update - 6 totlal tests + minor indentaion change in ui component --- .../BulkUploadRelationshipServiceTest.java | 297 ++++++++++++++---- .../BulkRelationshipUpload.svelte | 2 +- 2 files changed, 238 insertions(+), 61 deletions(-) diff --git a/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java index fa6fcfb922..4b34e90868 100644 --- a/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java +++ b/waltz-integration-test/src/test/java/org/finos/waltz/integration_test/inmem/service/BulkUploadRelationshipServiceTest.java @@ -3,15 +3,13 @@ import com.sun.org.slf4j.internal.Logger; import com.sun.org.slf4j.internal.LoggerFactory; import org.finos.waltz.common.SetUtilities; -import org.finos.waltz.common.exception.InsufficientPrivelegeException; +import org.finos.waltz.common.StringUtilities; import org.finos.waltz.integration_test.inmem.BaseInMemoryIntegrationTest; import org.finos.waltz.model.EntityKind; import org.finos.waltz.model.EntityReference; -import org.finos.waltz.model.app_group.AppGroupDetail; import org.finos.waltz.model.application.Application; import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipApplyResult; import org.finos.waltz.model.bulk_upload.entity_relationship.BulkUploadRelationshipValidationResult; -import org.finos.waltz.model.entity_relationship.EntityRelationship; import org.finos.waltz.model.entity_relationship.ImmutableEntityRelationship; import org.finos.waltz.model.measurable.Measurable; import org.finos.waltz.model.rel.ImmutableRelationshipKind; @@ -29,14 +27,15 @@ import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static java.lang.String.format; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -// do not run tests individually - run this test class instead public class BulkUploadRelationshipServiceTest extends BaseInMemoryIntegrationTest { @Autowired @@ -54,18 +53,9 @@ public class BulkUploadRelationshipServiceTest extends BaseInMemoryIntegrationTe @Autowired AppHelper appHelper; - @Autowired - ActorHelper actorHelper; - @Autowired ApplicationService applicationService; - @Autowired - AppGroupHelper appGroupHelper; - - @Autowired - AppGroupService appGroupService; - @Autowired MeasurableHelper measurableHelper; @@ -81,12 +71,18 @@ public void testBulkUploadWithTargetCategory() { List appList = new ArrayList(); - appHelper.createNewApp("tApp0", 65L, "t-0"); - appHelper.createNewApp("tApp1", 63L, "t-1"); - appHelper.createNewApp("tApp2", 66L, "t-2"); - appHelper.createNewApp("tApp3", 64L, "t-3"); + EntityReference app1 = appHelper.createNewApp("tApp0", 65L, "t-0"); + EntityReference app2 = appHelper.createNewApp("tApp1", 63L, "t-1"); + EntityReference app3 = appHelper.createNewApp("tApp2", 66L, "t-2"); + EntityReference app4 = appHelper.createNewApp("tApp3", 64L, "t-3"); + + List appIds = new ArrayList<>(); + appIds.add(app1.id()); + appIds.add(app2.id()); + appIds.add(app3.id()); + appIds.add(app4.id()); - List allApps = applicationService.findAll(); + List allApps = applicationService.findByIds(appIds); long cat1 = measurableHelper.createMeasurableCategory("testCat0"); long msblId1 = measurableHelper.createMeasurable("M_0", "measurable0", cat1); @@ -112,7 +108,16 @@ public void testBulkUploadWithTargetCategory() { .position(1) .build()); - RelationshipKind relationshipKind = relationshipKindService.getById(28L); + Collection relationshipKinds = relationshipKindService.findAll(); + + RelationshipKind relationshipKind = relationshipKinds + .stream() + .filter(t -> t.kindA() == EntityKind.APPLICATION + && t.kindB() == EntityKind.MEASURABLE + && t.categoryB() != null + && t.categoryB() == measurable1.categoryId()) + .collect(Collectors.toList()) + .get(0); // inserting relationships in the db to test against for(int i = 0; i < appList.size(); i++) { @@ -132,12 +137,18 @@ public void testBulkUploadWithTargetCategory() { .build()); } - appHelper.createNewApp("tApp5", 65L, "t-4"); - appHelper.createNewApp("tApp6", 63L, "t-5"); - appHelper.createNewApp("tApp7", 66L, "t-6"); - appHelper.createNewApp("tApp8", 64L, "t-7"); + EntityReference app5 = appHelper.createNewApp("tApp5", 65L, "t-4"); + EntityReference app6 = appHelper.createNewApp("tApp6", 63L, "t-5"); + EntityReference app7 = appHelper.createNewApp("tApp7", 66L, "t-6"); + EntityReference app8 = appHelper.createNewApp("tApp8", 64L, "t-7"); + + appIds.clear(); + appIds.add(app5.id()); + appIds.add(app6.id()); + appIds.add(app7.id()); + appIds.add(app8.id()); - allApps = applicationService.findAll(); + allApps = applicationService.findByIds(appIds); appList.clear(); appList = allApps.stream().map( @@ -148,8 +159,6 @@ public void testBulkUploadWithTargetCategory() { t.externalId().orElse("not"))) .collect(Collectors.toList()); - appList = appList.subList(4, appList.size()); - final EntityReference measurable1ref = EntityReference.mkRef( EntityKind.MEASURABLE, measurable1.id().get(), @@ -163,8 +172,8 @@ public void testBulkUploadWithTargetCategory() { testInput = testInput.concat(mkGoodTestRow("t-0222", "M_0", "changed desc")); testInput = testInput.concat(mkGoodTestRow("t-1222", "M_0", "changed desc")); - BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 28L); - BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 28L, "test_user"); + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, relationshipKind.id().get(), "test_user"); // expecting 4 relationships to be added assertEquals(4l, applyResult.recordsAdded().longValue()); @@ -181,11 +190,24 @@ public void testBulkUploadWithSourceCategory() { userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); - List appList = new ArrayList(); + long newMeasurableId = measurableHelper.createMeasurable("M_1", measurableHelper.createMeasurableCategory("testCat1")); + + EntityReference app9 = appHelper.createNewApp("tApp9", 65L, "t-9"); + EntityReference app10 = appHelper.createNewApp("tApp10", 63L, "t-10"); + EntityReference app11 = appHelper.createNewApp("tApp11", 66L, "t-11"); + EntityReference app12 = appHelper.createNewApp("tApp12", 64L, "t-12"); + + List appIds = new ArrayList<>(); + appIds.add(app9.id()); + appIds.add(app10.id()); + appIds.add(app11.id()); + appIds.add(app12.id()); - List allApps = applicationService.findAll(); - Measurable measurable1 = measurableService.getById(1l); + List appList; + + List allApps = applicationService.findByIds(appIds); + Measurable measurable1 = measurableService.getById(newMeasurableId); appList = allApps.stream().map( t -> EntityReference.mkRef(t.kind(), @@ -195,24 +217,31 @@ public void testBulkUploadWithSourceCategory() { t.externalId().orElse("not"))) .collect(Collectors.toList()); - appList = appList.subList(0, appList.size() - 4); - relationshipKindService.create(ImmutableRelationshipKind .builder() .kindB(EntityKind.APPLICATION) .kindA(EntityKind.MEASURABLE) .categoryA(measurable1.categoryId()) - .code("RELATES_TO") - .name("Relates to") - .reverseName("Is related by") - .description("test rk") + .code("RELATES_TO_NEW") + .name("Relates to new") + .reverseName("Is related by new") + .description("test rk new") .position(1) .build()); - RelationshipKind relationshipKind = relationshipKindService.getById(29L); + Collection relationshipKinds = relationshipKindService.findAll(); - // inserting relationships in the db to test against - for(int i = 0; i < appList.size(); i++) { + RelationshipKind relationshipKind = relationshipKinds + .stream() + .filter(t -> t.kindA() == EntityKind.MEASURABLE + && t.kindB() == EntityKind.APPLICATION + && t.categoryA() != null + && t.categoryA() == measurable1.categoryId()) + .collect(Collectors.toList()) + .get(0); + + // inserting two relationships in the db to test against + for(int i = 0; i < appList.size() - 2; i++) { EntityReference t = appList.get(i); entityRelationshipService.createRelationship( ImmutableEntityRelationship @@ -229,18 +258,7 @@ public void testBulkUploadWithSourceCategory() { .build()); } - allApps = applicationService.findAll(); - appList.clear(); - - appList = allApps.stream().map( - t -> EntityReference.mkRef(t.kind(), - t.id().get(), - t.name(), - t.description(), - t.externalId().orElse("not"))) - .collect(Collectors.toList()); - - appList = appList.subList(4, appList.size()); + appList = appList.subList(2, appList.size()); final EntityReference measurable1ref = EntityReference.mkRef( EntityKind.MEASURABLE, @@ -250,16 +268,16 @@ public void testBulkUploadWithSourceCategory() { measurable1.externalId().get()); String testInput = mkGoodTestInput(measurable1ref, appList, "testDesc"); - testInput = testInput.concat(mkGoodTestRow("M_0", "t-0", "changed desc")); - testInput = testInput.concat(mkGoodTestRow("M_0", "t-1", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("M_1", "t-9", "changed desc")); + testInput = testInput.concat(mkGoodTestRow("M_1", "t-10", "changed desc")); testInput = testInput.concat(mkGoodTestRow("M_12", "t-0", "changed desc")); testInput = testInput.concat(mkGoodTestRow("M_1222", "t-0", "changed desc")); - BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 29L); - BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 29L, "test_user"); + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult applyResult = bulkUploadRelationshipService.bulkApply(previewResult, relationshipKind.id().get(), "test_user"); // expecting 4 relationships to be added - assertEquals(4l, applyResult.recordsAdded().longValue()); + assertEquals(2l, applyResult.recordsAdded().longValue()); // expecting 2 records to be updated assertEquals(2l, applyResult.recordsUpdated().longValue()); @@ -268,6 +286,166 @@ public void testBulkUploadWithSourceCategory() { assertEquals(2l, applyResult.skippedRows().longValue()); } + @Test + public void testBulkUploadWithSourceAndTargetCategory() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + long measurableId = measurableHelper.createMeasurable("M_2", measurableHelper.createMeasurableCategory("testCat2")); + final Measurable measurable = measurableService.getById(measurableId); + + long measurableId1 = measurableHelper.createMeasurable("M_3", measurableHelper.createMeasurableCategory("testCat3")); + final Measurable measurable1 = measurableService.getById(measurableId1); + + relationshipKindService.create(ImmutableRelationshipKind + .builder() + .name("Measures") + .reverseName("Is measured by") + .kindA(EntityKind.MEASURABLE) + .kindB(EntityKind.MEASURABLE) + .position(3) + .code("MEASURED_BY") + .categoryA(measurable.categoryId()) + .categoryB(measurable1.categoryId()) + .description("description") + .build()); + + List relationshipKindList = relationshipKindService.findAll().stream().collect(Collectors.toList()); + RelationshipKind relationshipKind = null; + + for(int i = 0; i < relationshipKindList.size(); i++) { + RelationshipKind rk = relationshipKindList.get(i); + if(rk.kindA() == EntityKind.MEASURABLE + && rk.kindB() == EntityKind.MEASURABLE + && (rk.categoryA() != null && rk.categoryA().longValue() == measurable.categoryId()) + && (rk.categoryA() != null && rk.categoryB().longValue() == measurable1.categoryId())){ + relationshipKind = rk; + break; + } + } + + String header = "source_external_id\ttarget_external_id\tdescription\n"; + String addInput = header + mkGoodTestRow(measurable.externalId().get(), + measurable1.externalId().get(), + "new relation"); + + String updateInput = header + mkGoodTestRow(measurable.externalId().get(), + measurable1.externalId().get(), + "description"); + + String noneInput = updateInput + "M_2\tM_9\tdesc\n"; + + BulkUploadRelationshipValidationResult preview = bulkUploadRelationshipService.bulkPreview(addInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult addApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + preview = bulkUploadRelationshipService.bulkPreview(updateInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult updateApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + preview = bulkUploadRelationshipService.bulkPreview(noneInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult noneApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + assertEquals(1l, addApply.recordsAdded().longValue()); + assertEquals(1l, updateApply.recordsUpdated().longValue()); + assertEquals(2l, noneApply.skippedRows().longValue()); + } + + @Test + public void testbulkUploadWithoutCategory() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + EntityReference app9 = appHelper.createNewApp("tApp13", 65L, "t-13"); + EntityReference app10 = appHelper.createNewApp("tApp14", 63L, "t-14"); + EntityReference app11 = appHelper.createNewApp("tApp15", 66L, "t-15"); + EntityReference app12 = appHelper.createNewApp("tApp16", 64L, "t-16"); + + List appIds = new ArrayList<>(); + appIds.add(app9.id()); + appIds.add(app10.id()); + appIds.add(app11.id()); + appIds.add(app12.id()); + + List apps = applicationService.findByIds(appIds); + + List apprefs = apps + .stream() + .map(t -> { + return EntityReference.mkRef(EntityKind.APPLICATION, + t.id().get(), + t.name(), + t.description(), + t.externalId().get()); + }) + .collect(Collectors.toList()); + + relationshipKindService.create(ImmutableRelationshipKind.builder() + .position(4) + .kindA(EntityKind.APPLICATION) + .kindB(EntityKind.APPLICATION) + .name("Services") + .reverseName("Serviced by") + .code("SERVICE_TEST_2024") + .description("description") + .build()); + + RelationshipKind relationshipKind = relationshipKindService.findAll() + .stream() + .filter(t -> StringUtilities.safeEq(t.code(), "SERVICE_TEST_2024")) + .collect(Collectors.toList()) + .get(0); + + EntityReference app1 = apprefs.get(0); + + // create new relationships with 1 app + for(int i = 1; i < apprefs.size() - 2; i++) { + entityRelationshipService.createRelationship(ImmutableEntityRelationship + .builder() + .a(app1) + .b(apprefs.get(i)) + .relationship(relationshipKind.code()) + .lastUpdatedAt(LocalDateTime.now()) + .lastUpdatedBy("test_user") + .description("testDesc") + .provenance("testProv") + .build() + ); + } + + String addInput = mkGoodTestInput(app1, apprefs.subList(apprefs.size() - 2, apprefs.size()), "testDesc"); + String updateInput = mkGoodTestInput(app1, apprefs.subList(1, apprefs.size()), "changed desc"); + + BulkUploadRelationshipValidationResult preview = bulkUploadRelationshipService.bulkPreview(addInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult addApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + preview = bulkUploadRelationshipService.bulkPreview(updateInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult updateApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + preview = bulkUploadRelationshipService.bulkPreview(updateInput, relationshipKind.id().get()); + BulkUploadRelationshipApplyResult noneApply = bulkUploadRelationshipService.bulkApply(preview, relationshipKind.id().get(), "test_user"); + + assertEquals(2l, addApply.recordsAdded().longValue()); + assertEquals(3l, updateApply.recordsUpdated().longValue()); + assertEquals(3l, noneApply.skippedRows().longValue()); + } + + @Test + public void testUnauthorized() { + + userHelper.createUserWithSystemRoles("test_user", SetUtilities.asSet(SystemRole.ADMIN)); + + String header = "source_external_id\ttarget_external_id\tdescription\n"; + String testInput = header + mkGoodTestRow("asfsfee", "", ""); + testInput = testInput.concat(mkGoodTestRow("jde", "ere", "eerer")); + + BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 13L); + + try { + bulkUploadRelationshipService.bulkApply(previewResult, 13L, "error_test_user"); + } catch (Exception e) { + assertNotNull(e); + } + } + @Test public void testErrorInput() { @@ -277,11 +455,10 @@ public void testErrorInput() { testInput = testInput.concat(mkGoodTestRow("jde", "ere", "eerer")); BulkUploadRelationshipValidationResult previewResult = bulkUploadRelationshipService.bulkPreview(testInput, 13L); - assertNotNull(previewResult.parseError()); - BulkUploadRelationshipApplyResult applyResult = null; + try { - applyResult = bulkUploadRelationshipService.bulkApply(previewResult, 13L, "test_user"); + bulkUploadRelationshipService.bulkApply(previewResult, 13L, "test_user"); } catch (Exception e) { assertNotNull(e); } diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte index 1c2b68a201..dd60d19655 100644 --- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -2,7 +2,7 @@ import { displayError } from "../../../common/error-utils"; import LoadingPlaceholder from "../../../common/svelte/LoadingPlaceholder.svelte"; import SubSection from "../../../common/svelte/SubSection.svelte"; - import {entityRelationshipStore} from "../../../svelte-stores/entity-relationship-store.js"; + import { entityRelationshipStore } from "../../../svelte-stores/entity-relationship-store.js"; import _ from "lodash"; export let selectedRelationshipKind; From ddda3c9b181aa793d3b3f4f94d8751e1fe5484ad Mon Sep 17 00:00:00 2001 From: jainshrk Date: Fri, 18 Oct 2024 15:14:52 +0530 Subject: [PATCH 13/17] code cleanup --- .../client/svelte-stores/entity-relationship-store.js | 10 +++++----- .../bulk-relationships/BulkRelationshipUpload.svelte | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/waltz-ng/client/svelte-stores/entity-relationship-store.js b/waltz-ng/client/svelte-stores/entity-relationship-store.js index 46bb5010d8..982e6baaad 100644 --- a/waltz-ng/client/svelte-stores/entity-relationship-store.js +++ b/waltz-ng/client/svelte-stores/entity-relationship-store.js @@ -81,17 +81,17 @@ export function mkEntityRelationshipStore() { }; const bulkUploadRelationshipsPreview = (relationshipKindId, data) => remote - .execute( + .execute( "POST", `api/entity-relationship/bulk/preview/${relationshipKindId}`, data ); const bulkUploadRelationshipsApply = (relationshipKindId, data) => remote - .execute( - "POST", - `api/entity-relationship/bulk/apply/${relationshipKindId}`, - data + .execute( + "POST", + `api/entity-relationship/bulk/apply/${relationshipKindId}`, + data ); diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte index dd60d19655..371fc17eb8 100644 --- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte +++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte @@ -91,16 +91,16 @@

The bulk upload format should look like, column order is not important but the headers are:

-
Source External Id
+
sourceExternalId
This uniquely identifies the source (from) entity for that relationship.
-
Target External Id
+
targetExternalId
This uniquely identifies the target (to) entity for that relationship.
-
Description
+
description
This optionally adds a comment to the change.
- For Example: + An example input: 
 sourceExternalId	 targetExternalId	 description

From e222908eec0ad950bb8713f81cc2b168d12e7e8c Mon Sep 17 00:00:00 2001
From: jainshrk 
Date: Fri, 18 Oct 2024 15:15:26 +0530
Subject: [PATCH 14/17] code cleanup

---
 .../BulkUploadRelationshipService.java        | 46 +++++++++----------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
index c68b8666d1..823773d962 100644
--- a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
+++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
@@ -39,10 +39,12 @@ public class BulkUploadRelationshipService {
 
     private static final Logger LOG = LoggerFactory.getLogger(BulkUploadRelationshipService.class);
 
-    private final RelationshipKindService relationshipKindService;
-
     private static final String PROVENANCE = "bulkUploadRelationships";
 
+    private static final org.finos.waltz.schema.tables.EntityRelationship er = Tables.ENTITY_RELATIONSHIP;
+
+    private final RelationshipKindService relationshipKindService;
+
     private final EntityRelationshipService entityRelationshipService;
 
     private final UserRoleService userRoleService;
@@ -64,8 +66,6 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel
 
         RelationshipKind relationshipKind = relationshipKindService.getById(relationshipKindId);
         BulkUploadRelationshipParsedResult parsedResult = new BulkUploadRelationshipItemParser().parse(input, BulkUploadRelationshipItemParser.InputFormat.TSV);
-        Map sourceEntityRefMap;
-        Map targetEntityRefMap;
 
         if (parsedResult.error() != null) {
             return ImmutableBulkUploadRelationshipValidationResult
@@ -75,18 +75,10 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel
         }
 
         // load source entity ref map
-        if (relationshipKind.categoryA() != null) {
-            sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryA()));
-        } else {
-            sourceEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindA());
-        }
+        Map sourceEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindA(), Optional.ofNullable(relationshipKind.categoryA()));
 
         // load target entity ref map
-        if (relationshipKind.categoryB() != null) {
-            targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB(), mkRef(EntityKind.MEASURABLE_CATEGORY, relationshipKind.categoryB()));
-        } else {
-            targetEntityRefMap = loadExternalIdToEntityRefMap(dsl, relationshipKind.kindB());
-        }
+        Map targetEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindB(), Optional.ofNullable(relationshipKind.categoryB()));
 
         // enrich each row to a tuple
         final Map finalSourceEntityRefMap = sourceEntityRefMap;
@@ -211,15 +203,15 @@ public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidat
                 .stream()
                 .filter(d -> d.uploadOperation() == UploadOperation.UPDATE && d.error().isEmpty())
                 .map(d -> DSL
-                        .update(Tables.ENTITY_RELATIONSHIP)
-                        .set(Tables.ENTITY_RELATIONSHIP.DESCRIPTION, d.description())
-                        .set(Tables.ENTITY_RELATIONSHIP.LAST_UPDATED_AT, now)
-                        .set(Tables.ENTITY_RELATIONSHIP.LAST_UPDATED_BY, user)
-                        .where(Tables.ENTITY_RELATIONSHIP.RELATIONSHIP.eq(relationshipKind.code()))
-                            .and(Tables.ENTITY_RELATIONSHIP.KIND_A.eq(relationshipKind.kindA().name()))
-                            .and(Tables.ENTITY_RELATIONSHIP.KIND_B.eq(relationshipKind.kindB().name()))
-                            .and(Tables.ENTITY_RELATIONSHIP.ID_A.eq(d.sourceEntityRef().id()))
-                            .and(Tables.ENTITY_RELATIONSHIP.ID_B.eq(d.targetEntityRef().id())))
+                        .update(er)
+                        .set(er.DESCRIPTION, d.description())
+                        .set(er.LAST_UPDATED_AT, now)
+                        .set(er.LAST_UPDATED_BY, user)
+                        .where(er.RELATIONSHIP.eq(relationshipKind.code()))
+                            .and(er.KIND_A.eq(relationshipKind.kindA().name()))
+                            .and(er.KIND_B.eq(relationshipKind.kindB().name()))
+                            .and(er.ID_A.eq(d.sourceEntityRef().id()))
+                            .and(er.ID_B.eq(d.targetEntityRef().id())))
                 .collect(Collectors.toSet());
 
         final Set auditLogs = preview
@@ -271,6 +263,14 @@ public BulkUploadRelationshipApplyResult bulkApply(BulkUploadRelationshipValidat
                 });
     }
 
+    private Map loadExternalIdMap(DSLContext dsl,
+                              EntityKind entityKind, Optional category) {
+        if(!category.isPresent()) {
+            return loadExternalIdToEntityRefMap(dsl, entityKind);
+        }   else  {
+            return loadExternalIdToEntityRefMap(dsl, entityKind, mkRef(EntityKind.MEASURABLE_CATEGORY, category.get()));
+        }
+    }
     private String mkChangeMessage(EntityReference sourceRef, EntityReference targetRef, RelationshipKind relationshipKind, UploadOperation uploadOperation) {
         return format(
                 "Bulk Relationships Update - Operation %s, Relationship b/w %s -> %s, Relationship Kind %s. %s",

From e14b65ad221850b7380927c4ed720b53bf36ee19 Mon Sep 17 00:00:00 2001
From: jainshrk 
Date: Fri, 18 Oct 2024 15:32:16 +0530
Subject: [PATCH 15/17] add comment

---
 .../svelte/bulk-relationships/BulkRelationshipUpload.svelte      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
index 371fc17eb8..70747b6d4c 100644
--- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
+++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
@@ -5,6 +5,7 @@
 	import { entityRelationshipStore } from "../../../svelte-stores/entity-relationship-store.js";
 	import _ from "lodash";
 
+	// prop passed onto this component
     export let selectedRelationshipKind;
 
 	const MODES = {

From d2458dfe1af5fc15a68a7b285227a597e65add66 Mon Sep 17 00:00:00 2001
From: jainshrk 
Date: Fri, 18 Oct 2024 15:37:16 +0530
Subject: [PATCH 16/17] cleanup

---
 .../entity_relationship/BulkUploadRelationshipService.java  | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
index 823773d962..e2a8167f02 100644
--- a/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
+++ b/waltz-service/src/main/java/org/finos/waltz/service/entity_relationship/BulkUploadRelationshipService.java
@@ -75,14 +75,12 @@ public BulkUploadRelationshipValidationResult bulkPreview(String input, Long rel
         }
 
         // load source entity ref map
-        Map sourceEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindA(), Optional.ofNullable(relationshipKind.categoryA()));
+        final Map sourceEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindA(), Optional.ofNullable(relationshipKind.categoryA()));
 
         // load target entity ref map
-        Map targetEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindB(), Optional.ofNullable(relationshipKind.categoryB()));
+        final Map targetEntityRefMap = loadExternalIdMap(dsl, relationshipKind.kindB(), Optional.ofNullable(relationshipKind.categoryB()));
 
         // enrich each row to a tuple
-        final Map finalSourceEntityRefMap = sourceEntityRefMap;
-        final Map finalTargetEntityRefMap = targetEntityRefMap;
         final List>> listOfRows = parsedResult
                 .parsedItems()
                 .stream()

From 0fd9aad736e2d47bc61f6fe133fd09ea33997c1e Mon Sep 17 00:00:00 2001
From: jainshrk 
Date: Fri, 18 Oct 2024 15:39:56 +0530
Subject: [PATCH 17/17] remove comment

---
 .../svelte/bulk-relationships/BulkRelationshipUpload.svelte      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
index 70747b6d4c..371fc17eb8 100644
--- a/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
+++ b/waltz-ng/client/system/svelte/bulk-relationships/BulkRelationshipUpload.svelte
@@ -5,7 +5,6 @@
 	import { entityRelationshipStore } from "../../../svelte-stores/entity-relationship-store.js";
 	import _ from "lodash";
 
-	// prop passed onto this component
     export let selectedRelationshipKind;
 
 	const MODES = {