diff --git a/NEWS.md b/NEWS.md index 179ba7b5a..cbbd19ee8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,7 @@ ## v4.0.2 2024-12-06 +### Features +* Move Instance sub-entities population from database trigger to code ([MSEARCH-887](https://folio-org.atlassian.net/browse/MSEARCH-887)) + ### Bug fixes ### Dependencies diff --git a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java index b65fea9ed..868b7a6fa 100644 --- a/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java +++ b/src/main/java/org/folio/search/cql/CqlTermQueryConverter.java @@ -13,8 +13,8 @@ import java.util.Map; import java.util.Optional; import java.util.StringJoiner; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.folio.search.cql.builders.TermQueryBuilder; import org.folio.search.exception.RequestValidationException; import org.folio.search.exception.ValidationException; diff --git a/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java b/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java index 840122562..e369da8bb 100644 --- a/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java +++ b/src/main/java/org/folio/search/integration/message/interceptor/PopulateInstanceBatchInterceptor.java @@ -10,6 +10,7 @@ import static org.folio.search.utils.SearchUtils.SOURCE_CONSORTIUM_PREFIX; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -22,6 +23,7 @@ import org.folio.search.domain.dto.ResourceEventType; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.model.types.ResourceType; +import org.folio.search.service.InstanceChildrenResourceService; import org.folio.search.service.consortium.ConsortiumTenantExecutor; import org.folio.search.service.reindex.jdbc.MergeRangeRepository; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; @@ -39,13 +41,16 @@ public class PopulateInstanceBatchInterceptor implements BatchInterceptor repositories; private final ConsortiumTenantExecutor executionService; private final SystemUserScopedExecutionService systemUserScopedExecutionService; + private final InstanceChildrenResourceService instanceChildrenResourceService; public PopulateInstanceBatchInterceptor(List repositories, ConsortiumTenantExecutor executionService, - SystemUserScopedExecutionService systemUserScopedExecutionService) { + SystemUserScopedExecutionService systemUserScopedExecutionService, + InstanceChildrenResourceService instanceChildrenResourceService) { this.repositories = repositories.stream().collect(Collectors.toMap(ReindexJdbcRepository::entityType, identity())); this.executionService = executionService; this.systemUserScopedExecutionService = systemUserScopedExecutionService; + this.instanceChildrenResourceService = instanceChildrenResourceService; } @Override @@ -55,7 +60,7 @@ public ConsumerRecords intercept(ConsumerRecords isInstanceEvent(r.value())) .collect(Collectors.groupingBy(ConsumerRecord::key)); - List consumerRecords = new ArrayList<>(); + var consumerRecords = new ArrayList(); for (var entry : recordsById.entrySet()) { var list = entry.getValue(); if (list.size() > 1) { @@ -113,6 +118,11 @@ private void process(String tenant, List batch) { if (!idsToDrop.isEmpty()) { repository.deleteEntities(idsToDrop); } + + if (ResourceType.INSTANCE.getName().equals(recordCollection.getKey())) { + var noShadowCopiesInstanceEvents = recordByOperation.values().stream().flatMap(Collection::stream).toList(); + instanceChildrenResourceService.persistChildren(tenant, noShadowCopiesInstanceEvents); + } } } diff --git a/src/main/java/org/folio/search/model/BrowseResult.java b/src/main/java/org/folio/search/model/BrowseResult.java index bbc8966d5..d8ff1e099 100644 --- a/src/main/java/org/folio/search/model/BrowseResult.java +++ b/src/main/java/org/folio/search/model/BrowseResult.java @@ -8,7 +8,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; @Data @NoArgsConstructor diff --git a/src/main/java/org/folio/search/model/SearchResult.java b/src/main/java/org/folio/search/model/SearchResult.java index ab53d7da7..6bfba1dfe 100644 --- a/src/main/java/org/folio/search/model/SearchResult.java +++ b/src/main/java/org/folio/search/model/SearchResult.java @@ -6,7 +6,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; @Data @NoArgsConstructor diff --git a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java index d2ae58593..a5c960602 100644 --- a/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java +++ b/src/main/java/org/folio/search/service/InstanceChildrenResourceService.java @@ -8,10 +8,13 @@ import java.util.LinkedList; import java.util.List; +import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.domain.dto.ResourceEventType; import org.folio.search.model.event.SubResourceEvent; +import org.folio.search.service.consortium.ConsortiumTenantProvider; import org.folio.search.service.converter.preprocessor.extractor.ChildResourceExtractor; import org.folio.spring.tools.kafka.FolioMessageProducer; import org.springframework.stereotype.Component; @@ -27,6 +30,7 @@ public class InstanceChildrenResourceService { private final FolioMessageProducer messageProducer; private final List resourceExtractors; + private final ConsortiumTenantProvider consortiumTenantProvider; public void sendChildrenEvent(ResourceEvent event) { var needChildrenEvent = false; @@ -62,7 +66,7 @@ public List extractChildren(ResourceEvent event) { var events = new LinkedList(); if (isUpdateEventForResourceSharing(event)) { - for (ChildResourceExtractor resourceExtractor : resourceExtractors) { + for (var resourceExtractor : resourceExtractors) { events.addAll(resourceExtractor.prepareEventsOnSharing(event)); } } else if (startsWith(getResourceSource(event), SOURCE_CONSORTIUM_PREFIX)) { @@ -70,7 +74,7 @@ public List extractChildren(ResourceEvent event) { "processChildren::Finished instance children event processing. No additional action for shadow instance."); return events; } else { - for (ChildResourceExtractor resourceExtractor : resourceExtractors) { + for (var resourceExtractor : resourceExtractors) { events.addAll(resourceExtractor.prepareEvents(event)); } } @@ -81,4 +85,20 @@ public List extractChildren(ResourceEvent event) { return events; } + public void persistChildren(String tenantId, List events) { + var shared = consortiumTenantProvider.isCentralTenant(tenantId); + resourceExtractors.forEach(resourceExtractor -> resourceExtractor.persistChildren(shared, events)); + } + + public void persistChildrenOnReindex(String tenantId, List> instances) { + var events = instances.stream() + .map(instance -> new ResourceEvent() + .id(instance.get("id").toString()) + .type(ResourceEventType.REINDEX) + .tenant(tenantId) + ._new(instance)) + .toList(); + persistChildren(tenantId, events); + } + } diff --git a/src/main/java/org/folio/search/service/ResourceIdService.java b/src/main/java/org/folio/search/service/ResourceIdService.java index 95b93ce75..3acab31e9 100644 --- a/src/main/java/org/folio/search/service/ResourceIdService.java +++ b/src/main/java/org/folio/search/service/ResourceIdService.java @@ -16,7 +16,7 @@ import java.util.function.Consumer; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.configuration.properties.StreamIdsProperties; import org.folio.search.cql.CqlSearchQueryConverter; import org.folio.search.exception.SearchServiceException; diff --git a/src/main/java/org/folio/search/service/browse/CallNumberBrowseRangeService.java b/src/main/java/org/folio/search/service/browse/CallNumberBrowseRangeService.java index 074c27630..2489ac0d1 100644 --- a/src/main/java/org/folio/search/service/browse/CallNumberBrowseRangeService.java +++ b/src/main/java/org/folio/search/service/browse/CallNumberBrowseRangeService.java @@ -4,7 +4,7 @@ import static java.util.Collections.emptyList; import static java.util.function.Function.identity; import static java.util.stream.Stream.concat; -import static org.apache.commons.collections.CollectionUtils.isNotEmpty; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import static org.folio.search.model.types.ResourceType.INSTANCE; import static org.folio.search.utils.CollectionUtils.toLinkedHashMap; import static org.opensearch.index.query.QueryBuilders.existsQuery; diff --git a/src/main/java/org/folio/search/service/browse/CallNumberBrowseResultConverter.java b/src/main/java/org/folio/search/service/browse/CallNumberBrowseResultConverter.java index 386229f3a..38ae3d059 100644 --- a/src/main/java/org/folio/search/service/browse/CallNumberBrowseResultConverter.java +++ b/src/main/java/org/folio/search/service/browse/CallNumberBrowseResultConverter.java @@ -22,7 +22,7 @@ import java.util.Optional; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.folio.search.domain.dto.CallNumberBrowseItem; import org.folio.search.domain.dto.Instance; diff --git a/src/main/java/org/folio/search/service/browse/CallNumberBrowseService.java b/src/main/java/org/folio/search/service/browse/CallNumberBrowseService.java index a03d9cb45..2e5a2eb37 100644 --- a/src/main/java/org/folio/search/service/browse/CallNumberBrowseService.java +++ b/src/main/java/org/folio/search/service/browse/CallNumberBrowseService.java @@ -2,7 +2,7 @@ import static java.lang.Boolean.TRUE; import static java.util.Collections.singletonList; -import static org.apache.commons.collections.CollectionUtils.isEmpty; +import static org.apache.commons.collections4.CollectionUtils.isEmpty; import static org.folio.search.model.types.CallNumberTypeSource.FOLIO; import static org.folio.search.utils.CollectionUtils.mergeSafelyToList; diff --git a/src/main/java/org/folio/search/service/browse/SubjectBrowseService.java b/src/main/java/org/folio/search/service/browse/SubjectBrowseService.java index 5d5569798..dd02753f4 100644 --- a/src/main/java/org/folio/search/service/browse/SubjectBrowseService.java +++ b/src/main/java/org/folio/search/service/browse/SubjectBrowseService.java @@ -3,6 +3,8 @@ import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; import static org.folio.search.utils.SearchUtils.MISSING_FIRST_PROP; import static org.folio.search.utils.SearchUtils.MISSING_LAST_PROP; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; @@ -32,9 +34,6 @@ @RequiredArgsConstructor public class SubjectBrowseService extends AbstractBrowseServiceBySearchAfter { - private static final String SUBJECT_SOURCE_ID_FIELD = "sourceId"; - private static final String SUBJECT_TYPE_ID_FIELD = "typeId"; - private ConsortiumSearchHelper consortiumSearchHelper; @Autowired diff --git a/src/main/java/org/folio/search/service/converter/SearchFieldsProcessor.java b/src/main/java/org/folio/search/service/converter/SearchFieldsProcessor.java index 55867ecb6..721c6d376 100644 --- a/src/main/java/org/folio/search/service/converter/SearchFieldsProcessor.java +++ b/src/main/java/org/folio/search/service/converter/SearchFieldsProcessor.java @@ -9,7 +9,7 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.folio.search.model.converter.ConversionContext; import org.folio.search.model.metadata.SearchFieldDescriptor; diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractor.java index e512198c3..ebde8786f 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractor.java @@ -1,13 +1,81 @@ package org.folio.search.service.converter.preprocessor.extractor; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static org.apache.commons.collections4.MapUtils.getObject; +import static org.folio.search.utils.SearchConverterUtils.getNewAsMap; + +import java.util.HashSet; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import lombok.RequiredArgsConstructor; import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.domain.dto.ResourceEventType; +import org.folio.search.service.reindex.jdbc.InstanceChildResourceRepository; +import org.springframework.transaction.annotation.Transactional; + +@RequiredArgsConstructor +public abstract class ChildResourceExtractor { + + private final InstanceChildResourceRepository repository; + + public abstract List prepareEvents(ResourceEvent resource); + + public abstract List prepareEventsOnSharing(ResourceEvent resource); + + public abstract boolean hasChildResourceChanges(ResourceEvent event); + + protected abstract List> constructRelations(boolean shared, ResourceEvent event, + List> entities); + + protected abstract Map constructEntity(Map entityProperties); + + protected abstract String childrenFieldName(); + + @Transactional + public void persistChildren(boolean shared, List events) { + var instanceIdsForDeletion = events.stream() + .filter(event -> event.getType() != ResourceEventType.CREATE && event.getType() != ResourceEventType.REINDEX) + .map(ResourceEvent::getId) + .toList(); + if (!instanceIdsForDeletion.isEmpty()) { + repository.deleteByInstanceIds(instanceIdsForDeletion); + } -public interface ChildResourceExtractor { + var eventsForSaving = events.stream() + .filter(event -> event.getType() != ResourceEventType.DELETE) + .toList(); + if (eventsForSaving.isEmpty()) { + return; + } - List prepareEvents(ResourceEvent resource); + var entities = new HashSet>(); + var relations = new LinkedList>(); + eventsForSaving.forEach(event -> { + var entitiesFromEvent = extractEntities(event); + relations.addAll(constructRelations(shared, event, entitiesFromEvent)); + entities.addAll(entitiesFromEvent); + }); + repository.saveAll(entities, relations); + } - List prepareEventsOnSharing(ResourceEvent resource); + private List> extractEntities(ResourceEvent event) { + var entities = getChildResources(getNewAsMap(event)); + return entities.stream() + .map(this::constructEntity) + .filter(Objects::nonNull) + .toList(); + } - boolean hasChildResourceChanges(ResourceEvent event); + @SuppressWarnings("unchecked") + protected Set> getChildResources(Map event) { + var object = getObject(event, childrenFieldName(), emptyList()); + if (object == null) { + return emptySet(); + } + return new HashSet<>((List>) object); + } } diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ClassificationResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ClassificationResourceExtractor.java index 85f87aaa8..4fa63a050 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ClassificationResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ClassificationResourceExtractor.java @@ -1,8 +1,7 @@ package org.folio.search.service.converter.preprocessor.extractor.impl; import static java.util.Collections.emptyList; -import static java.util.Collections.emptySet; -import static org.apache.commons.collections4.MapUtils.getObject; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.CollectionUtils.subtract; @@ -11,14 +10,15 @@ import static org.folio.search.utils.SearchUtils.CLASSIFICATIONS_FIELD; import static org.folio.search.utils.SearchUtils.CLASSIFICATION_NUMBER_FIELD; import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; +import static org.folio.search.utils.SearchUtils.prepareForExpectedFormat; import java.util.ArrayList; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.MapUtils; import org.folio.search.domain.dto.ResourceEvent; @@ -38,12 +38,19 @@ @Log4j2 @Component -@RequiredArgsConstructor -public class ClassificationResourceExtractor implements ChildResourceExtractor { +public class ClassificationResourceExtractor extends ChildResourceExtractor { private final JsonConverter jsonConverter; private final FeatureConfigService featureConfigService; - private final ClassificationRepository classificationRepository; + private final ClassificationRepository repository; + + public ClassificationResourceExtractor(ClassificationRepository repository, JsonConverter jsonConverter, + FeatureConfigService featureConfigService) { + super(repository); + this.jsonConverter = jsonConverter; + this.featureConfigService = featureConfigService; + this.repository = repository; + } @Override public List prepareEvents(ResourceEvent event) { @@ -51,8 +58,8 @@ public List prepareEvents(ResourceEvent event) { return emptyList(); } - var oldClassifications = getClassifications(getOldAsMap(event)); - var newClassifications = getClassifications(getNewAsMap(event)); + var oldClassifications = getChildResources(getOldAsMap(event)); + var newClassifications = getChildResources(getNewAsMap(event)); if (oldClassifications.equals(newClassifications)) { return emptyList(); @@ -68,7 +75,7 @@ public List prepareEvents(ResourceEvent event) { idsForFetch.addAll(idsForCreate); idsForFetch.addAll(idsForDelete); - var entityAggList = classificationRepository.fetchByIds(idsForFetch); + var entityAggList = repository.fetchByIds(idsForFetch); var list = getResourceEventsForDeletion(idsForDelete, entityAggList, tenant); var list1 = entityAggList.stream() @@ -83,9 +90,9 @@ public List prepareEventsOnSharing(ResourceEvent event) { return emptyList(); } - var classifications = getClassifications(getOldAsMap(event)); + var classifications = getChildResources(getOldAsMap(event)); - if (!classifications.equals(getClassifications(getNewAsMap(event)))) { + if (!classifications.equals(getChildResources(getNewAsMap(event)))) { log.warn("Classifications are different on Update for instance sharing"); return emptyList(); } @@ -93,7 +100,7 @@ public List prepareEventsOnSharing(ResourceEvent event) { var tenant = event.getTenant(); var entitiesForDelete = toIds(classifications); - var entityAggList = classificationRepository.fetchByIds(entitiesForDelete); + var entityAggList = repository.fetchByIds(entitiesForDelete); return entityAggList.stream() .map(entities -> toResourceEvent(entities, tenant)) @@ -102,12 +109,45 @@ public List prepareEventsOnSharing(ResourceEvent event) { @Override public boolean hasChildResourceChanges(ResourceEvent event) { - var oldClassifications = getClassifications(getOldAsMap(event)); - var newClassifications = getClassifications(getNewAsMap(event)); + var oldClassifications = getChildResources(getOldAsMap(event)); + var newClassifications = getChildResources(getNewAsMap(event)); return !oldClassifications.equals(newClassifications); } + @Override + protected List> constructRelations(boolean shared, ResourceEvent event, + List> entities) { + return entities.stream() + .map(entity -> Map.of("instanceId", event.getId(), + "classificationId", entity.get("id"), + "tenantId", event.getTenant(), + "shared", shared)) + .toList(); + } + + @Override + protected Map constructEntity(Map entityProperties) { + var classificationNumber = prepareForExpectedFormat(entityProperties.get(CLASSIFICATION_NUMBER_FIELD), 50); + if (classificationNumber.isEmpty()) { + return null; + } + + var classificationTypeId = entityProperties.get(CLASSIFICATION_TYPE_FIELD); + var id = ShaUtils.sha(classificationNumber, Objects.toString(classificationTypeId, EMPTY)); + + var entity = new HashMap(); + entity.put("id", id); + entity.put(CLASSIFICATION_NUMBER_FIELD, classificationNumber); + entity.put(CLASSIFICATION_TYPE_FIELD, classificationTypeId); + return entity; + } + + @Override + protected String childrenFieldName() { + return CLASSIFICATIONS_FIELD; + } + private List getResourceEventsForDeletion(List idsForDelete, List entityAggList, String tenant) { @@ -157,13 +197,4 @@ private List toIds(Set> subtract) { MapUtils.getString(map, CLASSIFICATION_TYPE_FIELD))) .collect(Collectors.toCollection(ArrayList::new)); } - - @SuppressWarnings("unchecked") - private Set> getClassifications(Map event) { - var object = getObject(event, CLASSIFICATIONS_FIELD, emptyList()); - if (object == null) { - return emptySet(); - } - return new HashSet<>((List>) object); - } } diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ContributorResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ContributorResourceExtractor.java index 46dda3e67..5c0b201f4 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ContributorResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/ContributorResourceExtractor.java @@ -1,22 +1,24 @@ package org.folio.search.service.converter.preprocessor.extractor.impl; import static java.util.Collections.emptyList; -import static java.util.Collections.emptySet; -import static org.apache.commons.collections4.MapUtils.getObject; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.CollectionUtils.subtract; import static org.folio.search.utils.SearchConverterUtils.getNewAsMap; import static org.folio.search.utils.SearchConverterUtils.getOldAsMap; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; import static org.folio.search.utils.SearchUtils.CONTRIBUTORS_FIELD; +import static org.folio.search.utils.SearchUtils.CONTRIBUTOR_TYPE_FIELD; +import static org.folio.search.utils.SearchUtils.prepareForExpectedFormat; import java.util.ArrayList; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.MapUtils; import org.folio.search.domain.dto.ResourceEvent; @@ -34,16 +36,21 @@ @Log4j2 @Component -@RequiredArgsConstructor -public class ContributorResourceExtractor implements ChildResourceExtractor { +public class ContributorResourceExtractor extends ChildResourceExtractor { private final JsonConverter jsonConverter; - private final ContributorRepository contributorRepository; + private final ContributorRepository repository; + + public ContributorResourceExtractor(ContributorRepository repository, JsonConverter jsonConverter) { + super(repository); + this.jsonConverter = jsonConverter; + this.repository = repository; + } @Override public List prepareEvents(ResourceEvent event) { - var oldEntities = getEntities(getOldAsMap(event)); - var newEntities = getEntities(getNewAsMap(event)); + var oldEntities = getChildResources(getOldAsMap(event)); + var newEntities = getChildResources(getNewAsMap(event)); if (oldEntities.equals(newEntities)) { return emptyList(); @@ -60,7 +67,7 @@ public List prepareEvents(ResourceEvent event) { idsForFetch.addAll(idsForCreate); idsForFetch.addAll(idsForDelete); - var entityAggList = contributorRepository.fetchByIds(idsForFetch); + var entityAggList = repository.fetchByIds(idsForFetch); var list = getResourceEventsForDeletion(idsForDelete, entityAggList, tenant); var list1 = entityAggList.stream() @@ -71,9 +78,9 @@ public List prepareEvents(ResourceEvent event) { @Override public List prepareEventsOnSharing(ResourceEvent event) { - var entities = getEntities(getOldAsMap(event)); + var entities = getChildResources(getOldAsMap(event)); - if (!entities.equals(getEntities(getNewAsMap(event)))) { + if (!entities.equals(getChildResources(getNewAsMap(event)))) { log.warn("Contributors are different on Update for instance sharing"); return emptyList(); } @@ -81,7 +88,7 @@ public List prepareEventsOnSharing(ResourceEvent event) { var tenant = event.getTenant(); var ids = toIds(entities); - var entityAggList = contributorRepository.fetchByIds(ids); + var entityAggList = repository.fetchByIds(ids); return entityAggList.stream() .map(e -> toResourceEvent(e, tenant)) @@ -90,12 +97,50 @@ public List prepareEventsOnSharing(ResourceEvent event) { @Override public boolean hasChildResourceChanges(ResourceEvent event) { - var oldContributors = getEntities(getOldAsMap(event)); - var newContributors = getEntities(getNewAsMap(event)); + var oldContributors = getChildResources(getOldAsMap(event)); + var newContributors = getChildResources(getNewAsMap(event)); return !oldContributors.equals(newContributors); } + @Override + protected List> constructRelations(boolean shared, ResourceEvent event, + List> entities) { + return entities.stream() + .map(entity -> Map.of("instanceId", event.getId(), + "contributorId", entity.get("id"), + CONTRIBUTOR_TYPE_FIELD, entity.remove(CONTRIBUTOR_TYPE_FIELD), + "tenantId", event.getTenant(), + "shared", shared)) + .toList(); + } + + @Override + protected Map constructEntity(Map entityProperties) { + var contributorName = prepareForExpectedFormat(entityProperties.get("name"), 255); + if (contributorName.isBlank()) { + return null; + } + + var nameTypeId = entityProperties.get("contributorNameTypeId"); + var authorityId = entityProperties.get(AUTHORITY_ID_FIELD); + var id = ShaUtils.sha(contributorName, Objects.toString(nameTypeId, EMPTY), Objects.toString(authorityId, EMPTY)); + var typeId = entityProperties.get(CONTRIBUTOR_TYPE_FIELD); + + var entity = new HashMap(); + entity.put("id", id); + entity.put("name", contributorName); + entity.put("nameTypeId", nameTypeId); + entity.put(AUTHORITY_ID_FIELD, authorityId); + entity.put(CONTRIBUTOR_TYPE_FIELD, Objects.toString(typeId, EMPTY)); + return entity; + } + + @Override + protected String childrenFieldName() { + return CONTRIBUTORS_FIELD; + } + private List getResourceEventsForDeletion(List idsForDelete, List entityAggList, String tenant) { @@ -145,16 +190,7 @@ private List toIds(Set> subtract) { return subtract.stream() .map(map -> getEntityId(defaultIfBlank(MapUtils.getString(map, "name"), ""), MapUtils.getString(map, "contributorNameTypeId"), - MapUtils.getString(map, "authorityId"))) + MapUtils.getString(map, AUTHORITY_ID_FIELD))) .collect(Collectors.toCollection(ArrayList::new)); } - - @SuppressWarnings("unchecked") - private Set> getEntities(Map event) { - var object = getObject(event, CONTRIBUTORS_FIELD, emptyList()); - if (object == null) { - return emptySet(); - } - return new HashSet<>((List>) object); - } } diff --git a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/SubjectResourceExtractor.java b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/SubjectResourceExtractor.java index 22644876f..e239b6a41 100644 --- a/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/SubjectResourceExtractor.java +++ b/src/main/java/org/folio/search/service/converter/preprocessor/extractor/impl/SubjectResourceExtractor.java @@ -1,22 +1,26 @@ package org.folio.search.service.converter.preprocessor.extractor.impl; import static java.util.Collections.emptyList; -import static java.util.Collections.emptySet; -import static org.apache.commons.collections4.MapUtils.getObject; +import static org.apache.commons.lang3.StringUtils.EMPTY; import static org.apache.commons.lang3.StringUtils.defaultIfBlank; import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.CollectionUtils.subtract; import static org.folio.search.utils.SearchConverterUtils.getNewAsMap; import static org.folio.search.utils.SearchConverterUtils.getOldAsMap; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; import static org.folio.search.utils.SearchUtils.SUBJECTS_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; +import static org.folio.search.utils.SearchUtils.prepareForExpectedFormat; import java.util.ArrayList; -import java.util.HashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.MapUtils; import org.folio.search.domain.dto.ResourceEvent; @@ -34,16 +38,21 @@ @Log4j2 @Component -@RequiredArgsConstructor -public class SubjectResourceExtractor implements ChildResourceExtractor { +public class SubjectResourceExtractor extends ChildResourceExtractor { private final JsonConverter jsonConverter; - private final SubjectRepository subjectRepository; + private final SubjectRepository repository; + + public SubjectResourceExtractor(SubjectRepository repository, JsonConverter jsonConverter) { + super(repository); + this.repository = repository; + this.jsonConverter = jsonConverter; + } @Override public List prepareEvents(ResourceEvent event) { - var oldSubjects = getSubjects(getOldAsMap(event)); - var newSubjects = getSubjects(getNewAsMap(event)); + var oldSubjects = getChildResources(getOldAsMap(event)); + var newSubjects = getChildResources(getNewAsMap(event)); if (oldSubjects.equals(newSubjects)) { return emptyList(); @@ -60,7 +69,7 @@ public List prepareEvents(ResourceEvent event) { idsForFetch.addAll(idsForCreate); idsForFetch.addAll(idsForDelete); - var entityAggList = subjectRepository.fetchByIds(idsForFetch); + var entityAggList = repository.fetchByIds(idsForFetch); var list = getResourceEventsForDeletion(idsForDelete, entityAggList, tenant); var list1 = entityAggList.stream() @@ -71,9 +80,9 @@ public List prepareEvents(ResourceEvent event) { @Override public List prepareEventsOnSharing(ResourceEvent event) { - var subjects = getSubjects(getOldAsMap(event)); + var subjects = getChildResources(getOldAsMap(event)); - if (!subjects.equals(getSubjects(getNewAsMap(event)))) { + if (!subjects.equals(getChildResources(getNewAsMap(event)))) { log.warn("Subjects are different on Update for instance sharing"); return emptyList(); } @@ -81,7 +90,7 @@ public List prepareEventsOnSharing(ResourceEvent event) { var tenant = event.getTenant(); var ids = toIds(subjects); - var entityAggList = subjectRepository.fetchByIds(ids); + var entityAggList = repository.fetchByIds(ids); return entityAggList.stream() .map(entities -> toResourceEvent(entities, tenant)) @@ -90,12 +99,50 @@ public List prepareEventsOnSharing(ResourceEvent event) { @Override public boolean hasChildResourceChanges(ResourceEvent event) { - var oldSubjects = getSubjects(getOldAsMap(event)); - var newSubjects = getSubjects(getNewAsMap(event)); + var oldSubjects = getChildResources(getOldAsMap(event)); + var newSubjects = getChildResources(getNewAsMap(event)); return !oldSubjects.equals(newSubjects); } + @Override + protected List> constructRelations(boolean shared, ResourceEvent event, + List> entities) { + return entities.stream() + .map(entity -> Map.of("instanceId", event.getId(), + "subjectId", entity.get("id"), + "tenantId", event.getTenant(), + "shared", shared)) + .toList(); + } + + @Override + protected Map constructEntity(Map entityProperties) { + var subjectValue = prepareForExpectedFormat(entityProperties.get(SUBJECT_VALUE_FIELD), 255); + if (subjectValue.isEmpty()) { + return null; + } + + var authorityId = entityProperties.get(AUTHORITY_ID_FIELD); + var sourceId = entityProperties.get(SUBJECT_SOURCE_ID_FIELD); + var typeId = entityProperties.get(SUBJECT_TYPE_ID_FIELD); + var id = ShaUtils.sha(subjectValue, + Objects.toString(authorityId, EMPTY), Objects.toString(sourceId, EMPTY), Objects.toString(typeId, EMPTY)); + + var entity = new HashMap(); + entity.put("id", id); + entity.put(SUBJECT_VALUE_FIELD, subjectValue); + entity.put(AUTHORITY_ID_FIELD, authorityId); + entity.put(SUBJECT_SOURCE_ID_FIELD, sourceId); + entity.put(SUBJECT_TYPE_ID_FIELD, typeId); + return entity; + } + + @Override + protected String childrenFieldName() { + return SUBJECTS_FIELD; + } + private List getResourceEventsForDeletion(List idsForDelete, List entityAggList, String tenant) { @@ -143,19 +190,10 @@ private String getEntityId(String number, String authorityId, String sourceId, S @NotNull private List toIds(Set> subtract) { return subtract.stream() - .map(map -> getEntityId(defaultIfBlank(MapUtils.getString(map, "value"), ""), - MapUtils.getString(map, "authorityId"), - MapUtils.getString(map, "sourceId"), - MapUtils.getString(map, "typeId"))) + .map(map -> getEntityId(defaultIfBlank(MapUtils.getString(map, SUBJECT_VALUE_FIELD), ""), + MapUtils.getString(map, AUTHORITY_ID_FIELD), + MapUtils.getString(map, SUBJECT_SOURCE_ID_FIELD), + MapUtils.getString(map, SUBJECT_TYPE_ID_FIELD))) .collect(Collectors.toCollection(ArrayList::new)); } - - @SuppressWarnings("unchecked") - private Set> getSubjects(Map event) { - var object = getObject(event, SUBJECTS_FIELD, emptyList()); - if (object == null) { - return emptySet(); - } - return new HashSet<>((List>) object); - } } diff --git a/src/main/java/org/folio/search/service/metadata/LocalSearchFieldProvider.java b/src/main/java/org/folio/search/service/metadata/LocalSearchFieldProvider.java index b74b80455..50767c0d7 100644 --- a/src/main/java/org/folio/search/service/metadata/LocalSearchFieldProvider.java +++ b/src/main/java/org/folio/search/service/metadata/LocalSearchFieldProvider.java @@ -33,7 +33,7 @@ import java.util.stream.Stream; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.cql.SearchFieldModifier; import org.folio.search.exception.ResourceDescriptionException; import org.folio.search.model.Pair; diff --git a/src/main/java/org/folio/search/service/metadata/ResourceDescriptionService.java b/src/main/java/org/folio/search/service/metadata/ResourceDescriptionService.java index 6f92d7764..35c947111 100644 --- a/src/main/java/org/folio/search/service/metadata/ResourceDescriptionService.java +++ b/src/main/java/org/folio/search/service/metadata/ResourceDescriptionService.java @@ -17,7 +17,7 @@ import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.folio.search.exception.ResourceDescriptionException; import org.folio.search.model.metadata.ResourceDescription; diff --git a/src/main/java/org/folio/search/service/reindex/InstanceFetchService.java b/src/main/java/org/folio/search/service/reindex/InstanceFetchService.java index d46387693..82a98686c 100644 --- a/src/main/java/org/folio/search/service/reindex/InstanceFetchService.java +++ b/src/main/java/org/folio/search/service/reindex/InstanceFetchService.java @@ -13,7 +13,7 @@ import java.util.Map; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Item; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.domain.dto.ResourceEventType; diff --git a/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java b/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java index d4c433987..5d239f473 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexMergeRangeIndexService.java @@ -16,6 +16,7 @@ import org.folio.search.model.reindex.MergeRangeEntity; import org.folio.search.model.types.InventoryRecordType; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.service.InstanceChildrenResourceService; import org.folio.search.service.reindex.jdbc.MergeRangeRepository; import org.springframework.stereotype.Service; @@ -26,14 +27,17 @@ public class ReindexMergeRangeIndexService { private final Map repositories; private final InventoryService inventoryService; private final ReindexConfigurationProperties reindexConfig; + private final InstanceChildrenResourceService instanceChildrenResourceService; public ReindexMergeRangeIndexService(List repositories, InventoryService inventoryService, - ReindexConfigurationProperties reindexConfig) { + ReindexConfigurationProperties reindexConfig, + InstanceChildrenResourceService instanceChildrenResourceService) { this.repositories = repositories.stream() .collect(Collectors.toMap(MergeRangeRepository::entityType, Function.identity())); this.inventoryService = inventoryService; this.reindexConfig = reindexConfig; + this.instanceChildrenResourceService = instanceChildrenResourceService; } public void saveMergeRanges(List ranges) { @@ -75,6 +79,9 @@ public void saveEntities(ReindexRecordsEvent event) { .toList(); repositories.get(event.getRecordType().getEntityType()).saveEntities(event.getTenant(), entities); + if (event.getRecordType() == ReindexRecordsEvent.ReindexRecordType.INSTANCE) { + instanceChildrenResourceService.persistChildrenOnReindex(event.getTenant(), entities); + } } private List constructMergeRangeRecords(int recordsCount, diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java index 0a26026a2..60f01d291 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ClassificationRepository.java @@ -1,6 +1,9 @@ package org.folio.search.service.reindex.jdbc; import static org.folio.search.utils.JdbcUtils.getParamPlaceholder; +import static org.folio.search.utils.JdbcUtils.getParamPlaceholderForUuid; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_NUMBER_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; import java.sql.ResultSet; import java.sql.SQLException; @@ -9,6 +12,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.folio.search.configuration.properties.ReindexConfigurationProperties; @@ -18,12 +23,14 @@ import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Repository; +@Log4j2 @Repository -public class ClassificationRepository extends UploadRangeRepository { +public class ClassificationRepository extends UploadRangeRepository implements InstanceChildResourceRepository { private static final String SELECT_QUERY = """ SELECT @@ -61,6 +68,22 @@ public class ClassificationRepository extends UploadRangeRepository { c.id; """; + private static final String DELETE_QUERY = """ + DELETE + FROM %s.instance_classification + WHERE instance_id IN (%s); + """; + private static final String INSERT_ENTITIES_SQL = """ + INSERT INTO %s.classification (id, number, type_id) + VALUES (?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String INSERT_RELATIONS_SQL = """ + INSERT INTO %s.instance_classification (instance_id, classification_id, tenant_id, shared) + VALUES (?::uuid, ?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.classification_id >= ? AND ins.classification_id <= ?"; private static final String ID_RANGE_CLAS_WHERE_CLAUSE = "c.id >= ? AND c.id <= ?"; private static final String IDS_INS_WHERE_CLAUSE = "ins.classification_id IN (%1$s)"; @@ -126,6 +149,48 @@ protected RowMapper> rowToMapMapper() { }; } + @Override + public void deleteByInstanceIds(List instanceIds) { + var sql = DELETE_QUERY.formatted(JdbcUtils.getSchemaName(context), getParamPlaceholderForUuid(instanceIds.size())); + jdbcTemplate.update(sql, instanceIds.toArray()); + } + + @Override + public void saveAll(Set> entities, List> entityRelations) { + var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get(CLASSIFICATION_NUMBER_FIELD)); + statement.setObject(3, entity.get(CLASSIFICATION_TYPE_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save entities batch. Starting processing one-by-one", e); + for (var entity : entities) { + jdbcTemplate.update(entitiesSql, + entity.get("id"), entity.get(CLASSIFICATION_NUMBER_FIELD), entity.get(CLASSIFICATION_TYPE_FIELD)); + } + } + + var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(relationsSql, entityRelations, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("classificationId")); + statement.setString(3, (String) entityRelation.get("tenantId")); + statement.setObject(4, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save relations batch. Starting processing one-by-one", e); + for (var entityRelation : entityRelations) { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("classificationId"), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } + } + } + private RowMapper instanceClassificationAggRowMapper() { return (rs, rowNum) -> new InstanceClassificationEntityAgg( getId(rs), diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java index 8e1180179..a014e3860 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/ContributorRepository.java @@ -1,6 +1,9 @@ package org.folio.search.service.reindex.jdbc; import static org.folio.search.utils.JdbcUtils.getParamPlaceholder; +import static org.folio.search.utils.JdbcUtils.getParamPlaceholderForUuid; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.CONTRIBUTOR_TYPE_FIELD; import java.sql.ResultSet; import java.sql.SQLException; @@ -9,6 +12,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.folio.search.configuration.properties.ReindexConfigurationProperties; @@ -18,14 +23,16 @@ import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Repository; +@Log4j2 @Repository -public class ContributorRepository extends UploadRangeRepository { +public class ContributorRepository extends UploadRangeRepository implements InstanceChildResourceRepository { - public static final String SELECT_QUERY = """ + private static final String SELECT_QUERY = """ SELECT c.id, c.name, @@ -64,6 +71,22 @@ public class ContributorRepository extends UploadRangeRepository { c.id; """; + private static final String DELETE_QUERY = """ + DELETE + FROM %s.instance_contributor + WHERE instance_id IN (%s); + """; + private static final String INSERT_ENTITIES_SQL = """ + INSERT INTO %s.contributor (id, name, name_type_id, authority_id) + VALUES (?, ?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String INSERT_RELATIONS_SQL = """ + INSERT INTO %s.instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared) + VALUES (?::uuid, ?, ?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.contributor_id >= ? AND ins.contributor_id <= ?"; private static final String ID_RANGE_CONTR_WHERE_CLAUSE = "c.id >= ? AND c.id <= ?"; private static final String IDS_INS_WHERE_CLAUSE = "ins.contributor_id IN (%1$s)"; @@ -120,7 +143,7 @@ protected RowMapper> rowToMapMapper() { contributor.put("id", getId(rs)); contributor.put("name", getName(rs)); contributor.put("contributorNameTypeId", getNameTypeId(rs)); - contributor.put("authorityId", getAuthorityId(rs)); + contributor.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)); contributor.put("instances", maps); @@ -129,6 +152,50 @@ protected RowMapper> rowToMapMapper() { }; } + @Override + public void deleteByInstanceIds(List instanceIds) { + var sql = DELETE_QUERY.formatted(JdbcUtils.getSchemaName(context), getParamPlaceholderForUuid(instanceIds.size())); + jdbcTemplate.update(sql, instanceIds.toArray()); + } + + @Override + public void saveAll(Set> entities, List> entityRelations) { + var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get("name")); + statement.setObject(3, entity.get("nameTypeId")); + statement.setObject(4, entity.get(AUTHORITY_ID_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save entities batch. Starting processing one-by-one", e); + for (var entity : entities) { + jdbcTemplate.update(entitiesSql, + entity.get("id"), entity.get("name"), entity.get("nameTypeId"), entity.get(AUTHORITY_ID_FIELD)); + } + } + + var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(relationsSql, entityRelations, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("contributorId")); + statement.setString(3, (String) entityRelation.get(CONTRIBUTOR_TYPE_FIELD)); + statement.setString(4, (String) entityRelation.get("tenantId")); + statement.setObject(5, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save relations batch. Starting processing one-by-one", e); + for (var entityRelation : entityRelations) { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("contributorId"), + entityRelation.get(CONTRIBUTOR_TYPE_FIELD), entityRelation.get("tenantId"), entityRelation.get("shared")); + } + } + } + private RowMapper instanceAggRowMapper() { return (rs, rowNum) -> new InstanceContributorEntityAgg( getId(rs), diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/InstanceChildResourceRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/InstanceChildResourceRepository.java new file mode 100644 index 000000000..b550551d2 --- /dev/null +++ b/src/main/java/org/folio/search/service/reindex/jdbc/InstanceChildResourceRepository.java @@ -0,0 +1,12 @@ +package org.folio.search.service.reindex.jdbc; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface InstanceChildResourceRepository { + + void deleteByInstanceIds(List instanceIds); + + void saveAll(Set> entities, List> entityRelations); +} diff --git a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java index a45155efb..57a489892 100644 --- a/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java +++ b/src/main/java/org/folio/search/service/reindex/jdbc/SubjectRepository.java @@ -1,6 +1,11 @@ package org.folio.search.service.reindex.jdbc; import static org.folio.search.utils.JdbcUtils.getParamPlaceholder; +import static org.folio.search.utils.JdbcUtils.getParamPlaceholderForUuid; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; import java.sql.ResultSet; import java.sql.SQLException; @@ -9,6 +14,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.ListUtils; import org.folio.search.configuration.properties.ReindexConfigurationProperties; @@ -18,14 +25,16 @@ import org.folio.search.utils.JdbcUtils; import org.folio.search.utils.JsonConverter; import org.folio.spring.FolioExecutionContext; +import org.springframework.dao.DataAccessException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Repository; +@Log4j2 @Repository -public class SubjectRepository extends UploadRangeRepository { +public class SubjectRepository extends UploadRangeRepository implements InstanceChildResourceRepository { - public static final String SELECT_QUERY = """ + private static final String SELECT_QUERY = """ SELECT s.id, s.value, @@ -63,6 +72,22 @@ public class SubjectRepository extends UploadRangeRepository { s.id; """; + private static final String DELETE_QUERY = """ + DELETE + FROM %s.instance_subject + WHERE instance_id IN (%s); + """; + private static final String INSERT_ENTITIES_SQL = """ + INSERT INTO %s.subject (id, value, authority_id, source_id, type_id) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String INSERT_RELATIONS_SQL = """ + INSERT INTO %s.instance_subject (instance_id, subject_id, tenant_id, shared) + VALUES (?::uuid, ?, ?, ?) + ON CONFLICT DO NOTHING; + """; + private static final String ID_RANGE_INS_WHERE_CLAUSE = "ins.subject_id >= ? AND ins.subject_id <= ?"; private static final String ID_RANGE_SUBJ_WHERE_CLAUSE = "s.id >= ? AND s.id <= ?"; private static final String IDS_INS_WHERE_CLAUSE = "ins.subject_id IN (%1$s)"; @@ -121,8 +146,8 @@ protected RowMapper> rowToMapMapper() { return (rs, rowNum) -> { Map subject = new HashMap<>(); subject.put("id", getId(rs)); - subject.put("value", getValue(rs)); - subject.put("authorityId", getAuthorityId(rs)); + subject.put(SUBJECT_VALUE_FIELD, getValue(rs)); + subject.put(AUTHORITY_ID_FIELD, getAuthorityId(rs)); var maps = jsonConverter.fromJsonToListOfMaps(getInstances(rs)); subject.put("instances", maps); @@ -131,6 +156,50 @@ protected RowMapper> rowToMapMapper() { }; } + @Override + public void deleteByInstanceIds(List instanceIds) { + var sql = DELETE_QUERY.formatted(JdbcUtils.getSchemaName(context), getParamPlaceholderForUuid(instanceIds.size())); + jdbcTemplate.update(sql, instanceIds.toArray()); + } + + @Override + public void saveAll(Set> entities, List> entityRelations) { + var entitiesSql = INSERT_ENTITIES_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(entitiesSql, entities, BATCH_OPERATION_SIZE, + (statement, entity) -> { + statement.setString(1, (String) entity.get("id")); + statement.setString(2, (String) entity.get(SUBJECT_VALUE_FIELD)); + statement.setString(3, (String) entity.get(AUTHORITY_ID_FIELD)); + statement.setString(4, (String) entity.get(SUBJECT_SOURCE_ID_FIELD)); + statement.setString(5, (String) entity.get(SUBJECT_TYPE_ID_FIELD)); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save entities batch. Starting processing one-by-one", e); + for (var entity : entities) { + jdbcTemplate.update(entitiesSql, entity.get("id"), entity.get(SUBJECT_VALUE_FIELD), + entity.get(AUTHORITY_ID_FIELD), entity.get(SUBJECT_SOURCE_ID_FIELD), entity.get(SUBJECT_TYPE_ID_FIELD)); + } + } + + var relationsSql = INSERT_RELATIONS_SQL.formatted(JdbcUtils.getSchemaName(context)); + try { + jdbcTemplate.batchUpdate(relationsSql, entityRelations, BATCH_OPERATION_SIZE, + (statement, entityRelation) -> { + statement.setObject(1, entityRelation.get("instanceId")); + statement.setString(2, (String) entityRelation.get("subjectId")); + statement.setString(3, (String) entityRelation.get("tenantId")); + statement.setObject(4, entityRelation.get("shared")); + }); + } catch (DataAccessException e) { + log.warn("saveAll::Failed to save relations batch. Starting processing one-by-one", e); + for (var entityRelation : entityRelations) { + jdbcTemplate.update(relationsSql, entityRelation.get("instanceId"), entityRelation.get("subjectId"), + entityRelation.get("tenantId"), entityRelation.get("shared")); + } + } + } + private RowMapper instanceAggRowMapper() { return (rs, rowNum) -> new InstanceSubjectEntityAgg( getId(rs), diff --git a/src/main/java/org/folio/search/service/setter/AbstractAllValuesProcessor.java b/src/main/java/org/folio/search/service/setter/AbstractAllValuesProcessor.java index e019e519b..c7812da92 100644 --- a/src/main/java/org/folio/search/service/setter/AbstractAllValuesProcessor.java +++ b/src/main/java/org/folio/search/service/setter/AbstractAllValuesProcessor.java @@ -9,8 +9,8 @@ import java.util.Map.Entry; import java.util.Set; import java.util.function.Predicate; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.folio.search.model.service.MultilangValue; diff --git a/src/main/java/org/folio/search/service/setter/holding/HoldingAllFieldValuesProcessor.java b/src/main/java/org/folio/search/service/setter/holding/HoldingAllFieldValuesProcessor.java index 099339747..74427eb40 100644 --- a/src/main/java/org/folio/search/service/setter/holding/HoldingAllFieldValuesProcessor.java +++ b/src/main/java/org/folio/search/service/setter/holding/HoldingAllFieldValuesProcessor.java @@ -1,6 +1,6 @@ package org.folio.search.service.setter.holding; -import static org.apache.commons.collections.MapUtils.getObject; +import static org.apache.commons.collections4.MapUtils.getObject; import static org.folio.search.utils.SearchUtils.INSTANCE_HOLDING_FIELD_NAME; import java.util.Map; diff --git a/src/main/java/org/folio/search/service/setter/holding/HoldingsPublicNotesProcessor.java b/src/main/java/org/folio/search/service/setter/holding/HoldingsPublicNotesProcessor.java index 16535f679..0ab41834b 100644 --- a/src/main/java/org/folio/search/service/setter/holding/HoldingsPublicNotesProcessor.java +++ b/src/main/java/org/folio/search/service/setter/holding/HoldingsPublicNotesProcessor.java @@ -4,7 +4,7 @@ import java.util.Collection; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Holding; import org.folio.search.domain.dto.Instance; import org.folio.search.domain.dto.Note; diff --git a/src/main/java/org/folio/search/service/setter/instance/AbstractTagsProcessor.java b/src/main/java/org/folio/search/service/setter/instance/AbstractTagsProcessor.java index 168725848..469a81f75 100644 --- a/src/main/java/org/folio/search/service/setter/instance/AbstractTagsProcessor.java +++ b/src/main/java/org/folio/search/service/setter/instance/AbstractTagsProcessor.java @@ -9,7 +9,7 @@ import java.util.Set; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.folio.search.domain.dto.Instance; import org.folio.search.domain.dto.Tags; diff --git a/src/main/java/org/folio/search/service/setter/instance/SortContributorsProcessor.java b/src/main/java/org/folio/search/service/setter/instance/SortContributorsProcessor.java index bdfdcfe44..add0fcc7a 100644 --- a/src/main/java/org/folio/search/service/setter/instance/SortContributorsProcessor.java +++ b/src/main/java/org/folio/search/service/setter/instance/SortContributorsProcessor.java @@ -3,7 +3,7 @@ import static org.apache.commons.lang3.BooleanUtils.isTrue; import lombok.RequiredArgsConstructor; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Instance; import org.folio.search.service.setter.FieldProcessor; import org.springframework.stereotype.Component; diff --git a/src/main/java/org/folio/search/service/setter/item/ItemAllFieldValuesProcessor.java b/src/main/java/org/folio/search/service/setter/item/ItemAllFieldValuesProcessor.java index bccdf670e..18d21bbfd 100644 --- a/src/main/java/org/folio/search/service/setter/item/ItemAllFieldValuesProcessor.java +++ b/src/main/java/org/folio/search/service/setter/item/ItemAllFieldValuesProcessor.java @@ -1,6 +1,6 @@ package org.folio.search.service.setter.item; -import static org.apache.commons.collections.MapUtils.getObject; +import static org.apache.commons.collections4.MapUtils.getObject; import static org.folio.search.utils.CollectionUtils.noneMatch; import static org.folio.search.utils.SearchUtils.INSTANCE_ITEM_FIELD_NAME; diff --git a/src/main/java/org/folio/search/service/setter/item/ItemPublicNotesProcessor.java b/src/main/java/org/folio/search/service/setter/item/ItemPublicNotesProcessor.java index 31112ed23..f695da89b 100644 --- a/src/main/java/org/folio/search/service/setter/item/ItemPublicNotesProcessor.java +++ b/src/main/java/org/folio/search/service/setter/item/ItemPublicNotesProcessor.java @@ -4,7 +4,7 @@ import java.util.Collection; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.CirculationNote; import org.folio.search.domain.dto.Instance; import org.folio.search.domain.dto.Item; diff --git a/src/main/java/org/folio/search/utils/CollectionUtils.java b/src/main/java/org/folio/search/utils/CollectionUtils.java index 3fcaa2755..e414c3b12 100644 --- a/src/main/java/org/folio/search/utils/CollectionUtils.java +++ b/src/main/java/org/folio/search/utils/CollectionUtils.java @@ -5,8 +5,8 @@ import static java.util.stream.Collectors.toCollection; import static java.util.stream.Stream.empty; import static java.util.stream.StreamSupport.stream; -import static org.apache.commons.collections.CollectionUtils.isEmpty; -import static org.apache.commons.collections.CollectionUtils.isNotEmpty; +import static org.apache.commons.collections4.CollectionUtils.isEmpty; +import static org.apache.commons.collections4.CollectionUtils.isNotEmpty; import java.util.ArrayList; import java.util.Collection; diff --git a/src/main/java/org/folio/search/utils/SearchUtils.java b/src/main/java/org/folio/search/utils/SearchUtils.java index fde786482..73699aa63 100644 --- a/src/main/java/org/folio/search/utils/SearchUtils.java +++ b/src/main/java/org/folio/search/utils/SearchUtils.java @@ -1,6 +1,8 @@ package org.folio.search.utils; import static java.util.Locale.ROOT; +import static org.apache.commons.lang3.StringUtils.EMPTY; +import static org.apache.commons.lang3.StringUtils.truncate; import static org.folio.search.utils.CollectionUtils.mergeSafelyToSet; import static org.folio.spring.config.properties.FolioEnvironment.getFolioEnvName; @@ -15,7 +17,7 @@ import java.util.regex.Pattern; import lombok.AccessLevel; import lombok.NoArgsConstructor; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.domain.dto.ShelvingOrderAlgorithmType; @@ -53,6 +55,10 @@ public class SearchUtils { public static final String CONTRIBUTORS_FIELD = "contributors"; public static final String CLASSIFICATION_NUMBER_FIELD = "classificationNumber"; public static final String CLASSIFICATION_TYPE_FIELD = "classificationTypeId"; + public static final String CONTRIBUTOR_TYPE_FIELD = "contributorTypeId"; + public static final String SUBJECT_VALUE_FIELD = "value"; + public static final String SUBJECT_TYPE_ID_FIELD = "typeId"; + public static final String SUBJECT_SOURCE_ID_FIELD = "sourceId"; public static final String SUBJECT_AGGREGATION_NAME = "subjects.value"; public static final String SOURCE_CONSORTIUM_PREFIX = "CONSORTIUM-"; @@ -327,6 +333,13 @@ public static String buildPreferenceKey(String tenantId, String resource, String return tenantId + "-" + resource + "-" + query; } + /** + * Coverts to non-null string, replaces backslash with double backslash and truncates to expected length. + * */ + public static String prepareForExpectedFormat(Object value, int length) { + return truncate(Objects.toString(value, EMPTY).replace("\\", "\\\\"), length); + } + private static Object getMultilangValueObject(Object value) { return value instanceof MultilangValue v ? v.getMultilangValues() : value; } diff --git a/src/main/resources/changelog/changelog-master.xml b/src/main/resources/changelog/changelog-master.xml index 7f4323fa7..a12c27aa2 100644 --- a/src/main/resources/changelog/changelog-master.xml +++ b/src/main/resources/changelog/changelog-master.xml @@ -10,4 +10,5 @@ + diff --git a/src/main/resources/changelog/changes/v4.0/delete-instance-trigger.xml b/src/main/resources/changelog/changes/v4.0/delete-instance-trigger.xml new file mode 100644 index 000000000..5188e3c6d --- /dev/null +++ b/src/main/resources/changelog/changes/v4.0/delete-instance-trigger.xml @@ -0,0 +1,12 @@ + + + + + DROP TRIGGER IF EXISTS instance_trigger ON instance CASCADE; + + + diff --git a/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java b/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java new file mode 100644 index 000000000..972fbc0c2 --- /dev/null +++ b/src/test/java/org/folio/search/service/InstanceChildrenResourceServiceTest.java @@ -0,0 +1,183 @@ +package org.folio.search.service; + +import static java.util.Collections.singletonList; +import static org.assertj.core.api.Assertions.assertThat; +import static org.folio.search.utils.SearchUtils.SOURCE_CONSORTIUM_PREFIX; +import static org.folio.search.utils.SearchUtils.SOURCE_FIELD; +import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.domain.dto.ResourceEventType; +import org.folio.search.model.event.SubResourceEvent; +import org.folio.search.service.consortium.ConsortiumTenantProvider; +import org.folio.search.service.converter.preprocessor.extractor.ChildResourceExtractor; +import org.folio.search.service.converter.preprocessor.extractor.impl.ClassificationResourceExtractor; +import org.folio.search.service.converter.preprocessor.extractor.impl.ContributorResourceExtractor; +import org.folio.search.service.converter.preprocessor.extractor.impl.SubjectResourceExtractor; +import org.folio.spring.testing.type.UnitTest; +import org.folio.spring.tools.kafka.FolioMessageProducer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class InstanceChildrenResourceServiceTest { + + @Mock + private FolioMessageProducer messageProducer; + @Mock + private ConsortiumTenantProvider consortiumTenantProvider; + @Mock + private ClassificationResourceExtractor classificationResourceExtractor; + @Mock + private ContributorResourceExtractor contributorResourceExtractor; + @Mock + private SubjectResourceExtractor subjectResourceExtractor; + + private List resourceExtractors; + private InstanceChildrenResourceService service; + + @BeforeEach + void setUp() { + this.resourceExtractors = + List.of(classificationResourceExtractor, contributorResourceExtractor, subjectResourceExtractor); + service = new InstanceChildrenResourceService(messageProducer, resourceExtractors, consortiumTenantProvider); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2}) + void sendChildrenEvent(int extractorIndex) { + var event = new ResourceEvent() + ._new(Map.of(SOURCE_FIELD, "MARC")); + var expectedEvent = SubResourceEvent.fromResourceEvent(event); + when(resourceExtractors.get(extractorIndex).hasChildResourceChanges(event)).thenReturn(true); + + service.sendChildrenEvent(event); + + verify(messageProducer, times(1)).sendMessages(singletonList(expectedEvent)); + } + + @ParameterizedTest + @ValueSource(ints = {0, 1, 2}) + void sendChildrenEvent_resourceSharing(int extractorIndex) { + var event = resourceSharingEvent(); + var expectedEvent = SubResourceEvent.fromResourceEvent(event); + for (int i = 0; i < resourceExtractors.size(); i++) { + if (i != extractorIndex) { + lenient().when(resourceExtractors.get(extractorIndex).hasChildResourceChanges(event)).thenReturn(true); + } + } + + service.sendChildrenEvent(event); + + verify(messageProducer, times(1)).sendMessages(singletonList(expectedEvent)); + } + + @ParameterizedTest + @ValueSource(strings = {"MARC", "CONSORTIUM_MARC"}) + void sendChildrenEvent_noEvent(String source) { + var event = new ResourceEvent() + ._new(Map.of(SOURCE_FIELD, source)); + resourceExtractors.forEach(resourceExtractor -> + when(resourceExtractor.hasChildResourceChanges(event)).thenReturn(false)); + + service.sendChildrenEvent(event); + + verifyNoInteractions(messageProducer); + } + + @Test + void sendChildrenEvent_resourceSharing_noEvent() { + var event = resourceSharingEvent(); + resourceExtractors.forEach(resourceExtractor -> + when(resourceExtractor.hasChildResourceChanges(event)).thenReturn(true)); + + service.sendChildrenEvent(event); + + verifyNoInteractions(messageProducer); + } + + @Test + void extractChildren() { + var event = new ResourceEvent(); + resourceExtractors.forEach(resourceExtractor -> + when(resourceExtractor.prepareEvents(event)).thenReturn(List.of(new ResourceEvent(), new ResourceEvent()))); + + var result = service.extractChildren(event); + + assertThat(result).hasSize(6); + } + + @Test + void extractChildren_resourceSharing() { + var event = resourceSharingEvent(); + resourceExtractors.forEach(resourceExtractor -> + when(resourceExtractor.prepareEventsOnSharing(event)) + .thenReturn(List.of(new ResourceEvent(), new ResourceEvent()))); + + var result = service.extractChildren(event); + + assertThat(result).hasSize(6); + } + + @Test + void extractChildren_shadowInstance() { + var event = new ResourceEvent() + ._new(Map.of(SOURCE_FIELD, SOURCE_CONSORTIUM_PREFIX + "MARC")); + + var result = service.extractChildren(event); + + assertThat(result).isEmpty(); + resourceExtractors.forEach(Mockito::verifyNoInteractions); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void persistChildren(boolean shared) { + var events = List.of(new ResourceEvent(), new ResourceEvent()); + when(consortiumTenantProvider.isCentralTenant(TENANT_ID)).thenReturn(shared); + + service.persistChildren(TENANT_ID, events); + + resourceExtractors.forEach(resourceExtractor -> + verify(resourceExtractor).persistChildren(shared, events)); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void persistChildrenOnReindex(boolean shared) { + var id1 = UUID.randomUUID(); + var id2 = UUID.randomUUID(); + var instances = List.of(Map.of("id", id1), Map.of("id", id2)); + var expectedEvents = List.of( + new ResourceEvent().id(id1.toString()).type(ResourceEventType.REINDEX).tenant(TENANT_ID)._new(instances.get(0)), + new ResourceEvent().id(id2.toString()).type(ResourceEventType.REINDEX).tenant(TENANT_ID)._new(instances.get(1))); + when(consortiumTenantProvider.isCentralTenant(TENANT_ID)).thenReturn(shared); + + service.persistChildrenOnReindex(TENANT_ID, instances); + + resourceExtractors.forEach(resourceExtractor -> + verify(resourceExtractor).persistChildren(shared, expectedEvents)); + } + + private ResourceEvent resourceSharingEvent() { + return new ResourceEvent() + .type(ResourceEventType.UPDATE) + ._new(Map.of(SOURCE_FIELD, SOURCE_CONSORTIUM_PREFIX + "MARC")) + .old(Map.of(SOURCE_FIELD, "MARC")); + } +} diff --git a/src/test/java/org/folio/search/service/converter/SearchFieldsProcessorTest.java b/src/test/java/org/folio/search/service/converter/SearchFieldsProcessorTest.java index 81ba7306f..e131e9055 100644 --- a/src/test/java/org/folio/search/service/converter/SearchFieldsProcessorTest.java +++ b/src/test/java/org/folio/search/service/converter/SearchFieldsProcessorTest.java @@ -17,7 +17,7 @@ import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.NONE; import java.util.Map; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.MapUtils; import org.folio.search.domain.dto.Instance; import org.folio.search.model.converter.ConversionContext; import org.folio.search.model.metadata.ResourceDescription; diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractorTestBase.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractorTestBase.java new file mode 100644 index 000000000..f569f3203 --- /dev/null +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ChildResourceExtractorTestBase.java @@ -0,0 +1,65 @@ +package org.folio.search.service.converter.preprocessor.extractor; + +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATIONS_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; +import static org.folio.search.utils.SearchUtils.CONTRIBUTORS_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECTS_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; +import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.verify; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.folio.search.domain.dto.ResourceEvent; +import org.folio.search.domain.dto.ResourceEventType; +import org.folio.search.service.reindex.jdbc.InstanceChildResourceRepository; + +public abstract class ChildResourceExtractorTestBase { + + void persistChildrenTest(ChildResourceExtractor extractor, InstanceChildResourceRepository repository, + Supplier> eventBodySupplier) { + var eventBody = eventBodySupplier.get(); + var events = List.of( + resourceEvent(ResourceEventType.CREATE, eventBody), + resourceEvent(ResourceEventType.REINDEX, eventBody), + resourceEvent(ResourceEventType.UPDATE, noMainValuesBody()), + resourceEvent(ResourceEventType.UPDATE, eventBodySupplier.get()), + resourceEvent(ResourceEventType.DELETE, eventBodySupplier.get())); + + var instanceIdsForDeletion = List.of(events.get(2).getId(), events.get(3).getId(), events.get(4).getId()); + + extractor.persistChildren(false, events); + + verify(repository).deleteByInstanceIds(instanceIdsForDeletion); + verify(repository).saveAll(argThat(set -> set.size() == 2), argThat(list -> list.size() == 3)); + } + + private Map noMainValuesBody() { + return Map.of(CONTRIBUTORS_FIELD, List.of(Map.of( + AUTHORITY_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_SOURCE_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_TYPE_ID_FIELD, UUID.randomUUID().toString() + )), + SUBJECTS_FIELD, List.of(Map.of( + AUTHORITY_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_SOURCE_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_TYPE_ID_FIELD, UUID.randomUUID().toString() + )), + CLASSIFICATIONS_FIELD, List.of(Map.of( + CLASSIFICATION_TYPE_FIELD, UUID.randomUUID().toString() + ))); + } + + private ResourceEvent resourceEvent(ResourceEventType type, Map body) { + return new ResourceEvent() + .id(UUID.randomUUID().toString()) + .type(type) + .tenant(TENANT_ID) + ._new(body); + } +} diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ClassificationResourceExtractorTest.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ClassificationResourceExtractorTest.java new file mode 100644 index 000000000..c1b1955c7 --- /dev/null +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ClassificationResourceExtractorTest.java @@ -0,0 +1,48 @@ +package org.folio.search.service.converter.preprocessor.extractor; + +import static org.folio.search.utils.SearchUtils.CLASSIFICATIONS_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_NUMBER_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.commons.lang3.RandomStringUtils; +import org.folio.search.service.FeatureConfigService; +import org.folio.search.service.converter.preprocessor.extractor.impl.ClassificationResourceExtractor; +import org.folio.search.service.reindex.jdbc.ClassificationRepository; +import org.folio.search.utils.JsonConverter; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class ClassificationResourceExtractorTest extends ChildResourceExtractorTestBase { + + @Mock + private JsonConverter jsonConverter; + @Mock + private FeatureConfigService featureConfigService; + @Mock + private ClassificationRepository repository; + + @InjectMocks + private ClassificationResourceExtractor extractor; + + @Test + void persistChildren() { + persistChildrenTest(extractor, repository, classificationsBodySupplier()); + } + + private static Supplier> classificationsBodySupplier() { + return () -> Map.of(CLASSIFICATIONS_FIELD, List.of(Map.of( + CLASSIFICATION_NUMBER_FIELD, RandomStringUtils.insecure().nextAlphanumeric(55), + CLASSIFICATION_TYPE_FIELD, UUID.randomUUID().toString() + ))); + } +} diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ContributorResourceExtractorTest.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ContributorResourceExtractorTest.java new file mode 100644 index 000000000..e14cece8c --- /dev/null +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/ContributorResourceExtractorTest.java @@ -0,0 +1,48 @@ +package org.folio.search.service.converter.preprocessor.extractor; + +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.CONTRIBUTORS_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.commons.lang3.RandomStringUtils; +import org.folio.search.service.converter.preprocessor.extractor.impl.ContributorResourceExtractor; +import org.folio.search.service.reindex.jdbc.ContributorRepository; +import org.folio.search.utils.JsonConverter; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class ContributorResourceExtractorTest extends ChildResourceExtractorTestBase { + + @Mock + private JsonConverter jsonConverter; + @Mock + private ContributorRepository repository; + + @InjectMocks + private ContributorResourceExtractor extractor; + + @Test + void persistChildren() { + persistChildrenTest(extractor, repository, contributorsBodySupplier()); + } + + private static Supplier> contributorsBodySupplier() { + return () -> Map.of(CONTRIBUTORS_FIELD, List.of(Map.of( + "name", RandomStringUtils.insecure().nextAlphanumeric(260), + AUTHORITY_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_SOURCE_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_TYPE_ID_FIELD, UUID.randomUUID().toString() + ))); + } +} diff --git a/src/test/java/org/folio/search/service/converter/preprocessor/extractor/SubjectResourceExtractorTest.java b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/SubjectResourceExtractorTest.java new file mode 100644 index 000000000..67d142c77 --- /dev/null +++ b/src/test/java/org/folio/search/service/converter/preprocessor/extractor/SubjectResourceExtractorTest.java @@ -0,0 +1,49 @@ +package org.folio.search.service.converter.preprocessor.extractor; + +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECTS_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; + +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.commons.lang3.RandomStringUtils; +import org.folio.search.service.converter.preprocessor.extractor.impl.SubjectResourceExtractor; +import org.folio.search.service.reindex.jdbc.SubjectRepository; +import org.folio.search.utils.JsonConverter; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@UnitTest +@ExtendWith(MockitoExtension.class) +class SubjectResourceExtractorTest extends ChildResourceExtractorTestBase { + + @Mock + private JsonConverter jsonConverter; + @Mock + private SubjectRepository repository; + + @InjectMocks + private SubjectResourceExtractor extractor; + + @Test + void persistChildren() { + persistChildrenTest(extractor, repository, subjectsBodySupplier()); + } + + private static Supplier> subjectsBodySupplier() { + return () -> Map.of(SUBJECTS_FIELD, List.of(Map.of( + SUBJECT_VALUE_FIELD, RandomStringUtils.insecure().nextAlphanumeric(260), + AUTHORITY_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_SOURCE_ID_FIELD, UUID.randomUUID().toString(), + SUBJECT_TYPE_ID_FIELD, UUID.randomUUID().toString() + ))); + } +} diff --git a/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java index bc444c118..8023d2b64 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexMergeRangeIndexServiceTest.java @@ -10,6 +10,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.sql.Timestamp; @@ -17,6 +18,8 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; import org.assertj.core.api.Condition; import org.folio.search.configuration.properties.ReindexConfigurationProperties; import org.folio.search.integration.folio.InventoryService; @@ -24,13 +27,18 @@ import org.folio.search.model.reindex.MergeRangeEntity; import org.folio.search.model.types.InventoryRecordType; import org.folio.search.model.types.ReindexEntityType; +import org.folio.search.service.InstanceChildrenResourceService; import org.folio.search.service.reindex.jdbc.HoldingRepository; import org.folio.search.service.reindex.jdbc.ItemRepository; +import org.folio.search.service.reindex.jdbc.MergeInstanceRepository; import org.folio.search.service.reindex.jdbc.MergeRangeRepository; +import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; import org.folio.spring.testing.type.UnitTest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; @@ -40,18 +48,24 @@ @ExtendWith(MockitoExtension.class) class ReindexMergeRangeIndexServiceTest { - private @Mock MergeRangeRepository repository; + private @Mock MergeInstanceRepository instanceRepository; private @Mock ItemRepository itemRepository; private @Mock HoldingRepository holdingRepository; private @Mock InventoryService inventoryService; private @Mock ReindexConfigurationProperties config; + private @Mock InstanceChildrenResourceService instanceChildrenResourceService; private ReindexMergeRangeIndexService service; + private Map repositoryMap; @BeforeEach void setUp() { - when(repository.entityType()).thenReturn(INSTANCE); - service = new ReindexMergeRangeIndexService(List.of(repository), inventoryService, config); + var repositories = List.of(instanceRepository, itemRepository, holdingRepository); + repositories.forEach(repository -> when(repository.entityType()).thenCallRealMethod()); + service = new ReindexMergeRangeIndexService( + repositories, inventoryService, config, instanceChildrenResourceService); + repositoryMap = repositories.stream() + .collect(Collectors.toMap(ReindexJdbcRepository::entityType, Function.identity())); } @Test @@ -60,7 +74,7 @@ void saveMergeRanges_positive() { service.saveMergeRanges(List.of()); // assert - verify(repository).saveMergeRanges(Mockito.anyList()); + verify(repositoryMap.values().iterator().next()).saveMergeRanges(Mockito.anyList()); } @Test @@ -92,22 +106,28 @@ void updateFinishDate() { service.updateFinishDate(ReindexEntityType.INSTANCE, rangeId.toString()); - verify(repository).setIndexRangeFinishDate(eq(rangeId), captor.capture()); + verify(instanceRepository).setIndexRangeFinishDate(eq(rangeId), captor.capture()); var timestamp = captor.getValue(); assertThat(timestamp).isAfterOrEqualTo(testStartTime); } - @Test - void saveEntities() { + @EnumSource(value = ReindexRecordsEvent.ReindexRecordType.class) + @ParameterizedTest + void saveEntities(ReindexRecordsEvent.ReindexRecordType recordType) { var entities = Map.of("id", UUID.randomUUID()); var event = new ReindexRecordsEvent(); event.setTenant(TENANT_ID); - event.setRecordType(ReindexRecordsEvent.ReindexRecordType.INSTANCE); + event.setRecordType(recordType); event.setRecords(List.of(entities)); service.saveEntities(event); - verify(repository).saveEntities(TENANT_ID, List.of(entities)); + verify(repositoryMap.get(recordType.getEntityType())).saveEntities(TENANT_ID, List.of(entities)); + if (recordType == ReindexRecordsEvent.ReindexRecordType.INSTANCE) { + verify(instanceChildrenResourceService).persistChildrenOnReindex(TENANT_ID, List.of(entities)); + } else { + verifyNoInteractions(instanceChildrenResourceService); + } } } diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java new file mode 100644 index 000000000..2c0ab55d4 --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ClassificationRepositoryIT.java @@ -0,0 +1,131 @@ +package org.folio.search.service.reindex.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_NUMBER_FIELD; +import static org.folio.search.utils.SearchUtils.CLASSIFICATION_TYPE_FIELD; +import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.folio.search.configuration.properties.ReindexConfigurationProperties; +import org.folio.search.utils.JsonConverter; +import org.folio.spring.FolioExecutionContext; +import org.folio.spring.FolioModuleMetadata; +import org.folio.spring.testing.extension.EnablePostgres; +import org.folio.spring.testing.type.IntegrationTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest; +import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.dao.InvalidDataAccessResourceUsageException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +@IntegrationTest +@JdbcTest +@EnablePostgres +@AutoConfigureJson +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +class ClassificationRepositoryIT { + + private @SpyBean JdbcTemplate jdbcTemplate; + private @MockBean FolioExecutionContext context; + private ClassificationRepository repository; + private ReindexConfigurationProperties properties; + + @BeforeEach + void setUp() { + properties = new ReindexConfigurationProperties(); + var jsonConverter = new JsonConverter(new ObjectMapper()); + repository = spy(new ClassificationRepository(jdbcTemplate, jsonConverter, context, properties)); + when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { + @Override + public String getModuleName() { + return null; + } + + @Override + public String getDBSchemaName(String tenantId) { + return "public"; + } + }); + when(context.getTenantId()).thenReturn(TENANT_ID); + } + + @Test + @Sql("/sql/populate-classifications.sql") + void deleteByInstanceIds_oneClassificationRemovedAndOneInstanceCounterDecremented() { + var instanceIds = List.of("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "b3bae8a9-cfb1-4afe-83d5-2cdae4580e07"); + + // act + repository.deleteByInstanceIds(instanceIds); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(1) + .extracting("number", "instances") + .contains( + tuple("Sci-Fi", List.of( + Map.of("count", 1, "shared", true, "tenantId", "consortium"), + Map.of("count", 1, "shared", false, "tenantId", "member_tenant")))); + } + + @Test + void saveAll() { + var entities = Set.of(classificationEntity("1"), classificationEntity("2")); + var entityRelations = List.of( + classificationRelation("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "1"), + classificationRelation("b3bae8a9-cfb1-4afe-83d5-2cdae4580e07", "2"), + classificationRelation("9ec55e4f-6a76-427c-b47b-197046f44a54", "2")); + + repository.saveAll(entities, entityRelations); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(2) + .extracting("number", "instances") + .contains( + tuple("number1", List.of(Map.of("count", 1, "shared", false, "tenantId", TENANT_ID))), + tuple("number2", List.of(Map.of("count", 2, "shared", false, "tenantId", TENANT_ID)))); + } + + @Test + void saveAll_batchFailure() { + doThrow(InvalidDataAccessResourceUsageException.class) + .when(jdbcTemplate).batchUpdate(anyString(), anyCollection(), anyInt(), any()); + + saveAll(); + } + + private Map classificationEntity(String id) { + return Map.of( + "id", id, + CLASSIFICATION_NUMBER_FIELD, "number" + id, + CLASSIFICATION_TYPE_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6" + ); + } + + private Map classificationRelation(String instanceId, String classificationId) { + return Map.of( + "instanceId", instanceId, + "classificationId", classificationId, + "tenantId", TENANT_ID, + "shared", false + ); + } +} diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java new file mode 100644 index 000000000..66e4059a9 --- /dev/null +++ b/src/test/java/org/folio/search/service/reindex/jdbc/ContributorRepositoryIT.java @@ -0,0 +1,137 @@ +package org.folio.search.service.reindex.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.CONTRIBUTOR_TYPE_FIELD; +import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.folio.search.configuration.properties.ReindexConfigurationProperties; +import org.folio.search.utils.JsonConverter; +import org.folio.spring.FolioExecutionContext; +import org.folio.spring.FolioModuleMetadata; +import org.folio.spring.testing.extension.EnablePostgres; +import org.folio.spring.testing.type.IntegrationTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest; +import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.dao.InvalidDataAccessResourceUsageException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.jdbc.Sql; + +@IntegrationTest +@JdbcTest +@EnablePostgres +@AutoConfigureJson +@AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) +class ContributorRepositoryIT { + + private @SpyBean JdbcTemplate jdbcTemplate; + private @MockBean FolioExecutionContext context; + private ContributorRepository repository; + private ReindexConfigurationProperties properties; + + @BeforeEach + void setUp() { + properties = new ReindexConfigurationProperties(); + var jsonConverter = new JsonConverter(new ObjectMapper()); + repository = spy(new ContributorRepository(jdbcTemplate, jsonConverter, context, properties)); + when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { + @Override + public String getModuleName() { + return null; + } + + @Override + public String getDBSchemaName(String tenantId) { + return "public"; + } + }); + when(context.getTenantId()).thenReturn(TENANT_ID); + } + + @Test + @Sql("/sql/populate-contributors.sql") + void deleteByInstanceIds_oneContributorRemovedAndOneInstanceCounterDecremented() { + var instanceIds = List.of("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "b3bae8a9-cfb1-4afe-83d5-2cdae4580e07"); + + // act + repository.deleteByInstanceIds(instanceIds); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(1) + .extracting("name", "instances") + .contains( + tuple("Sci-Fi", List.of( + Map.of("count", 1, "shared", true, "tenantId", "consortium", + "typeId", List.of("aab8fff4-49c6-4578-979e-439b6ba3600b")), + Map.of("count", 1, "shared", false, "tenantId", "member_tenant", + "typeId", List.of("9ec55e4f-6a76-427c-b47b-197046f44a53"))))); + } + + @Test + void saveAll() { + var entities = Set.of(contributorEntity("1"), contributorEntity("2")); + var entityRelations = List.of( + contributorRelation("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "1"), + contributorRelation("b3bae8a9-cfb1-4afe-83d5-2cdae4580e07", "2"), + contributorRelation("9ec55e4f-6a76-427c-b47b-197046f44a54", "2")); + + repository.saveAll(entities, entityRelations); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(2) + .extracting("name", "instances") + .contains( + tuple("name1", List.of(Map.of("count", 1, "shared", false, "tenantId", TENANT_ID, + "typeId", List.of("b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5")))), + tuple("name2", List.of(Map.of("count", 2, "shared", false, "tenantId", TENANT_ID, + "typeId", List.of("b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5"))))); + } + + @Test + void saveAll_batchFailure() { + doThrow(InvalidDataAccessResourceUsageException.class) + .when(jdbcTemplate).batchUpdate(anyString(), anyCollection(), anyInt(), any()); + + saveAll(); + } + + private Map contributorEntity(String id) { + return Map.of( + "id", id, + "name", "name" + id, + "nameTypeId", "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6", + AUTHORITY_ID_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5" + ); + } + + private Map contributorRelation(String instanceId, String contributorId) { + return Map.of( + "instanceId", instanceId, + "contributorId", contributorId, + CONTRIBUTOR_TYPE_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5", + "tenantId", TENANT_ID, + "shared", false + ); + } +} diff --git a/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java b/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java index aee594b63..1842f5263 100644 --- a/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java +++ b/src/test/java/org/folio/search/service/reindex/jdbc/SubjectRepositoryIT.java @@ -2,11 +2,23 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import static org.folio.search.utils.SearchUtils.AUTHORITY_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_SOURCE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_TYPE_ID_FIELD; +import static org.folio.search.utils.SearchUtils.SUBJECT_VALUE_FIELD; import static org.folio.search.utils.TestConstants.TENANT_ID; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.ObjectMapper; import java.util.List; +import java.util.Map; +import java.util.Set; import org.assertj.core.api.Condition; import org.folio.search.configuration.properties.ReindexConfigurationProperties; import org.folio.search.model.reindex.UploadRangeEntity; @@ -18,11 +30,12 @@ import org.folio.spring.testing.type.IntegrationTest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; import org.springframework.boot.test.autoconfigure.jdbc.JdbcTest; import org.springframework.boot.test.autoconfigure.json.AutoConfigureJson; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.dao.InvalidDataAccessResourceUsageException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.test.context.jdbc.Sql; @@ -33,7 +46,7 @@ @AutoConfigureTestDatabase(replace = AutoConfigureTestDatabase.Replace.NONE) class SubjectRepositoryIT { - private @Autowired JdbcTemplate jdbcTemplate; + private @SpyBean JdbcTemplate jdbcTemplate; private @MockBean FolioExecutionContext context; private SubjectRepository repository; private ReindexConfigurationProperties properties; @@ -42,7 +55,7 @@ class SubjectRepositoryIT { void setUp() { properties = new ReindexConfigurationProperties(); var jsonConverter = new JsonConverter(new ObjectMapper()); - repository = new SubjectRepository(jdbcTemplate, jsonConverter, context, properties); + repository = spy(new SubjectRepository(jdbcTemplate, jsonConverter, context, properties)); when(context.getFolioModuleMetadata()).thenReturn(new FolioModuleMetadata() { @Override public String getModuleName() { @@ -100,4 +113,70 @@ void fetchBy_returnListOfMaps() { tuple("Alternative History", null), tuple("History", "79144653-7a98-4dfb-aa6a-13ad49e80952")); } + + @Test + @Sql("/sql/populate-subjects.sql") + void deleteByInstanceIds_oneSubjectRemovedAndOneInstanceCounterDecremented() { + var instanceIds = List.of("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "b3bae8a9-cfb1-4afe-83d5-2cdae4580e07"); + + // act + repository.deleteByInstanceIds(instanceIds); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(16) + .extracting("value", "instances") + .contains( + tuple("Sci-Fi", List.of( + Map.of("count", 1, "shared", true, "tenantId", "consortium"), + Map.of("count", 1, "shared", false, "tenantId", "member_tenant")))); + } + + @Test + void saveAll() { + var entities = Set.of(subjectEntity("1"), subjectEntity("2")); + var entityRelations = List.of( + subjectRelation("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "1"), + subjectRelation("b3bae8a9-cfb1-4afe-83d5-2cdae4580e07", "2"), + subjectRelation("9ec55e4f-6a76-427c-b47b-197046f44a54", "2")); + + repository.saveAll(entities, entityRelations); + + // assert + var ranges = repository.fetchByIdRange("0", "50"); + assertThat(ranges) + .hasSize(2) + .extracting("value", "instances") + .contains( + tuple("value1", List.of(Map.of("count", 1, "shared", false, "tenantId", TENANT_ID))), + tuple("value2", List.of(Map.of("count", 2, "shared", false, "tenantId", TENANT_ID)))); + } + + @Test + void saveAll_batchFailure() { + doThrow(InvalidDataAccessResourceUsageException.class) + .when(jdbcTemplate).batchUpdate(anyString(), anyCollection(), anyInt(), any()); + + saveAll(); + } + + private Map subjectEntity(String id) { + return Map.of( + "id", id, + SUBJECT_VALUE_FIELD, "value" + id, + AUTHORITY_ID_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6", + SUBJECT_SOURCE_ID_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5", + SUBJECT_TYPE_ID_FIELD, "b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d4" + ); + } + + private Map subjectRelation(String instanceId, String subjectId) { + return Map.of( + "instanceId", instanceId, + "subjectId", subjectId, + "tenantId", TENANT_ID, + "shared", false + ); + } } diff --git a/src/test/java/org/folio/search/service/setter/authority/LccnAuthorityProcessorTest.java b/src/test/java/org/folio/search/service/setter/authority/LccnAuthorityProcessorTest.java index ca1126b6f..80471dce6 100644 --- a/src/test/java/org/folio/search/service/setter/authority/LccnAuthorityProcessorTest.java +++ b/src/test/java/org/folio/search/service/setter/authority/LccnAuthorityProcessorTest.java @@ -11,7 +11,7 @@ import java.util.Set; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Authority; import org.folio.search.domain.dto.Identifier; import org.folio.search.integration.folio.ReferenceDataService; diff --git a/src/test/java/org/folio/search/service/setter/instance/IsbnProcessorTest.java b/src/test/java/org/folio/search/service/setter/instance/IsbnProcessorTest.java index 0497b58a3..5459be6eb 100644 --- a/src/test/java/org/folio/search/service/setter/instance/IsbnProcessorTest.java +++ b/src/test/java/org/folio/search/service/setter/instance/IsbnProcessorTest.java @@ -15,7 +15,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Identifier; import org.folio.search.domain.dto.Instance; import org.folio.search.integration.folio.ReferenceDataService; diff --git a/src/test/java/org/folio/search/service/setter/instance/IssnProcessorTest.java b/src/test/java/org/folio/search/service/setter/instance/IssnProcessorTest.java index 141137b4e..0864ccb2e 100644 --- a/src/test/java/org/folio/search/service/setter/instance/IssnProcessorTest.java +++ b/src/test/java/org/folio/search/service/setter/instance/IssnProcessorTest.java @@ -16,7 +16,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Identifier; import org.folio.search.domain.dto.Instance; import org.folio.search.integration.folio.ReferenceDataService; diff --git a/src/test/java/org/folio/search/service/setter/instance/LccnInstanceProcessorTest.java b/src/test/java/org/folio/search/service/setter/instance/LccnInstanceProcessorTest.java index f3f75cb46..6f054d133 100644 --- a/src/test/java/org/folio/search/service/setter/instance/LccnInstanceProcessorTest.java +++ b/src/test/java/org/folio/search/service/setter/instance/LccnInstanceProcessorTest.java @@ -11,7 +11,7 @@ import java.util.Set; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Identifier; import org.folio.search.domain.dto.Instance; import org.folio.search.integration.folio.ReferenceDataService; diff --git a/src/test/java/org/folio/search/service/setter/instance/OclcProcessorTest.java b/src/test/java/org/folio/search/service/setter/instance/OclcProcessorTest.java index 71f94c70f..177f37d55 100644 --- a/src/test/java/org/folio/search/service/setter/instance/OclcProcessorTest.java +++ b/src/test/java/org/folio/search/service/setter/instance/OclcProcessorTest.java @@ -15,7 +15,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; -import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections4.CollectionUtils; import org.folio.search.domain.dto.Identifier; import org.folio.search.domain.dto.Instance; import org.folio.search.integration.folio.ReferenceDataService; diff --git a/src/test/java/org/folio/search/support/api/InventoryApi.java b/src/test/java/org/folio/search/support/api/InventoryApi.java index 41631e5b4..4e4243648 100644 --- a/src/test/java/org/folio/search/support/api/InventoryApi.java +++ b/src/test/java/org/folio/search/support/api/InventoryApi.java @@ -1,7 +1,7 @@ package org.folio.search.support.api; import static java.util.Collections.emptyMap; -import static org.apache.commons.collections.MapUtils.getString; +import static org.apache.commons.collections4.MapUtils.getString; import static org.assertj.core.api.Assertions.assertThat; import static org.folio.search.domain.dto.ResourceEventType.CREATE; import static org.folio.search.domain.dto.ResourceEventType.DELETE; @@ -24,8 +24,8 @@ import java.util.function.Consumer; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.folio.search.domain.dto.Instance; import org.folio.search.domain.dto.ResourceEvent; import org.folio.search.model.types.ResourceType; diff --git a/src/test/resources/sql/populate-classifications.sql b/src/test/resources/sql/populate-classifications.sql new file mode 100644 index 000000000..06da8f18e --- /dev/null +++ b/src/test/resources/sql/populate-classifications.sql @@ -0,0 +1,12 @@ +INSERT INTO classification (id, number, type_id) +VALUES + ('1', 'Genre', 'b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6'), + ('2', 'Sci-Fi', 'dfb20d52-7f1f-4b5b-a492-2e47d2c0ac59'); + + +INSERT INTO instance_classification (instance_id, classification_id, tenant_id, shared) +VALUES + ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid, '1', 'member_tenant', false), + ('b3bae8a9-cfb1-4afe-83d5-2cdae4580e07'::uuid, '2', 'consortium', true), + ('9ec55e4f-6a76-427c-b47b-197046f44a54'::uuid, '2', 'member_tenant', false), + ('aab8fff4-49c6-4578-979e-439b6ba3600a'::uuid, '2', 'consortium', true); \ No newline at end of file diff --git a/src/test/resources/sql/populate-contributors.sql b/src/test/resources/sql/populate-contributors.sql new file mode 100644 index 000000000..e200346ca --- /dev/null +++ b/src/test/resources/sql/populate-contributors.sql @@ -0,0 +1,12 @@ +INSERT INTO contributor (id, name, name_type_id, authority_id) +VALUES + ('1', 'Genre', 'b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d5', 'b7df83a1-8b15-46c1-9a4c-9d2dbb3cf4d6'), + ('2', 'Sci-Fi', 'dfb20d52-7f1f-4b5b-a492-2e47d2c0ac58', 'dfb20d52-7f1f-4b5b-a492-2e47d2c0ac59'); + + +INSERT INTO instance_contributor (instance_id, contributor_id, type_id, tenant_id, shared) +VALUES + ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid, '1', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', 'member_tenant', false), + ('b3bae8a9-cfb1-4afe-83d5-2cdae4580e07'::uuid, '2', 'b3bae8a9-cfb1-4afe-83d5-2cdae4580e06', 'consortium', true), + ('9ec55e4f-6a76-427c-b47b-197046f44a54'::uuid, '2', '9ec55e4f-6a76-427c-b47b-197046f44a53', 'member_tenant', false), + ('aab8fff4-49c6-4578-979e-439b6ba3600a'::uuid, '2', 'aab8fff4-49c6-4578-979e-439b6ba3600b', 'consortium', true); \ No newline at end of file