diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index fa7681ad9..a61a6fedb 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -13,6 +13,15 @@ "modulePermissions": [ ] }, + { + "methods": [ "POST" ], + "pathPattern": "/bulk-operations/query", + "permissionsRequired": [ "bulk-operations.item.query.post" ], + "modulePermissions": [ + "fqm.entityTypes.item.get", + "fqm.query.async.post" + ] + }, { "methods": [ "POST" ], "pathPattern": "/bulk-operations/{operationId}/content-update", @@ -164,6 +173,8 @@ "pathPattern": "/bulk-operations/{operationId}", "permissionsRequired": [ "bulk-operations.item.get" ], "modulePermissions": [ + "fqm.query.sync.get", + "fqm.query.async.results.get" ] }, { @@ -239,6 +250,11 @@ "displayName" : "upload identifiers list to initiate bulk-operation", "description" : "Upload identifiers list to initiate bulk-operation" }, + { + "permissionName" : "bulk-operations.item.query.post", + "displayName" : "trigger bulk edit by query", + "description" : "Trigger bulk edit by query" + }, { "permissionName" : "bulk-operations.item.content-update.post", "displayName" : "upload content updates for bulk operation", @@ -316,7 +332,8 @@ "bulk-operations.download.item.get", "bulk-operations.list-users.collection.get", "bulk-operations.files.item.delete", - "bulk-operations.item.cancel.post" + "bulk-operations.item.cancel.post", + "bulk-operations.item.query.post" ] } ], @@ -416,6 +433,14 @@ { "id": "instance-formats", "version": "2.0" + }, + { + "id": "fqm-query", + "version": "1.0" + }, + { + "id": "entity-types", + "version": "1.0" } ], "launchDescriptor": { diff --git a/pom.xml b/pom.xml index 928812ea6..2a004956c 100644 --- a/pom.xml +++ b/pom.xml @@ -45,6 +45,7 @@ 12.1 2.19.2 3.5.0 + 1.1.0-SNAPSHOT 1.17.6 2.35.0 @@ -79,6 +80,12 @@ ${folio-spring-cql.version} + + org.folio + lib-fqm-query-processor + ${lib-fqm-query-processor.version} + + org.springframework.boot spring-boot-starter-web @@ -332,6 +339,7 @@ true ApiUtil.java true + SubmitQuery=org.folio.querytool.domain.dto.SubmitQuery true java diff --git a/src/main/java/org/folio/bulkops/client/EntityTypeClient.java b/src/main/java/org/folio/bulkops/client/EntityTypeClient.java new file mode 100644 index 000000000..3fd6af880 --- /dev/null +++ b/src/main/java/org/folio/bulkops/client/EntityTypeClient.java @@ -0,0 +1,14 @@ +package org.folio.bulkops.client; + +import org.folio.querytool.domain.dto.EntityType; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; + +import java.util.UUID; + +@FeignClient(name = "entity-types") +public interface EntityTypeClient { + @GetMapping("/{entityTypeId}") + EntityType getEntityType(@RequestHeader UUID entityTypeId); +} diff --git a/src/main/java/org/folio/bulkops/client/QueryClient.java b/src/main/java/org/folio/bulkops/client/QueryClient.java new file mode 100644 index 000000000..a88cd2a13 --- /dev/null +++ b/src/main/java/org/folio/bulkops/client/QueryClient.java @@ -0,0 +1,27 @@ +package org.folio.bulkops.client; + +import org.folio.querytool.domain.dto.QueryDetails; +import org.folio.querytool.domain.dto.QueryIdentifier; +import org.folio.querytool.domain.dto.SubmitQuery; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestParam; + +import java.util.List; +import java.util.UUID; + +@FeignClient(name = "query") +public interface QueryClient { + + @PostMapping("") + QueryIdentifier executeQuery(@RequestBody SubmitQuery submitQuery); + + @GetMapping("/{queryId}") + QueryDetails getQuery(@RequestHeader UUID queryId); + + @GetMapping("/{queryId}/sortedIds") + List getSortedIds(@RequestHeader UUID queryId, @RequestParam Integer offset, @RequestParam Integer limit); +} diff --git a/src/main/java/org/folio/bulkops/controller/BulkOperationController.java b/src/main/java/org/folio/bulkops/controller/BulkOperationController.java index 9ae5607ff..1d2d18bc5 100644 --- a/src/main/java/org/folio/bulkops/controller/BulkOperationController.java +++ b/src/main/java/org/folio/bulkops/controller/BulkOperationController.java @@ -24,6 +24,7 @@ import org.folio.bulkops.service.LogFilesService; import org.folio.bulkops.service.PreviewService; import org.folio.bulkops.service.RuleService; +import org.folio.querytool.domain.dto.SubmitQuery; import org.folio.spring.cql.JpaCqlRepository; import org.folio.spring.data.OffsetRequest; import org.springframework.core.io.ByteArrayResource; @@ -170,4 +171,9 @@ public ResponseEntity cancelOperationById(UUID operationId) { bulkOperationService.cancelOperationById(operationId); return new ResponseEntity<>(HttpStatus.OK); } + + @Override + public ResponseEntity triggerBulkEditByQuery(UUID xOkapiUserId, SubmitQuery submitQuery) { + return new ResponseEntity<>(bulkOperationMapper.mapToDto(bulkOperationService.triggerByQuery(xOkapiUserId, submitQuery)), HttpStatus.OK); + } } diff --git a/src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java b/src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java index fd6b7654c..5acab4790 100644 --- a/src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java +++ b/src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java @@ -77,4 +77,6 @@ public class BulkOperation { private LocalDateTime endTime; private String errorMessage; private boolean expired; + private UUID fqlQueryId; + private String fqlQuery; } diff --git a/src/main/java/org/folio/bulkops/service/BulkOperationService.java b/src/main/java/org/folio/bulkops/service/BulkOperationService.java index e87c55ba1..fe72377c8 100644 --- a/src/main/java/org/folio/bulkops/service/BulkOperationService.java +++ b/src/main/java/org/folio/bulkops/service/BulkOperationService.java @@ -7,15 +7,17 @@ import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.folio.bulkops.domain.dto.ApproachType.IN_APP; import static org.folio.bulkops.domain.dto.ApproachType.MANUAL; +import static org.folio.bulkops.domain.dto.ApproachType.QUERY; import static org.folio.bulkops.domain.dto.BulkOperationStep.UPLOAD; -import static org.folio.bulkops.domain.dto.OperationStatusType.APPLY_CHANGES; import static org.folio.bulkops.domain.dto.OperationStatusType.COMPLETED; import static org.folio.bulkops.domain.dto.OperationStatusType.COMPLETED_WITH_ERRORS; import static org.folio.bulkops.domain.dto.OperationStatusType.DATA_MODIFICATION; +import static org.folio.bulkops.domain.dto.OperationStatusType.EXECUTING_QUERY; import static org.folio.bulkops.domain.dto.OperationStatusType.FAILED; import static org.folio.bulkops.domain.dto.OperationStatusType.NEW; import static org.folio.bulkops.domain.dto.OperationStatusType.RETRIEVING_RECORDS; import static org.folio.bulkops.domain.dto.OperationStatusType.REVIEW_CHANGES; +import static org.folio.bulkops.domain.dto.OperationStatusType.SAVED_IDENTIFIERS; import static org.folio.bulkops.domain.dto.OperationStatusType.SAVING_RECORDS_LOCALLY; import static org.folio.bulkops.util.Constants.FIELD_ERROR_MESSAGE_PATTERN; import static org.folio.bulkops.util.Utils.resolveEntityClass; @@ -66,6 +68,7 @@ import org.folio.bulkops.repository.BulkOperationExecutionRepository; import org.folio.bulkops.repository.BulkOperationRepository; import org.folio.bulkops.util.Utils; +import org.folio.querytool.domain.dto.SubmitQuery; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; @@ -102,8 +105,9 @@ public class BulkOperationService { private final DataProcessorFactory dataProcessorFactory; private final ErrorService errorService; private final LogFilesService logFilesService; - private final NoteTableUpdater noteTableUpdater; private final RecordUpdateService recordUpdateService; + private final EntityTypeService entityTypeService; + private final QueryService queryService; private static final int OPERATION_UPDATING_STEP = 100; private static final String PREVIEW_JSON_PATH_TEMPLATE = "%s/json/%s-Updates-Preview-%s.json"; @@ -172,6 +176,22 @@ public BulkOperation uploadCsvFile(EntityType entityType, IdentifierType identif return bulkOperationRepository.save(operation); } + public BulkOperation triggerByQuery(UUID userId, SubmitQuery submitQuery) { + var queryId = queryService.executeQuery(submitQuery); + var entityType = entityTypeService.getEntityTypeById(submitQuery.getEntityTypeId()); + return bulkOperationRepository.save(BulkOperation.builder() + .id(UUID.randomUUID()) + .entityType(entityType) + .approach(QUERY) + .identifierType(IdentifierType.ID) + .status(EXECUTING_QUERY) + .startTime(LocalDateTime.now()) + .userId(userId) + .fqlQuery(submitQuery.getFqlQuery()) + .fqlQueryId(queryId) + .build()); + } + public void confirm(BulkOperation operation) { operation.setProcessedNumOfRecords(0); @@ -415,7 +435,7 @@ public BulkOperation startBulkOperation(UUID bulkOperationId, UUID xOkapiUserId, private String executeDataExportJob(BulkOperationStep step, ApproachType approach, BulkOperation operation, String errorMessage) { try { - if (NEW.equals(operation.getStatus())) { + if (Set.of(NEW, SAVED_IDENTIFIERS).contains(operation.getStatus())) { if (MANUAL != approach) { var job = dataExportSpringClient.upsertJob(Job.builder() .type(ExportType.BULK_EDIT_IDENTIFIERS) @@ -520,20 +540,29 @@ public void clearOperationProcessing(BulkOperation operation) { public BulkOperation getOperationById(UUID bulkOperationId) { var operation = getBulkOperationOrThrow(bulkOperationId); - if (DATA_MODIFICATION.equals(operation.getStatus())) { - var processing = dataProcessingRepository.findByBulkOperationId(bulkOperationId); - if (processing.isPresent() && StatusType.ACTIVE.equals(processing.get().getStatus())) { - operation.setProcessedNumOfRecords(processing.get().getProcessedNumOfRecords()); - return operation; + return switch (operation.getStatus()) { + case EXECUTING_QUERY -> queryService.checkQueryExecutionStatus(operation); + case SAVED_IDENTIFIERS -> startBulkOperation(operation.getId(), operation.getUserId(), new BulkOperationStart() + .step(UPLOAD) + .approach(IN_APP) + .entityType(operation.getEntityType()) + .entityCustomIdentifierType(IdentifierType.ID)); + case DATA_MODIFICATION -> { + var processing = dataProcessingRepository.findByBulkOperationId(bulkOperationId); + if (processing.isPresent() && StatusType.ACTIVE.equals(processing.get().getStatus())) { + operation.setProcessedNumOfRecords(processing.get().getProcessedNumOfRecords()); + } + yield operation; } - } else if (APPLY_CHANGES.equals(operation.getStatus())) { - var execution = executionRepository.findByBulkOperationId(bulkOperationId); - if (execution.isPresent() && StatusType.ACTIVE.equals(execution.get().getStatus())) { - operation.setProcessedNumOfRecords(execution.get().getProcessedRecords()); - return operation; + case APPLY_CHANGES -> { + var execution = executionRepository.findByBulkOperationId(bulkOperationId); + if (execution.isPresent() && StatusType.ACTIVE.equals(execution.get().getStatus())) { + operation.setProcessedNumOfRecords(execution.get().getProcessedRecords()); + } + yield operation; } - } - return operation; + default -> operation; + }; } public BulkOperation getBulkOperationOrThrow(UUID operationId) { diff --git a/src/main/java/org/folio/bulkops/service/EntityTypeService.java b/src/main/java/org/folio/bulkops/service/EntityTypeService.java new file mode 100644 index 000000000..47ef42c20 --- /dev/null +++ b/src/main/java/org/folio/bulkops/service/EntityTypeService.java @@ -0,0 +1,23 @@ +package org.folio.bulkops.service; + +import lombok.RequiredArgsConstructor; +import org.folio.bulkops.client.EntityTypeClient; +import org.folio.bulkops.domain.dto.EntityType; +import org.springframework.stereotype.Service; + +import java.util.UUID; + +@Service +@RequiredArgsConstructor +public class EntityTypeService { + private final EntityTypeClient entityTypeClient; + + public EntityType getEntityTypeById(UUID entityTypeId) { + var alias = entityTypeClient.getEntityType(entityTypeId).getLabelAlias(); + return switch (alias) { + case "Items" -> EntityType.ITEM; + case "Users" -> EntityType.USER; + default -> throw new IllegalArgumentException(String.format("Entity type %s is not supported", alias)); + }; + } +} diff --git a/src/main/java/org/folio/bulkops/service/QueryService.java b/src/main/java/org/folio/bulkops/service/QueryService.java new file mode 100644 index 000000000..b1317816e --- /dev/null +++ b/src/main/java/org/folio/bulkops/service/QueryService.java @@ -0,0 +1,89 @@ +package org.folio.bulkops.service; + +import static org.folio.bulkops.domain.dto.OperationStatusType.CANCELLED; +import static org.folio.bulkops.domain.dto.OperationStatusType.FAILED; +import static org.folio.bulkops.domain.dto.OperationStatusType.RETRIEVING_IDENTIFIERS; +import static org.folio.bulkops.domain.dto.OperationStatusType.SAVED_IDENTIFIERS; +import static org.folio.bulkops.util.Constants.NEW_LINE_SEPARATOR; +import static org.folio.spring.scope.FolioExecutionScopeExecutionContextManager.getRunnableWithCurrentFolioContext; + +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.folio.bulkops.client.QueryClient; +import org.folio.bulkops.client.RemoteFileSystemClient; +import org.folio.bulkops.domain.entity.BulkOperation; +import org.folio.bulkops.repository.BulkOperationRepository; +import org.folio.querytool.domain.dto.SubmitQuery; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.time.LocalDateTime; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +@Service +@Log4j2 +@RequiredArgsConstructor +public class QueryService { + public static final String QUERY_FILENAME_TEMPLATE = "%1$s/Query-%1$s.csv"; + + private final QueryClient queryClient; + private final BulkOperationRepository bulkOperationRepository; + private final RemoteFileSystemClient remoteFileSystemClient; + + private final ExecutorService executor = Executors.newCachedThreadPool(); + + public UUID executeQuery(SubmitQuery submitQuery) { + return queryClient.executeQuery(submitQuery).getQueryId(); + } + + public BulkOperation checkQueryExecutionStatus(BulkOperation bulkOperation) { + var queryResult = queryClient.getQuery(bulkOperation.getFqlQueryId()); + return switch (queryResult.getStatus()) { + case SUCCESS -> { + if (queryResult.getTotalRecords() == 0) { + yield failBulkOperation(bulkOperation, "No records found for the query"); + } + executor.execute(getRunnableWithCurrentFolioContext(() -> saveIdentifiers(bulkOperation))); + bulkOperation.setStatus(RETRIEVING_IDENTIFIERS); + yield bulkOperationRepository.save(bulkOperation); + } + case FAILED -> failBulkOperation(bulkOperation, queryResult.getFailureReason()); + case CANCELLED -> cancelBulkOperation((bulkOperation)); + case IN_PROGRESS -> bulkOperation; + }; + } + + private void saveIdentifiers(BulkOperation bulkOperation) { + try { + var identifiersString = queryClient.getSortedIds(bulkOperation.getFqlQueryId(), 0, Integer.MAX_VALUE).stream() + .map(UUID::toString) + .collect(Collectors.joining(NEW_LINE_SEPARATOR)); + var path = String.format(QUERY_FILENAME_TEMPLATE, bulkOperation.getId()); + remoteFileSystemClient.put(new ByteArrayInputStream(identifiersString.getBytes()), path); + bulkOperation.setLinkToTriggeringCsvFile(path); + bulkOperation.setStatus(SAVED_IDENTIFIERS); + bulkOperationRepository.save(bulkOperation); + } catch (Exception e) { + var errorMessage = "Failed to save identifiers, reason: " + e.getMessage(); + log.error(errorMessage); + failBulkOperation(bulkOperation, errorMessage); + } + } + + private BulkOperation failBulkOperation(BulkOperation bulkOperation, String errorMessage) { + bulkOperation.setStatus(FAILED); + bulkOperation.setErrorMessage(errorMessage); + bulkOperation.setEndTime(LocalDateTime.now()); + return bulkOperationRepository.save(bulkOperation); + } + + private BulkOperation cancelBulkOperation(BulkOperation bulkOperation) { + bulkOperation.setStatus(CANCELLED); + bulkOperation.setErrorMessage("Query execution was cancelled"); + bulkOperation.setEndTime(LocalDateTime.now()); + return bulkOperationRepository.save(bulkOperation); + } +} diff --git a/src/main/resources/db/changelog/changelog-master.xml b/src/main/resources/db/changelog/changelog-master.xml index 6cb284384..a8b2f9bdf 100644 --- a/src/main/resources/db/changelog/changelog-master.xml +++ b/src/main/resources/db/changelog/changelog-master.xml @@ -22,4 +22,5 @@ + diff --git a/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.sql b/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.sql new file mode 100644 index 000000000..e0d393bf1 --- /dev/null +++ b/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.sql @@ -0,0 +1,6 @@ +ALTER TABLE bulk_operation ADD COLUMN IF NOT EXISTS fql_query_id UUID; +ALTER TABLE bulk_operation ADD COLUMN IF NOT EXISTS fql_query TEXT; + +ALTER TYPE OperationStatusType ADD VALUE IF NOT EXISTS 'EXECUTING_QUERY'; +ALTER TYPE OperationStatusType ADD VALUE IF NOT EXISTS 'RETRIEVING_IDENTIFIERS'; +ALTER TYPE OperationStatusType ADD VALUE IF NOT EXISTS 'SAVED_IDENTIFIERS'; diff --git a/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.xml b/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.xml new file mode 100644 index 000000000..338c99a79 --- /dev/null +++ b/src/main/resources/db/changelog/changes/26-01-2024_add_query_fields_and_statuses.xml @@ -0,0 +1,13 @@ + + + + + + + + + diff --git a/src/main/resources/swagger.api/bulk-operations.yaml b/src/main/resources/swagger.api/bulk-operations.yaml index 15a3f9920..b3a3f0f88 100644 --- a/src/main/resources/swagger.api/bulk-operations.yaml +++ b/src/main/resources/swagger.api/bulk-operations.yaml @@ -68,6 +68,46 @@ paths: application/json: schema: $ref: "#/components/schemas/errors" + /bulk-operations/query: + post: + description: Trigger bulk edit by query + operationId: triggerBulkEditByQuery + parameters: + - name: X-Okapi-User-Id + in: header + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: "SubmitQuery" + responses: + "201": + description: Accepted + content: + application/json: + schema: + $ref: "#/components/schemas/bulkOperationDto" + "400": + description: Bad Request + content: + application/json: + schema: + $ref: "#/components/schemas/errors" + "422": + description: Unprocessable entity + content: + application/json: + schema: + $ref: "#/components/schemas/errors" + "500": + description: Internal server errors, e.g. due to misconfiguration + content: + application/json: + schema: + $ref: "#/components/schemas/errors" /bulk-operations/{operationId}/content-update: post: description: Upload content updates diff --git a/src/main/resources/swagger.api/schemas/approach_type.json b/src/main/resources/swagger.api/schemas/approach_type.json index 2d17178cd..0c0471baf 100644 --- a/src/main/resources/swagger.api/schemas/approach_type.json +++ b/src/main/resources/swagger.api/schemas/approach_type.json @@ -5,7 +5,8 @@ "type": "string", "enum": [ "MANUAL", - "IN_APP" + "IN_APP", + "QUERY" ] } } diff --git a/src/main/resources/swagger.api/schemas/bulk_operation_dto.json b/src/main/resources/swagger.api/schemas/bulk_operation_dto.json index 7067fb230..39ab10580 100644 --- a/src/main/resources/swagger.api/schemas/bulk_operation_dto.json +++ b/src/main/resources/swagger.api/schemas/bulk_operation_dto.json @@ -140,6 +140,15 @@ "description": "Were logs expired", "type": "boolean", "default": false + }, + "fqlQueryId": { + "description": "FQL query identifier", + "type": "string", + "format": "uuid" + }, + "fqlQuery": { + "description": "FQL query string", + "type": "string" } }, "additionalProperties": false, diff --git a/src/main/resources/swagger.api/schemas/operation_status_type.json b/src/main/resources/swagger.api/schemas/operation_status_type.json index aaffcd464..f69c9acd1 100644 --- a/src/main/resources/swagger.api/schemas/operation_status_type.json +++ b/src/main/resources/swagger.api/schemas/operation_status_type.json @@ -15,7 +15,10 @@ "COMPLETED_WITH_ERRORS", "CANCELLED", "SCHEDULED", - "FAILED" + "FAILED", + "EXECUTING_QUERY", + "RETRIEVING_IDENTIFIERS", + "SAVED_IDENTIFIERS" ] } } diff --git a/src/main/resources/swagger.api/schemas/query_request.json b/src/main/resources/swagger.api/schemas/query_request.json new file mode 100644 index 000000000..32b2867f7 --- /dev/null +++ b/src/main/resources/swagger.api/schemas/query_request.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Query request", + "QueryRequest": { + "description": "Query request", + "type": "object", + "properties": { + "queryString" : { + "description": "Query string value", + "type": "string" + }, + "entityTypeId": { + "description": "Entity type identifier", + "type": "string", + "format": "uuid" + } + }, + "required": [ + "queryString", + "entityTypeId" + ], + "additionalProperties": false + } +} diff --git a/src/test/java/org/folio/bulkops/service/BulkOperationServiceTest.java b/src/test/java/org/folio/bulkops/service/BulkOperationServiceTest.java index 2f5cc2538..f4145fdc8 100644 --- a/src/test/java/org/folio/bulkops/service/BulkOperationServiceTest.java +++ b/src/test/java/org/folio/bulkops/service/BulkOperationServiceTest.java @@ -3,11 +3,12 @@ import static java.util.Objects.nonNull; import static org.folio.bulkops.domain.dto.EntityType.HOLDINGS_RECORD; import static org.folio.bulkops.domain.dto.EntityType.ITEM; +import static org.folio.bulkops.domain.dto.OperationStatusType.EXECUTING_QUERY; +import static org.folio.bulkops.domain.dto.OperationStatusType.SAVED_IDENTIFIERS; import static org.folio.bulkops.util.Constants.ADMINISTRATIVE_NOTE; import static org.folio.bulkops.util.Constants.ADMINISTRATIVE_NOTES; import static org.folio.bulkops.util.Constants.APPLY_TO_ITEMS; import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED; -import static org.folio.bulkops.util.UnifiedTableHeaderBuilder.getHeaders; import static org.folio.bulkops.domain.dto.BulkOperationStep.COMMIT; import static org.folio.bulkops.domain.dto.BulkOperationStep.EDIT; import static org.folio.bulkops.domain.dto.EntityType.USER; @@ -19,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; @@ -32,7 +32,6 @@ import static org.testcontainers.shaded.org.hamcrest.MatcherAssert.assertThat; import static org.testcontainers.shaded.org.hamcrest.Matchers.containsString; import static org.testcontainers.shaded.org.hamcrest.Matchers.equalTo; -import static org.testcontainers.shaded.org.hamcrest.Matchers.hasSize; import static org.testcontainers.shaded.org.hamcrest.Matchers.is; import static org.testcontainers.shaded.org.hamcrest.Matchers.notNullValue; @@ -46,18 +45,17 @@ import java.time.LocalDate; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.UUID; import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.folio.bulkops.BaseTest; import org.folio.bulkops.client.BulkEditClient; import org.folio.bulkops.client.DataExportSpringClient; import org.folio.bulkops.client.RemoteFileSystemClient; import org.folio.bulkops.domain.bean.HoldingsRecord; -import org.folio.bulkops.domain.bean.HoldingsRecordsSource; import org.folio.bulkops.domain.bean.Item; import org.folio.bulkops.domain.bean.ItemCollection; import org.folio.bulkops.domain.bean.ItemLocation; @@ -80,7 +78,6 @@ import org.folio.bulkops.domain.dto.IdentifierType; import org.folio.bulkops.domain.dto.OperationStatusType; import org.folio.bulkops.domain.dto.Parameter; -import org.folio.bulkops.domain.dto.Row; import org.folio.bulkops.domain.dto.UpdateActionType; import org.folio.bulkops.domain.dto.UpdateOptionType; import org.folio.bulkops.domain.entity.BulkOperation; @@ -94,10 +91,10 @@ import org.folio.bulkops.repository.BulkOperationExecutionContentRepository; import org.folio.bulkops.repository.BulkOperationExecutionRepository; import org.folio.bulkops.repository.BulkOperationRepository; +import org.folio.querytool.domain.dto.SubmitQuery; import org.folio.s3.client.FolioS3Client; import org.folio.s3.client.RemoteStorageWriter; import org.folio.spring.scope.FolioExecutionContextSetter; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; @@ -149,6 +146,12 @@ class BulkOperationServiceTest extends BaseTest { @MockBean private NoteTableUpdater noteTableUpdater; + @MockBean + private QueryService queryService; + + @MockBean + private EntityTypeService entityTypeService; + @Test @SneakyThrows void shouldUploadIdentifiers() { @@ -873,6 +876,9 @@ void shouldReturnBulkOperationById(OperationStatusType statusType) { .processedRecords(5) .build())); + when(queryService.checkQueryExecutionStatus(any(BulkOperation.class))) + .thenReturn(new BulkOperation()); + var operation = bulkOperationService.getOperationById(operationId); if (DATA_MODIFICATION.equals(operation.getStatus()) || APPLY_CHANGES.equals(operation.getStatus())) { @@ -1033,6 +1039,56 @@ void shouldUpdateItemsWhenCommittingHoldingsRecordDiscoverySuppressed(DiscoveryS } } + @Test + void testQueryExecution() { + var fqlQueryId = UUID.randomUUID(); + var entityTypeId = UUID.randomUUID(); + var query = "query"; + var submitQuery = new SubmitQuery() + .fqlQuery(query) + .entityTypeId(entityTypeId); + when(queryService.executeQuery(submitQuery)).thenReturn(fqlQueryId); + when(entityTypeService.getEntityTypeById(entityTypeId)).thenReturn(ITEM); + + bulkOperationService.triggerByQuery(UUID.randomUUID(), submitQuery); + + verify(queryService).executeQuery(submitQuery); + var operationCaptor = ArgumentCaptor.forClass(BulkOperation.class); + verify(bulkOperationRepository).save(operationCaptor.capture()); + var operation = operationCaptor.getValue(); + Assertions.assertThat(operation.getEntityType()).isEqualTo(ITEM); + Assertions.assertThat(operation.getIdentifierType()).isEqualTo(IdentifierType.ID); + Assertions.assertThat(operation.getStatus()).isEqualTo(EXECUTING_QUERY); + Assertions.assertThat(operation.getFqlQueryId()).isEqualTo(fqlQueryId); + Assertions.assertThat(operation.getFqlQuery()).isEqualTo(query); + } + + @Test + void shouldCheckQueryExecution() { + var operationId = UUID.randomUUID(); + var operation = new BulkOperation(); + operation.setStatus(EXECUTING_QUERY); + when(bulkOperationRepository.findById(operationId)).thenReturn(Optional.of(operation)); + + bulkOperationService.getOperationById(operationId); + + verify(queryService).checkQueryExecutionStatus(operation); + } + + @Test + void shouldStartDataExportJobWhenIdentifiersWereSaved() { + var operationId = UUID.randomUUID(); + var operation = new BulkOperation(); + operation.setId(operationId); + operation.setStatus(SAVED_IDENTIFIERS); + operation.setEntityType(ITEM); + when(bulkOperationRepository.findById(operationId)).thenReturn(Optional.of(operation)); + + bulkOperationService.getOperationById(operationId); + + verify(dataExportSpringClient).upsertJob(any(Job.class)); + } + private List renameAdministrativeNotesHeader(List headers) { headers.forEach(cell -> { if (ADMINISTRATIVE_NOTES.equalsIgnoreCase(cell.getValue())) { diff --git a/src/test/java/org/folio/bulkops/service/EntityTypeServiceTest.java b/src/test/java/org/folio/bulkops/service/EntityTypeServiceTest.java new file mode 100644 index 000000000..be0cc8f79 --- /dev/null +++ b/src/test/java/org/folio/bulkops/service/EntityTypeServiceTest.java @@ -0,0 +1,36 @@ +package org.folio.bulkops.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +import org.folio.bulkops.client.EntityTypeClient; +import org.folio.querytool.domain.dto.EntityType; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.UUID; + +@ExtendWith(MockitoExtension.class) +class EntityTypeServiceTest { + @Mock + private EntityTypeClient entityTypeClient; + @InjectMocks + private EntityTypeService entityTypeService; + + @ParameterizedTest + @CsvSource(textBlock = """ + ae6a4972-0e0d-4069-9936-99ba6e91658d | Users | USER + a5f253dc-3215-46bb-9714-2b26d8b3b613 | Items | ITEM + """, delimiter = '|') + void testGetEntityType(UUID entityTypeId, String alias, org.folio.bulkops.domain.dto.EntityType expectedType) { + when(entityTypeClient.getEntityType(entityTypeId)).thenReturn(new EntityType().labelAlias(alias)); + + var actualType = entityTypeService.getEntityTypeById(entityTypeId); + + assertEquals(expectedType, actualType); + } +} diff --git a/src/test/java/org/folio/bulkops/service/QueryServiceTest.java b/src/test/java/org/folio/bulkops/service/QueryServiceTest.java new file mode 100644 index 000000000..be5ac924d --- /dev/null +++ b/src/test/java/org/folio/bulkops/service/QueryServiceTest.java @@ -0,0 +1,127 @@ +package org.folio.bulkops.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.folio.bulkops.domain.dto.OperationStatusType.CANCELLED; +import static org.folio.bulkops.domain.dto.OperationStatusType.EXECUTING_QUERY; +import static org.folio.bulkops.service.QueryService.QUERY_FILENAME_TEMPLATE; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.folio.bulkops.BaseTest; +import org.folio.bulkops.client.QueryClient; +import org.folio.bulkops.client.RemoteFileSystemClient; +import org.folio.bulkops.domain.dto.OperationStatusType; +import org.folio.bulkops.domain.entity.BulkOperation; +import org.folio.bulkops.repository.BulkOperationRepository; +import org.folio.querytool.domain.dto.QueryDetails; +import org.folio.spring.scope.FolioExecutionContextSetter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; + +import java.io.ByteArrayInputStream; +import java.util.List; +import java.util.UUID; + +class QueryServiceTest extends BaseTest { + @MockBean + private QueryClient queryClient; + @MockBean + private BulkOperationRepository bulkOperationRepository; + @MockBean + private RemoteFileSystemClient remoteFileSystemClient; + @Autowired + private QueryService queryService; + + @Test + void shouldSaveIdentifiersOnSuccessfulQueryExecution() { + try (var context = new FolioExecutionContextSetter(folioExecutionContext)) { + var operationId = UUID.randomUUID(); + var fqlQueryId = UUID.randomUUID(); + var expectedPath = String.format(QUERY_FILENAME_TEMPLATE, operationId); + var operation = BulkOperation.builder() + .id(operationId) + .fqlQueryId(fqlQueryId) + .build(); + + when(queryClient.getQuery(fqlQueryId)).thenReturn(new QueryDetails() + .status(QueryDetails.StatusEnum.SUCCESS) + .totalRecords(2)); + when(queryClient.getSortedIds(fqlQueryId, 0, Integer.MAX_VALUE)) + .thenReturn(List.of(UUID.randomUUID(), UUID.randomUUID())); + + queryService.checkQueryExecutionStatus(operation); + + await().untilAsserted(() -> + verify(remoteFileSystemClient).put(any(ByteArrayInputStream.class), eq(expectedPath))); + } + } + + @ParameterizedTest + @EnumSource(value = QueryDetails.StatusEnum.class, names = {"SUCCESS", "FAILED"}, mode = EnumSource.Mode.INCLUDE) + void shouldFailOperationIfNoMatchFoundOrQueryFails(QueryDetails.StatusEnum status) { + var operationId = UUID.randomUUID(); + var fqlQueryId = UUID.randomUUID(); + var expectedPath = String.format(QUERY_FILENAME_TEMPLATE, operationId); + var operation = BulkOperation.builder() + .id(operationId) + .fqlQueryId(fqlQueryId) + .build(); + + when(queryClient.getQuery(fqlQueryId)).thenReturn(new QueryDetails() + .status(status) + .totalRecords(0)); + + queryService.checkQueryExecutionStatus(operation); + + verify(remoteFileSystemClient, times(0)).put(any(ByteArrayInputStream.class), eq(expectedPath)); + var operationCaptor = ArgumentCaptor.forClass(BulkOperation.class); + verify(bulkOperationRepository).save(operationCaptor.capture()); + assertThat(operationCaptor.getValue().getStatus()).isEqualTo(OperationStatusType.FAILED); + assertThat(operationCaptor.getValue().getEndTime()).isNotNull(); + } + + @Test + void shouldCancelOperationIfQueryWasCancelled() { + var operationId = UUID.randomUUID(); + var fqlQueryId = UUID.randomUUID(); + var operation = BulkOperation.builder() + .id(operationId) + .fqlQueryId(fqlQueryId) + .build(); + + when(queryClient.getQuery(fqlQueryId)).thenReturn(new QueryDetails() + .status(QueryDetails.StatusEnum.CANCELLED)); + + queryService.checkQueryExecutionStatus(operation); + + var operationCaptor = ArgumentCaptor.forClass(BulkOperation.class); + verify(bulkOperationRepository).save(operationCaptor.capture()); + assertThat(operationCaptor.getValue().getStatus()).isEqualTo(CANCELLED); + } + + @Test + void shouldReturnOperationIfQueryInProgress() { + var operationId = UUID.randomUUID(); + var fqlQueryId = UUID.randomUUID(); + var operation = BulkOperation.builder() + .id(operationId) + .fqlQueryId(fqlQueryId) + .status(EXECUTING_QUERY) + .build(); + + when(queryClient.getQuery(fqlQueryId)).thenReturn(new QueryDetails() + .status(QueryDetails.StatusEnum.IN_PROGRESS)); + + var result = queryService.checkQueryExecutionStatus(operation); + + assertThat(result.getStatus()).isEqualTo(EXECUTING_QUERY); + } +}