Merge branch 'master' into MODBULKOPS-451

# Conflicts:
#	src/main/java/org/folio/bulkops/controller/BulkOperationController.java
#	src/main/java/org/folio/bulkops/service/ErrorService.java
#	src/main/resources/swagger.api/bulk-operations.yaml
#	src/test/java/org/folio/bulkops/service/ErrorServiceTest.java

src/main/java/org/folio/bulkops/controller/BulkOperationController.java

@@ -26,6 +26,7 @@
 import org.folio.bulkops.domain.dto.BulkOperationStart;
 import org.folio.bulkops.domain.dto.BulkOperationStep;
 import org.folio.bulkops.domain.dto.EntityType;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.dto.Errors;
 import org.folio.bulkops.domain.dto.FileContentType;
 import org.folio.bulkops.domain.dto.IdentifierType;
@@ -92,8 +93,8 @@ public ResponseEntity<BulkOperationCollection> getBulkOperationCollection(String
   }
 
   @Override
-  public ResponseEntity<Errors> getErrorsPreviewByOperationId(UUID operationId, Integer limit, Integer offset) {
-    return new ResponseEntity<>(errorService.getErrorsPreviewByBulkOperationId(operationId, limit, offset), HttpStatus.OK);
+  public ResponseEntity<Errors> getErrorsPreviewByOperationId(UUID operationId, Integer limit, Integer offset, ErrorType errorType) {
+    return new ResponseEntity<>(errorService.getErrorsPreviewByBulkOperationId(operationId, limit, offset, errorType), HttpStatus.OK);
   }
 
   @Override

src/main/java/org/folio/bulkops/domain/converter/BaseConverter.java

@@ -14,10 +14,12 @@
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.folio.bulkops.domain.bean.Tags;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.exception.ConverterException;
 
 import com.opencsv.bean.AbstractBeanField;
 import com.opencsv.exceptions.CsvConstraintViolationException;
+import org.folio.bulkops.exception.ReferenceDataNotFoundException;
 
 public abstract class BaseConverter<T> extends AbstractBeanField<String, T> {
 
@@ -58,9 +60,12 @@ protected synchronized String convertToWrite(Object object) {
     }
     try {
       return convertToString((T) object);
+    } catch (ReferenceDataNotFoundException e) {
+      failed = true;
+      throw new ConverterException(this.getField(), object, e.getMessage(), ErrorType.WARNING);
     } catch (Exception e) {
       failed = true;
-      throw new ConverterException(this.getField(), object, e.getMessage());
+      throw new ConverterException(this.getField(), object, e.getMessage(), ErrorType.ERROR);
     }
   }
 

src/main/java/org/folio/bulkops/domain/converter/DepartmentsConverter.java

@@ -12,8 +12,10 @@
 
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.format.SpecialCharacterEscaper;
 import org.folio.bulkops.exception.ConverterException;
+import org.folio.bulkops.exception.ReferenceDataNotFoundException;
 import org.folio.bulkops.service.UserReferenceHelper;
 
 public class DepartmentsConverter extends BaseConverter<Set<UUID>> {
@@ -43,8 +45,10 @@ public String convertToString(Set<UUID> value) {
           .map(v -> SpecialCharacterEscaper.escape(v.getName()))
           .collect(Collectors.joining(ARRAY_DELIMITER));
       }
+    } catch (ReferenceDataNotFoundException e) {
+      throw new ConverterException(this.getField(), value, e.getMessage(), ErrorType.WARNING);
     } catch (Exception e) {
-      throw new ConverterException(this.getField(), value, e.getMessage());
+      throw new ConverterException(this.getField(), value, e.getMessage(), ErrorType.ERROR);
     }
     return EMPTY;
   }

src/main/java/org/folio/bulkops/domain/entity/BulkOperation.java

@@ -81,6 +81,7 @@ public class BulkOperation {
   private int committedNumOfRecords;
   private int matchedNumOfErrors;
   private int committedNumOfErrors;
+  private int committedNumOfWarnings;
   private LocalDateTime startTime;
   private LocalDateTime endTime;
   private String errorMessage;

src/main/java/org/folio/bulkops/domain/entity/BulkOperationExecutionContent.java

@@ -16,6 +16,9 @@
 import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.With;
+import org.folio.bulkops.domain.dto.ErrorType;
+import org.hibernate.annotations.JdbcType;
+import org.hibernate.dialect.PostgreSQLEnumJdbcType;
 
 @Data
 @Builder
@@ -41,4 +44,8 @@ public class BulkOperationExecutionContent {
   private String uiErrorMessage;
 
   private String linkToFailedEntity;
+
+  @Enumerated
+  @JdbcType(PostgreSQLEnumJdbcType.class)
+  private ErrorType errorType;
 }

src/main/java/org/folio/bulkops/exception/ConverterException.java

@@ -4,6 +4,7 @@
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import org.folio.bulkops.domain.dto.ErrorType;
 
 import java.lang.reflect.Field;
 
@@ -14,4 +15,5 @@ public class ConverterException extends CsvRuntimeException {
   private final transient Field field;
   private final transient Object value;
   private final String message;
+  private final ErrorType errorType;
 }

src/main/java/org/folio/bulkops/exception/ReferenceDataNotFoundException.java

@@ -0,0 +1,8 @@
+package org.folio.bulkops.exception;
+
+public class ReferenceDataNotFoundException extends RuntimeException {
+
+  public ReferenceDataNotFoundException(String message) {
+    super(message);
+  }
+}

src/main/java/org/folio/bulkops/processor/FolioAbstractDataProcessor.java

@@ -4,10 +4,12 @@
 import java.util.function.Consumer;
 
 import org.folio.bulkops.domain.bean.BulkOperationsEntity;
+import org.folio.bulkops.domain.bean.Error;
 import org.folio.bulkops.domain.bean.User;
 import org.folio.bulkops.domain.dto.Action;
 import org.folio.bulkops.domain.dto.BulkOperationRule;
 import org.folio.bulkops.domain.dto.BulkOperationRuleCollection;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.dto.UpdateOptionType;
 import org.folio.bulkops.exception.RuleValidationException;
 import org.folio.bulkops.exception.RuleValidationTenantsException;
@@ -60,7 +62,7 @@ public UpdatedEntityHolder process(String identifier, T entity, BulkOperationRul
       validator().validate(rules);
     } catch (RuleValidationException e) {
       log.warn(String.format("Rule validation exception: %s", e.getMessage()));
-      errorService.saveError(rules.getBulkOperationRules().get(0).getBulkOperationId(), identifier, e.getMessage());
+      errorService.saveError(rules.getBulkOperationRules().get(0).getBulkOperationId(), identifier, e.getMessage(), ErrorType.ERROR);
     } catch (Exception e) {
       log.error(e.getMessage());
     }
@@ -79,14 +81,14 @@ public UpdatedEntityHolder process(String identifier, T entity, BulkOperationRul
             }
         } catch (RuleValidationException e) {
           log.warn(String.format("Rule validation exception: %s", e.getMessage()));
-          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage());
+          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage(), ErrorType.ERROR);
         } catch (RuleValidationTenantsException e) {
           log.info("current tenant: {}", folioExecutionContext.getTenantId());
-          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage());
+          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage(), ErrorType.ERROR);
           log.error(e.getMessage());
         } catch (Exception e) {
           log.error(String.format("%s id=%s, error: %s", updated.getRecordBulkOperationEntity().getClass().getSimpleName(), "id", e.getMessage()));
-          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage());
+          errorService.saveError(rule.getBulkOperationId(), identifier, e.getMessage(), ErrorType.ERROR);
         }
       }
     }

src/main/java/org/folio/bulkops/processor/FolioAbstractUpdateProcessor.java

@@ -4,6 +4,7 @@
 
 import lombok.extern.log4j.Log4j2;
 import org.folio.bulkops.domain.bean.BulkOperationsEntity;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.entity.BulkOperation;
 import org.folio.bulkops.service.ErrorService;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -20,7 +21,7 @@ private void setErrorService(ErrorService errorService) {
   @Override
   public void updateAssociatedRecords(T t, BulkOperation operation, boolean notChanged) {
     if (notChanged) {
-      errorService.saveError(operation.getId(), t.getIdentifier(operation.getIdentifierType()), MSG_NO_CHANGE_REQUIRED);
+      errorService.saveError(operation.getId(), t.getIdentifier(operation.getIdentifierType()), MSG_NO_CHANGE_REQUIRED, ErrorType.WARNING);
     }
   }
 }

src/main/java/org/folio/bulkops/processor/folio/FolioInstanceUpdateProcessor.java

@@ -28,6 +28,7 @@
 import org.folio.bulkops.domain.bean.ItemCollection;
 import org.folio.bulkops.domain.dto.BulkOperationRule;
 import org.folio.bulkops.domain.dto.EntityType;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.entity.BulkOperation;
 import org.folio.bulkops.processor.FolioAbstractUpdateProcessor;
 import org.folio.bulkops.processor.permissions.check.PermissionsValidator;
@@ -83,7 +84,7 @@ public void updateAssociatedRecords(ExtendedInstance extendedInstance, BulkOpera
       .isPresent();
     if (notChanged) {
       var errorMessage = buildErrorMessage(recordsUpdated, instance.getDiscoverySuppress());
-      errorService.saveError(operation.getId(), instance.getIdentifier(operation.getIdentifierType()), errorMessage);
+      errorService.saveError(operation.getId(), instance.getIdentifier(operation.getIdentifierType()), errorMessage, ErrorType.WARNING);
     }
   }
 
@@ -114,7 +115,7 @@ private boolean applyRuleToAssociatedRecords(ExtendedInstance extendedInstance,
             for (var holdingId : consortiaHoldingsEntry.getValue()) {
               var errorMessage = String.format(ERROR_NO_AFFILIATION_TO_EDIT_HOLDINGS, user.getUsername(), holdingId, memberTenantForHoldings);
               log.error(errorMessage);
-              errorService.saveError(operation.getId(), instance.getIdentifier(operation.getIdentifierType()), errorMessage);
+              errorService.saveError(operation.getId(), instance.getIdentifier(operation.getIdentifierType()), errorMessage, ErrorType.ERROR);
             }
             continue;
           }

src/main/java/org/folio/bulkops/processor/folio/HoldingsUpdateProcessor.java

@@ -18,6 +18,7 @@
 import org.folio.bulkops.domain.bean.Item;
 import org.folio.bulkops.domain.dto.BulkOperationRule;
 import org.folio.bulkops.domain.dto.EntityType;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.entity.BulkOperation;
 import org.folio.bulkops.processor.FolioAbstractUpdateProcessor;
 import org.folio.bulkops.processor.permissions.check.PermissionsValidator;
@@ -91,7 +92,7 @@ public void updateAssociatedRecords(ExtendedHoldingsRecord extendedHoldingsRecor
     }
     if (notChanged) {
       var errorMessage = buildErrorMessage(itemsUpdated, holdingsRecord.getDiscoverySuppress());
-      errorService.saveError(operation.getId(), holdingsRecord.getIdentifier(operation.getIdentifierType()), errorMessage);
+      errorService.saveError(operation.getId(), holdingsRecord.getIdentifier(operation.getIdentifierType()), errorMessage, ErrorType.WARNING);
     }
   }
 

src/main/java/org/folio/bulkops/processor/marc/MarcInstanceDataProcessor.java

@@ -18,6 +18,7 @@
 import org.apache.commons.lang3.StringUtils;
 import org.folio.bulkops.domain.dto.BulkOperationMarcRule;
 import org.folio.bulkops.domain.dto.BulkOperationMarcRuleCollection;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.dto.MarcActionDataInner;
 import org.folio.bulkops.domain.dto.MarcDataType;
 import org.folio.bulkops.domain.entity.BulkOperation;
@@ -53,10 +54,10 @@ public void update(BulkOperation operation, Record marcRecord, BulkOperationMarc
         applyRuleToRecord(bulkOperationMarcRule, marcRecord);
       } catch (RuleValidationException e) {
         log.warn(String.format("Rule validation exception: %s", e.getMessage()));
-        errorService.saveError(operation.getId(), getIdentifier(operation, marcRecord), e.getMessage());
+        errorService.saveError(operation.getId(), getIdentifier(operation, marcRecord), e.getMessage(), ErrorType.ERROR);
       } catch (Exception e) {
         log.error(String.format("MARC record HRID=%s error: %s", marcRecord.getControlNumber(), e.getMessage()));
-        errorService.saveError(operation.getId(), getIdentifier(operation, marcRecord), e.getMessage());
+        errorService.saveError(operation.getId(), getIdentifier(operation, marcRecord), e.getMessage(), ErrorType.ERROR);
       }
     });
     var updatedRecord = marcRecord.toString();

src/main/java/org/folio/bulkops/processor/marc/MarcInstanceUpdateProcessor.java

@@ -72,6 +72,7 @@ public void updateMarcRecords(BulkOperation bulkOperation) throws IOException {
         bulkOperation.setLinkToCommittedRecordsMarcFile(null);
         bulkOperation.setLinkToCommittedRecordsErrorsCsvFile(errorService.uploadErrorsToStorage(bulkOperation.getId()));
         bulkOperation.setCommittedNumOfErrors(errorService.getCommittedNumOfErrors(bulkOperation.getId()));
+        bulkOperation.setCommittedNumOfWarnings(errorService.getCommittedNumOfWarnings(bulkOperation.getId()));
         bulkOperation.setEndTime(LocalDateTime.now());
         bulkOperation.setStatus(bulkOperation.getCommittedNumOfErrors() == 0 ? COMPLETED : COMPLETED_WITH_ERRORS);
       }

src/main/java/org/folio/bulkops/repository/BulkOperationExecutionContentRepository.java

@@ -3,6 +3,7 @@
 import java.util.Optional;
 import java.util.UUID;
 
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.entity.BulkOperationExecutionContent;
 import org.folio.spring.data.OffsetRequest;
 import org.springframework.data.domain.Page;
@@ -11,9 +12,10 @@
 
 @Repository
 public interface BulkOperationExecutionContentRepository extends JpaRepository<BulkOperationExecutionContent, UUID> {
-  Page<BulkOperationExecutionContent> findByBulkOperationIdAndErrorMessageIsNotNull(UUID bulkOperationId, OffsetRequest offsetRequest);
+  Page<BulkOperationExecutionContent> findByBulkOperationIdAndErrorMessageIsNotNullOrderByErrorType(UUID bulkOperationId, OffsetRequest offsetRequest);
+  Page<BulkOperationExecutionContent> findByBulkOperationIdAndErrorMessageIsNotNullAndErrorTypeIsOrderByErrorType(UUID bulkOperationId, OffsetRequest offsetRequest, ErrorType errorType);
   Optional<BulkOperationExecutionContent> findFirstByBulkOperationIdAndIdentifier(UUID bulkOperationId, String identifier);
-  int countAllByBulkOperationIdAndErrorMessageIsNotNull(UUID bulkOperationId);
+  int countAllByBulkOperationIdAndErrorMessageIsNotNullAndErrorTypeIs(UUID bulkOperationId, ErrorType errorType);
 
   void deleteByBulkOperationId(UUID bulkOperationId);
 }

src/main/java/org/folio/bulkops/service/BulkOperationService.java

@@ -69,6 +69,7 @@
 import org.folio.bulkops.domain.dto.BulkOperationStep;
 import org.folio.bulkops.domain.dto.DataImportJobExecution;
 import org.folio.bulkops.domain.dto.EntityType;
+import org.folio.bulkops.domain.dto.ErrorType;
 import org.folio.bulkops.domain.dto.IdentifierType;
 import org.folio.bulkops.domain.dto.OperationStatusType;
 import org.folio.bulkops.domain.dto.QueryRequest;
@@ -354,6 +355,7 @@ public void writeBeanToCsv(BulkOperation operation, BulkOperationsEntityCsvWrite
           .identifier(bean.getIdentifier(operation.getIdentifierType()))
           .bulkOperationId(operation.getId())
           .state(StateType.FAILED)
+          .errorType(e.getErrorType())
           .errorMessage(format(FIELD_ERROR_MESSAGE_PATTERN, e.getField().getName(), e.getMessage()))
           .build());
       }
@@ -442,13 +444,13 @@ public void commit(BulkOperation operation) {
               bulkOperationExecutionContents.forEach(errorService::saveError);
             }
           } catch (OptimisticLockingException e) {
-            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), e.getCsvErrorMessage(), e.getUiErrorMessage(), e.getLinkToFailedEntity());
+            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), e.getCsvErrorMessage(), e.getUiErrorMessage(), e.getLinkToFailedEntity(), ErrorType.ERROR);
           } catch (WritePermissionDoesNotExist e) {
             var userName = userClient.getUserById(folioExecutionContext.getUserId().toString()).getUsername();
             var errorMessage = String.format(e.getMessage(), userName, IdentifiersResolver.resolve(operation.getIdentifierType()), original.getIdentifier(operation.getIdentifierType()))  ;
-            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), errorMessage);
+            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), errorMessage, ErrorType.ERROR);
           } catch (Exception e) {
-            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), e.getMessage());
+            errorService.saveError(operationId, original.getIdentifier(operation.getIdentifierType()), e.getMessage(), ErrorType.ERROR);
           }
           execution = execution
             .withStatus(originalFileIterator.hasNext() ? StatusType.ACTIVE : StatusType.COMPLETED)
@@ -480,6 +482,7 @@ public void commit(BulkOperation operation) {
     var linkToCommittingErrorsFile = errorService.uploadErrorsToStorage(operationId);
     operation.setLinkToCommittedRecordsErrorsCsvFile(linkToCommittingErrorsFile);
     operation.setCommittedNumOfErrors(errorService.getCommittedNumOfErrors(operationId));
+    operation.setCommittedNumOfWarnings(errorService.getCommittedNumOfWarnings(operationId));
 
     if (!FAILED.equals(operation.getStatus())) {
       operation.setStatus(isEmpty(linkToCommittingErrorsFile) ? COMPLETED : COMPLETED_WITH_ERRORS);
@@ -603,7 +606,7 @@ public void apply(BulkOperation operation) {
           bulkOperationRepository.save(operation);
         }
       }
-      csvToBean.getCapturedExceptions().forEach(e -> errorService.saveError(operation.getId(), Utils.getIdentifierForManualApproach(e.getLine(), operation.getIdentifierType()), e.getMessage()));
+      csvToBean.getCapturedExceptions().forEach(e -> errorService.saveError(operation.getId(), Utils.getIdentifierForManualApproach(e.getLine(), operation.getIdentifierType()), e.getMessage(), ErrorType.ERROR));
       csvToBean.getCapturedExceptions().clear();
       operation.setProcessedNumOfRecords(processedNumOfRecords);
       operation.setStatus(REVIEW_CHANGES);