Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MODEXPW-529] Create CSV file for claims #609

Merged
merged 19 commits into from
Dec 19, 2024
Merged
Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
4f3cfee
[MODEXPW-529] Rename and restructure converters and mapper
Saba-Zedginidze-EPAM Dec 12, 2024
2b100ac
[MODEXPW-529] Rename converters
Saba-Zedginidze-EPAM Dec 12, 2024
c028c09
[MODEXPW-529] Add csv mapper and converter
Saba-Zedginidze-EPAM Dec 12, 2024
c624e54
[MODEXPW-529] Use new mapper together with old one in tasklets
Saba-Zedginidze-EPAM Dec 12, 2024
6a42965
[MODEXPW-529] Simplify existing acq tests to speed up builds
Saba-Zedginidze-EPAM Dec 12, 2024
860947d
[MODEXPW-529] Simplify edifact mapper test
Saba-Zedginidze-EPAM Dec 12, 2024
bff6250
[MODEXPW-529] Add CsvMapperTest (WIP)
Saba-Zedginidze-EPAM Dec 12, 2024
7ffb7d0
[MODEXPW-529] Fix failing tests
Saba-Zedginidze-EPAM Dec 13, 2024
f73ff45
[MODEXPW-529] Move tests to appropriate packages
Saba-Zedginidze-EPAM Dec 13, 2024
eb25f26
[MODEXPW-529] Finish CsvMapper test, todo 2 field extractions
Saba-Zedginidze-EPAM Dec 13, 2024
ff8bbab
[MODEXPW-529] Fix sonar issues
Saba-Zedginidze-EPAM Dec 13, 2024
75fedad
[MODEXPW-529] Add title extraction logic
Saba-Zedginidze-EPAM Dec 16, 2024
a77e468
[MODEXPW-529] Add quantity calculation logic
Saba-Zedginidze-EPAM Dec 16, 2024
86db180
[MODEXPW-529] Update pointer
Saba-Zedginidze-EPAM Dec 16, 2024
c1d4a97
[MODEXPW-529] Fix sonar issue
Saba-Zedginidze-EPAM Dec 17, 2024
7ad2c9f
Merge branch 'master' into MODEXPW-529
Saba-Zedginidze-EPAM Dec 17, 2024
db5ab0d
[MODEXPW-529] Improve grouping logic
Saba-Zedginidze-EPAM Dec 17, 2024
a3137fe
Merge remote-tracking branch 'origin/MODEXPW-529' into MODEXPW-529
Saba-Zedginidze-EPAM Dec 17, 2024
25898d0
[MODEXPW-529] Update pointer and rename export config
Saba-Zedginidze-EPAM Dec 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion folio-export-common

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.folio.dew.batch.acquisitions.edifact.config;

import org.folio.dew.batch.acquisitions.edifact.mapper.CsvMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.CompOrderEdiConverter;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.CompPoLineEdiConverter;
import org.folio.dew.batch.acquisitions.edifact.mapper.EdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.services.ConfigurationService;
import org.folio.dew.batch.acquisitions.edifact.services.ExpenseClassService;
import org.folio.dew.batch.acquisitions.edifact.services.HoldingService;
import org.folio.dew.batch.acquisitions.edifact.services.IdentifierTypeService;
import org.folio.dew.batch.acquisitions.edifact.services.LocationService;
import org.folio.dew.batch.acquisitions.edifact.services.MaterialTypeService;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;

@Configuration
@ComponentScan({ "org.folio.dew.batch.acquisitions.edifact" })
public class ExportConfig {
SerhiiNosko marked this conversation as resolved.
Show resolved Hide resolved

@Bean
CompPoLineEdiConverter compositePOLineConverter(IdentifierTypeService identifierTypeService, MaterialTypeService materialTypeService,
ExpenseClassService expenseClassService, LocationService locationService, HoldingService holdingService) {
return new CompPoLineEdiConverter(identifierTypeService, materialTypeService, expenseClassService, locationService, holdingService);
}

@Bean
CompOrderEdiConverter compositePurchaseOrderConverter(CompPoLineEdiConverter compPoLineEdiConverter, ConfigurationService configurationService) {
return new CompOrderEdiConverter(compPoLineEdiConverter, configurationService);
}

@Bean
ExportResourceMapper edifactMapper(CompOrderEdiConverter compOrderEdiConverter) {
return new EdifactMapper(compOrderEdiConverter);
}

@Bean
ExportResourceMapper csvMapper(OrdersService ordersService) {
return new CsvMapper(ordersService);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
import java.util.Map;

import org.apache.commons.collections4.CollectionUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.Piece;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.folio.dew.error.NotFoundException;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.scope.context.ChunkContext;
Expand All @@ -23,28 +23,41 @@
public class MapToEdifactClaimsTasklet extends MapToEdifactTasklet {

public static final String CLAIM_PIECE_IDS = "claimPieceIds";
private final ExportResourceMapper edifactMapper;
private final ExportResourceMapper csvMapper;

public MapToEdifactClaimsTasklet(ObjectMapper ediObjectMapper, OrdersService ordersService,
PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper) {
super(ediObjectMapper, ordersService, purchaseOrdersToEdifactMapper);
ExportResourceMapper edifactMapper, ExportResourceMapper csvMapper) {
super(ediObjectMapper, ordersService);
this.edifactMapper = edifactMapper;
this.csvMapper = csvMapper;
}

@Override
protected ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return switch (ediOrdersExportConfig.getFileFormat()) {
case EDI -> edifactMapper;
case CSV -> csvMapper;
};
}

@Override
protected List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return CollectionUtils.isEmpty(ediOrdersExportConfig.getClaimPieceIds())
? List.of(CLAIM_PIECE_IDS)
: List.of();
}

@Override
protected EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
protected ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
var pieces = ordersService.getPiecesByIdsAndReceivingStatus(ediExportConfig.getClaimPieceIds(), Piece.ReceivingStatusEnum.LATE);
if (pieces.isEmpty()) {
throw new NotFoundException(Piece.class);
}

var poLineQuery = convertIdsToCqlQuery(pieces.stream().map(Piece::getPoLineId).toList());
var compOrders = getCompositeOrders(poLineQuery);
return new EdifactExportHolder(compOrders, pieces);
return new ExportHolder(compOrders, pieces);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,13 @@
import java.util.stream.Collectors;

import org.apache.commons.collections4.CollectionUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.client.DataExportSpringClient;
import org.folio.dew.domain.dto.ExportConfigCollection;
import org.folio.dew.domain.dto.ExportType;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.springframework.batch.core.configuration.annotation.StepScope;
import org.springframework.batch.core.scope.context.ChunkContext;
import org.springframework.stereotype.Component;
Expand All @@ -34,23 +34,31 @@
public class MapToEdifactOrdersTasklet extends MapToEdifactTasklet {

private final DataExportSpringClient dataExportSpringClient;
private final ExportResourceMapper edifactMapper;

public MapToEdifactOrdersTasklet(ObjectMapper ediObjectMapper, OrdersService ordersService,
DataExportSpringClient dataExportSpringClient,
PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper) {
super(ediObjectMapper, ordersService, purchaseOrdersToEdifactMapper);
ExportResourceMapper edifactMapper) {
super(ediObjectMapper, ordersService);
this.edifactMapper = edifactMapper;
this.dataExportSpringClient = dataExportSpringClient;
}

@Override
protected List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return List.of();
}

@Override
protected EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
protected ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig, Map<String, Object> jobParameters) {
var poLineQuery = getPoLineQuery(ediExportConfig);
var compOrders = getCompositeOrders(poLineQuery);
return new EdifactExportHolder(compOrders, List.of());
return new ExportHolder(compOrders, List.of());
}

@Override
protected ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig) {
return edifactMapper;
}

protected String getPoLineQuery(VendorEdiOrdersExportConfig ediConfig) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,17 @@

import org.apache.commons.lang3.StringUtils;
import org.folio.dew.batch.ExecutionContextUtils;
import org.folio.dew.batch.acquisitions.edifact.PurchaseOrdersToEdifactMapper;
import org.folio.dew.batch.acquisitions.edifact.exceptions.CompositeOrderMappingException;
import org.folio.dew.batch.acquisitions.edifact.exceptions.EdifactException;
import org.folio.dew.batch.acquisitions.edifact.mapper.ExportResourceMapper;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.CompositePoLine;
import org.folio.dew.domain.dto.CompositePurchaseOrder;
import org.folio.dew.domain.dto.JobParameterNames;
import org.folio.dew.domain.dto.PoLine;
import org.folio.dew.domain.dto.PurchaseOrder;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.EdifactExportHolder;
import org.folio.dew.domain.dto.acquisitions.edifact.ExportHolder;
import org.folio.dew.error.NotFoundException;
import org.springframework.batch.core.StepContribution;
import org.springframework.batch.core.scope.context.ChunkContext;
Expand All @@ -41,7 +41,6 @@ public abstract class MapToEdifactTasklet implements Tasklet {

private final ObjectMapper ediObjectMapper;
protected final OrdersService ordersService;
private final PurchaseOrdersToEdifactMapper purchaseOrdersToEdifactMapper;

@Override
public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception {
Expand All @@ -54,7 +53,7 @@ public RepeatStatus execute(StepContribution contribution, ChunkContext chunkCon
persistPoLineIds(chunkContext, holder.orders());

String jobName = jobParameters.get(JobParameterNames.JOB_NAME).toString();
var edifactStringResult = purchaseOrdersToEdifactMapper.convertOrdersToEdifact(holder.orders(), holder.pieces(), ediExportConfig, jobName);
var edifactStringResult = getExportResourceMapper(ediExportConfig).convertForExport(holder.orders(), holder.pieces(), ediExportConfig, jobName);

// save edifact file content in memory
ExecutionContextUtils.addToJobExecutionContext(chunkContext.getStepContext().getStepExecution(), "edifactOrderAsString", edifactStringResult, "");
Expand Down Expand Up @@ -122,9 +121,11 @@ private <T> T convertTo(Object value, Class<T> c) {
}
}

protected abstract ExportResourceMapper getExportResourceMapper(VendorEdiOrdersExportConfig ediOrdersExportConfig);

protected abstract List<String> getExportConfigMissingFields(VendorEdiOrdersExportConfig ediOrdersExportConfig);

protected abstract EdifactExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig,
Map<String, Object> jobParameters) throws JsonProcessingException, EDIStreamException;
protected abstract ExportHolder buildEdifactExportHolder(ChunkContext chunkContext, VendorEdiOrdersExportConfig ediExportConfig,
Map<String, Object> jobParameters) throws JsonProcessingException, EDIStreamException;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package org.folio.dew.batch.acquisitions.edifact.mapper;

import static java.util.stream.Collectors.groupingBy;
import static org.folio.dew.utils.Constants.LINE_BREAK;

import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.commons.lang3.tuple.Pair;
import org.folio.dew.batch.acquisitions.edifact.mapper.converter.ClaimCsvConverter;
import org.folio.dew.batch.acquisitions.edifact.services.OrdersService;
import org.folio.dew.domain.dto.CompositePoLine;
import org.folio.dew.domain.dto.CompositePurchaseOrder;
import org.folio.dew.domain.dto.Piece;
import org.folio.dew.domain.dto.VendorEdiOrdersExportConfig;
import org.folio.dew.domain.dto.acquisitions.edifact.ClaimCsvEntry;

public class CsvMapper implements ExportResourceMapper {

private final OrdersService ordersService;

public CsvMapper(OrdersService ordersService) {
this.ordersService = ordersService;
}

@Override
public String convertForExport(List<CompositePurchaseOrder> compPOs, List<Piece> pieces, VendorEdiOrdersExportConfig ediExportConfig, String jobName) {
var claimCsvConverter = new ClaimCsvConverter();
var csvResult = new StringBuilder(claimCsvConverter.getCsvHeaders()).append(LINE_BREAK);
getClaimEntries(compPOs, pieces).stream()
.map(claimCsvConverter::convertEntryToCsv)
.map(line -> line.concat(LINE_BREAK))
.forEachOrdered(csvResult::append);
return csvResult.toString();
}

private List<ClaimCsvEntry> getClaimEntries(List<CompositePurchaseOrder> orders, List<Piece> pieces) {
// Map each PoLine ID to its corresponding Pieces
var poLineIdToPieces = pieces.stream().collect(groupingBy(Piece::getPoLineId));
// Map each PoLine ID to its corresponding PoLine
var poLineIdToPoLine = orders.stream().flatMap(order -> order.getCompositePoLines().stream())
.collect(Collectors.toMap(CompositePoLine::getId, Function.identity()));
// Map each Piece ID to its corresponding Title
var pieceIdToTitle = poLineIdToPieces.entrySet().stream()
.flatMap(entry -> entry.getValue().stream()
.map(piece -> Pair.of(piece.getId(), getTitleById(poLineIdToPoLine.get(entry.getKey()), piece)))) // Pair of Piece ID and Title
.collect(Collectors.toMap(Pair::getLeft, Pair::getRight));

// Key extractor for grouping pieces by: Po Line Number, Display Summary, Chronology, Enumeration, and Title
Function<Piece, ClaimCsvEntry> keyExtractor = piece ->
new ClaimCsvEntry(poLineIdToPoLine.get(piece.getPoLineId()), piece, pieceIdToTitle.get(piece.getId()), 0);

// Group pieces by the previously defined key (Overridden equals and hashCode methods in ClaimCsvEntry)
// Only a single piece from each group is used, as they share all necessary attributes
Map<ClaimCsvEntry, Long> claimedPieces = pieces.stream()
.collect(Collectors.groupingBy(keyExtractor, Collectors.counting()));

// Return a list of ClaimCsvEntry objects, each representing a group of claimed pieces
return claimedPieces.entrySet().stream()
.map(entry -> entry.getKey().withQuantity(entry.getValue()))
.sorted(Comparator.comparing(o -> o.compositePoLine().getPoLineNumber()))
.toList();
}

private String getTitleById(CompositePoLine poLine, Piece piece) {
return Boolean.TRUE.equals(poLine.getIsPackage())
? ordersService.getTitleById(piece.getTitleId()).getTitle()
: poLine.getTitleOrPackage();
}

}
Loading
Loading