Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[MODFIN-391]. Minimize amount of requests to retrieve transactions for ledger #270

Merged
merged 7 commits into from
Jan 9, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
@@ -343,6 +343,7 @@
"finance-storage.budgets.collection.get",
"finance-storage.fiscal-years.item.get",
"finance-storage.transactions.collection.get",
"finance-storage.transaction-totals.collection.get",
"acquisitions-units-storage.units.collection.get",
"acquisitions-units-storage.memberships.collection.get"
]
@@ -364,6 +365,7 @@
"finance-storage.budgets.collection.get",
"finance-storage.fiscal-years.item.get",
"finance-storage.transactions.collection.get",
"finance-storage.transaction-totals.collection.get",
"acquisitions-units-storage.units.collection.get",
"acquisitions-units-storage.memberships.collection.get"
]
@@ -865,6 +867,10 @@
"id": "finance-storage.transactions",
"version":"5.0"
},
{
"id": "finance-storage.transaction-totals",
"version": "1.0"
},
{
"id": "configuration",
"version":"2.0"
2 changes: 2 additions & 0 deletions ramls/transaction.raml
Original file line number Diff line number Diff line change
@@ -11,6 +11,8 @@ types:
errors: !include raml-util/schemas/errors.schema
transaction: !include acq-models/mod-finance/schemas/transaction.json
transaction-collection: !include acq-models/mod-finance/schemas/transaction_collection.json
transaction-total: !include acq-models/mod-finance/schemas/transaction_total.json
transaction-total-collection: !include acq-models/mod-finance/schemas/transaction_total_collection.json
batch: !include acq-models/mod-finance/schemas/batch.json
UUID:
type: string
10 changes: 8 additions & 2 deletions src/main/java/org/folio/config/ServicesConfiguration.java
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@
import org.folio.services.protection.ProtectionService;
import org.folio.services.transactions.TransactionApiService;
import org.folio.services.transactions.TransactionService;
import org.folio.services.transactions.TransactionTotalService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;

@@ -64,6 +65,11 @@ TransactionService transactionService(RestClient restClient, FiscalYearService f
return new TransactionService(restClient, fiscalYearService);
}

@Bean
TransactionTotalService transactionTotalService(RestClient restClient) {
return new TransactionTotalService(restClient);
}

@Bean
BudgetExpenseClassTotalsService budgetExpenseClassTotalsService(RestClient restClient,
ExpenseClassService expenseClassService,
@@ -133,8 +139,8 @@ LedgerRolloverBudgetsService ledgerRolloverBudgetsService(RestClient restClient)

@Bean
LedgerTotalsService ledgerTotalsService(FiscalYearService fiscalYearService, BudgetService budgetService,
TransactionService transactionService) {
return new LedgerTotalsService(fiscalYearService, budgetService, transactionService);
TransactionTotalService transactionTotalService) {
return new LedgerTotalsService(fiscalYearService, budgetService, transactionTotalService);
}

@Bean
Original file line number Diff line number Diff line change
@@ -6,17 +6,17 @@

import org.folio.rest.jaxrs.model.Budget;
import org.folio.rest.jaxrs.model.Ledger;
import org.folio.rest.jaxrs.model.Transaction;
import org.folio.rest.jaxrs.model.TransactionTotal;

public class LedgerFiscalYearTransactionsHolder {
private final String fiscalYearId;
private final Ledger ledger;
private final List<Budget> ledgerBudgets;
private final List<String> ledgerFundIds;
private List<Transaction> toAllocations;
private List<Transaction> fromAllocations;
private List<Transaction> toTransfers;
private List<Transaction> fromTransfers;
private List<TransactionTotal> toAllocations;
private List<TransactionTotal> fromAllocations;
private List<TransactionTotal> toTransfers;
private List<TransactionTotal> fromTransfers;

public LedgerFiscalYearTransactionsHolder( String fiscalYearId, Ledger ledger, List<Budget> ledgerBudgets) {
this.fiscalYearId = fiscalYearId;
@@ -29,22 +29,22 @@ public LedgerFiscalYearTransactionsHolder( String fiscalYearId, Ledger ledger, L
this.fromTransfers = new ArrayList<>();
}

public LedgerFiscalYearTransactionsHolder withToAllocations(List<Transaction> allocations) {
public LedgerFiscalYearTransactionsHolder withToAllocations(List<TransactionTotal> allocations) {
this.toAllocations = allocations;
return this;
}

public LedgerFiscalYearTransactionsHolder withFromAllocations(List<Transaction> allocations) {
public LedgerFiscalYearTransactionsHolder withFromAllocations(List<TransactionTotal> allocations) {
this.fromAllocations = allocations;
return this;
}

public LedgerFiscalYearTransactionsHolder withToTransfers(List<Transaction> transfers) {
public LedgerFiscalYearTransactionsHolder withToTransfers(List<TransactionTotal> transfers) {
this.toTransfers = transfers;
return this;
}

public LedgerFiscalYearTransactionsHolder withFromTransfers(List<Transaction> transfers) {
public LedgerFiscalYearTransactionsHolder withFromTransfers(List<TransactionTotal> transfers) {
this.fromTransfers = transfers;
return this;
}
@@ -57,19 +57,19 @@ public Ledger getLedger() {
return this.ledger;
}

public List<Transaction> getFromAllocations() {
public List<TransactionTotal> getFromAllocations() {
return fromAllocations;
}

public List<Transaction> getToAllocations() {
public List<TransactionTotal> getToAllocations() {
return toAllocations;
}

public List<Transaction> getToTransfers() {
public List<TransactionTotal> getToTransfers() {
return toTransfers;
}

public List<Transaction> getFromTransfers() {
public List<TransactionTotal> getFromTransfers() {
return fromTransfers;
}

13 changes: 13 additions & 0 deletions src/main/java/org/folio/models/TransactionTotalSetting.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.folio.models;

import lombok.AllArgsConstructor;
import lombok.Getter;

@Getter
@AllArgsConstructor
public enum TransactionTotalSetting {
FROM_FUND_ID("fromFundId"),
TO_FUND_ID("toFundId");

private final String value;
}
2 changes: 2 additions & 0 deletions src/main/java/org/folio/rest/util/BudgetUtils.java
Original file line number Diff line number Diff line change
@@ -10,10 +10,12 @@
import org.folio.rest.jaxrs.model.Transaction.TransactionType;

import io.vertx.core.json.JsonObject;
import org.folio.rest.jaxrs.model.TransactionTotal;

public final class BudgetUtils {

public static final List<TransactionType> TRANSFER_TRANSACTION_TYPES = List.of(TransactionType.TRANSFER, TransactionType.ROLLOVER_TRANSFER);
public static final List<TransactionTotal.TransactionType> TRANSFER_TRANSACTION_TOTAL_TYPES = List.of(TransactionTotal.TransactionType.TRANSFER, TransactionTotal.TransactionType.ROLLOVER_TRANSFER);

private BudgetUtils() {

2 changes: 2 additions & 0 deletions src/main/java/org/folio/rest/util/ResourcePathResolver.java
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ private ResourcePathResolver() {
public static final String LEDGER_ROLLOVERS_PROGRESS_STORAGE = "ledgerRolloverProgress";
public static final String GROUPS = "groups";
public static final String TRANSACTIONS = "transactions";
public static final String TRANSACTION_TOTALS = "transactionTotals";
public static final String BATCH_TRANSACTIONS = "batchTransactions";
public static final String BATCH_TRANSACTIONS_STORAGE = "batchTransactionsStorage";
public static final String CONFIGURATIONS = "configurations";
@@ -53,6 +54,7 @@ private ResourcePathResolver() {
apis.put(LEDGER_ROLLOVERS_PROGRESS_STORAGE, "/finance-storage/ledger-rollovers-progress");
apis.put(GROUPS, "/finance-storage/groups");
apis.put(TRANSACTIONS, "/finance-storage/transactions");
apis.put(TRANSACTION_TOTALS, "/finance-storage/transaction-totals");
apis.put(BATCH_TRANSACTIONS, "/finance/transactions/batch-all-or-nothing");
apis.put(BATCH_TRANSACTIONS_STORAGE, "/finance-storage/transactions/batch-all-or-nothing");
apis.put(CONFIGURATIONS, "/configurations/entries");
33 changes: 15 additions & 18 deletions src/main/java/org/folio/services/ledger/LedgerTotalsService.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package org.folio.services.ledger;

import static org.folio.rest.util.BudgetUtils.TRANSFER_TRANSACTION_TYPES;
import static org.folio.rest.util.BudgetUtils.TRANSFER_TRANSACTION_TOTAL_TYPES;
import static org.folio.rest.util.HelperUtils.collectResultsOnSuccess;
import static org.folio.rest.util.HelperUtils.removeInitialAllocationByFunds;

import java.math.BigDecimal;
import java.util.ArrayList;
@@ -20,15 +19,14 @@
import org.folio.rest.jaxrs.model.FiscalYear;
import org.folio.rest.jaxrs.model.Ledger;
import org.folio.rest.jaxrs.model.LedgersCollection;
import org.folio.rest.jaxrs.model.Transaction;
import org.folio.rest.jaxrs.model.Transaction.TransactionType;
import org.folio.rest.jaxrs.model.TransactionTotal;
import org.folio.rest.util.ErrorCodes;
import org.folio.rest.util.HelperUtils;
import org.folio.services.budget.BudgetService;
import org.folio.services.fiscalyear.FiscalYearService;

import io.vertx.core.Future;
import org.folio.services.transactions.TransactionService;
import org.folio.services.transactions.TransactionTotalService;

public class LedgerTotalsService {

@@ -37,12 +35,13 @@ public class LedgerTotalsService {

private final FiscalYearService fiscalYearService;
private final BudgetService budgetService;
private final TransactionService transactionService;
private final TransactionTotalService transactionTotalService;

public LedgerTotalsService(FiscalYearService fiscalYearService, BudgetService budgetService, TransactionService transactionService) {
public LedgerTotalsService(FiscalYearService fiscalYearService, BudgetService budgetService,
TransactionTotalService transactionTotalService) {
this.fiscalYearService = fiscalYearService;
this.budgetService = budgetService;
this.transactionService = transactionService;
this.transactionTotalService = transactionTotalService;
}

public Future<Ledger> populateLedgerTotals(Ledger ledger, String fiscalYearId, RequestContext requestContext) {
@@ -80,18 +79,17 @@ public Future<Ledger> populateLedgerTotals(Ledger ledger, FiscalYear fiscalYear,

private void updateLedgerWithAllocation(LedgerFiscalYearTransactionsHolder holder) {
holder.withToAllocations(new ArrayList<>(holder.getToAllocations()));
removeInitialAllocationByFunds(holder.getToAllocations());
Ledger ledger = holder.getLedger();
ledger.withAllocationTo(HelperUtils.calculateTotals(holder.getToAllocations(), Transaction::getAmount))
.withAllocationFrom(HelperUtils.calculateTotals(holder.getFromAllocations(), Transaction::getAmount));
ledger.withAllocationTo(HelperUtils.calculateTotals(holder.getToAllocations(), TransactionTotal::getAmount))
.withAllocationFrom(HelperUtils.calculateTotals(holder.getFromAllocations(), TransactionTotal::getAmount));
}

private Future<LedgerFiscalYearTransactionsHolder> updateHolderWithAllocations(LedgerFiscalYearTransactionsHolder holder,
RequestContext requestContext) {
List<String> ledgerFundIds = holder.getLedgerFundIds();
String fiscalYearId = holder.getFiscalYearId();
var fromAllocations = transactionService.getTransactionsFromFunds(ledgerFundIds, fiscalYearId, List.of(TransactionType.ALLOCATION), requestContext);
var toAllocations = transactionService.getTransactionsToFunds(ledgerFundIds, fiscalYearId, List.of(TransactionType.ALLOCATION), requestContext);
var fromAllocations = transactionTotalService.getTransactionsFromFunds(ledgerFundIds, fiscalYearId, List.of(TransactionTotal.TransactionType.ALLOCATION), requestContext);
var toAllocations = transactionTotalService.getTransactionsToFunds(ledgerFundIds, fiscalYearId, List.of(TransactionTotal.TransactionType.ALLOCATION), requestContext);
return GenericCompositeFuture.join(List.of(fromAllocations, toAllocations))
.map(cf -> holder.withToAllocations(toAllocations.result()).withFromAllocations(fromAllocations.result()));
}
@@ -100,9 +98,8 @@ private Future<LedgerFiscalYearTransactionsHolder> updateHolderWithTransfers(Led
RequestContext requestContext) {
List<String> ledgerFundIds = holder.getLedgerFundIds();
String fiscalYearId = holder.getFiscalYearId();
var fromTransfer = transactionService.getTransactionsFromFunds(ledgerFundIds, fiscalYearId, TRANSFER_TRANSACTION_TYPES, requestContext);
var toTransfer = transactionService.getTransactionsToFunds(ledgerFundIds, fiscalYearId, TRANSFER_TRANSACTION_TYPES, requestContext);

var fromTransfer = transactionTotalService.getTransactionsFromFunds(ledgerFundIds, fiscalYearId, TRANSFER_TRANSACTION_TOTAL_TYPES, requestContext);
var toTransfer = transactionTotalService.getTransactionsToFunds(ledgerFundIds, fiscalYearId, TRANSFER_TRANSACTION_TOTAL_TYPES, requestContext);
return GenericCompositeFuture.join(List.of(fromTransfer, toTransfer))
.map(f -> holder.withToTransfers(toTransfer.result()).withFromTransfers(fromTransfer.result()));
}
@@ -146,8 +143,8 @@ private LedgerFiscalYearTransactionsHolder buildHolderSkeleton(String fiscalYear
*/
private void updateLedgerWithCalculatedFields(LedgerFiscalYearTransactionsHolder holder) {
Ledger ledger = holder.getLedger();
double toTransfer = HelperUtils.calculateTotals(holder.getToTransfers(), Transaction::getAmount);
double fromTransfer = HelperUtils.calculateTotals(holder.getFromTransfers(), Transaction::getAmount);
double toTransfer = HelperUtils.calculateTotals(holder.getToTransfers(), TransactionTotal::getAmount);
double fromTransfer = HelperUtils.calculateTotals(holder.getFromTransfers(), TransactionTotal::getAmount);
BigDecimal netTransfers = BigDecimal.valueOf(toTransfer).subtract(BigDecimal.valueOf(fromTransfer));
ledger.withNetTransfers(netTransfers.doubleValue());

Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package org.folio.services.transactions;

import io.vertx.core.Future;
import lombok.extern.log4j.Log4j2;
import org.folio.models.TransactionTotalSetting;
import org.folio.rest.core.RestClient;
import org.folio.rest.core.models.RequestContext;
import org.folio.rest.core.models.RequestEntry;
import org.folio.rest.jaxrs.model.TransactionTotal;
import org.folio.rest.jaxrs.model.TransactionTotalCollection;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Function;

import static one.util.streamex.StreamEx.ofSubLists;
import static org.folio.rest.util.HelperUtils.collectResultsOnSuccess;
import static org.folio.rest.util.HelperUtils.convertIdsToCqlQuery;
import static org.folio.rest.util.ResourcePathResolver.TRANSACTION_TOTALS;
import static org.folio.rest.util.ResourcePathResolver.resourcesPath;

@Log4j2
public class TransactionTotalService {

private static final int MAX_FUND_PER_QUERY = 5;

private final RestClient restClient;

public TransactionTotalService(RestClient restClient) {
this.restClient = restClient;
}

public Future<List<TransactionTotal>> getTransactionsFromFunds(List<String> fundIds, String fiscalYearId,
List<TransactionTotal.TransactionType> trTypes, RequestContext requestContext) {
return getTransactionsFromOrToFunds(fundIds, fiscalYearId, trTypes, TransactionTotalSetting.FROM_FUND_ID, requestContext);
}

public Future<List<TransactionTotal>> getTransactionsToFunds(List<String> fundIds, String fiscalYearId,
List<TransactionTotal.TransactionType> trTypes, RequestContext requestContext) {
return getTransactionsFromOrToFunds(fundIds, fiscalYearId, trTypes, TransactionTotalSetting.TO_FUND_ID, requestContext);
}

private Future<List<TransactionTotal>> getTransactionsFromOrToFunds(List<String> fundIds, String fiscalYearId, List<TransactionTotal.TransactionType> trTypes,
TransactionTotalSetting setting, RequestContext requestContext) {
return collectResultsOnSuccess(ofSubLists(new ArrayList<>(fundIds), MAX_FUND_PER_QUERY)
.map(ids -> getTransactionsByFundChunk(ids, fiscalYearId, trTypes, setting, requestContext)
.map(transactions -> filterFundIdsByAllocationDirection(fundIds, transactions, setting)))
.toList())
.map(lists -> lists.stream().flatMap(Collection::stream).toList());
}

private Future<List<TransactionTotal>> getTransactionsByFundChunk(List<String> fundIds, String fiscalYearId, List<TransactionTotal.TransactionType> trTypes,
TransactionTotalSetting setting, RequestContext requestContext) {
var fundQuery = convertIdsToCqlQuery(fundIds, setting.getValue(), "==", " OR ");
var trTypeValues = trTypes.stream().map(TransactionTotal.TransactionType::value).toList();
var trTypeQuery = convertIdsToCqlQuery(trTypeValues, "transactionType", "==", " OR ");
var query = String.format("(fiscalYearId==%s AND %s) AND %s", fiscalYearId, trTypeQuery, fundQuery);
return getAllTransactionsByQuery(query, requestContext);
}

private List<TransactionTotal> filterFundIdsByAllocationDirection(List<String> fundIds, List<TransactionTotal> transactions,
TransactionTotalSetting setting) {
// Note that here getToFundId() is used when direction is from (a negation is used afterward)
Function<TransactionTotal, String> getFundId = setting == TransactionTotalSetting.FROM_FUND_ID
? TransactionTotal::getToFundId : TransactionTotal::getFromFundId;
return transactions.stream()
.filter(transaction -> !fundIds.contains(getFundId.apply(transaction)))
.toList();
}

private Future<List<TransactionTotal>> getAllTransactionsByQuery(String query, RequestContext requestContext) {
return getTransactionCollectionByQuery(query, requestContext)
.map(TransactionTotalCollection::getTransactionTotals);
}

private Future<TransactionTotalCollection> getTransactionCollectionByQuery(String query, RequestContext requestContext) {
var requestEntry = new RequestEntry(resourcesPath(TRANSACTION_TOTALS))
.withOffset(0)
.withLimit(Integer.MAX_VALUE)
.withQuery(query);
return restClient.get(requestEntry.buildEndpoint(), TransactionTotalCollection.class, requestContext);
}
}
4 changes: 4 additions & 0 deletions src/test/java/org/folio/ApiTestSuite.java
Original file line number Diff line number Diff line change
@@ -54,6 +54,7 @@
import org.folio.services.protection.AcqUnitsServiceTest;
import org.folio.services.protection.ProtectionServiceTest;
import org.folio.services.transactions.TransactionApiServiceTest;
import org.folio.services.transactions.TransactionTotalApiServiceTest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Nested;
@@ -251,4 +252,7 @@ class FinanceDataApiTestNested extends FinanceDataApiTest {}

@Nested
class FinanceDataServiceTestNested extends FinanceDataServiceTest {}

@Nested
class TransactionTotalApiServiceTestNested extends TransactionTotalApiServiceTest {}
}
Loading
Loading