Skip to content

Commit

Permalink
Merge pull request #14 from transcom/Performance-Update
Browse files Browse the repository at this point in the history 
Performance Update to identifying duplicates
  • Loading branch information
@TevinAdams
TevinAdams authored Jun 21, 2024
2 parents 28e7599 + e196c73 commit 9f138a0
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 36 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
</parent>
<groupId>com.milmove.trdmlambda</groupId>
<artifactId>trdm-lambda</artifactId>
<version>1.0.3.7</version>
<version>1.0.3.8</version>
<name>trdm java spring interface</name>
<description>Project for deploying a Java TRDM interfacer for TGET data.</description>
<properties>
Expand Down
40 changes: 32 additions & 8 deletions src/main/java/com/milmove/trdmlambda/milmove/service/DatabaseService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,12 +76,10 @@ public void updateTransportationAccountingCodes(List<TransportationAccountingCod
logger.info("updating " + codes.size() + " TAC(s)...");
logger.info(codes.toString());


String sql = "UPDATE transportation_accounting_codes SET tac=?, loa_id=?, loa_sys_id=?, tac_fy_txt=?, tac_fn_bl_mod_cd=?, org_grp_dfas_cd=?, tac_mvt_dsg_id=?, tac_ty_cd=?, tac_use_cd=?, tac_maj_clmt_id=?, tac_bill_act_txt=?, tac_cost_ctr_nm=?, buic=?, tac_hist_cd=?, tac_stat_cd=?, trnsprtn_acnt_tx=?, trnsprtn_acnt_bgn_dt=?, trnsprtn_acnt_end_dt=?, dd_actvty_adrs_id=?, tac_blld_add_frst_ln_tx=?, tac_blld_add_scnd_ln_tx=?, tac_blld_add_thrd_ln_tx=?, tac_blld_add_frth_ln_tx=?, tac_fnct_poc_nm=?, updated_at=? WHERE id=?";

Connection conn = this.getConnection();
conn.setAutoCommit(false);

try (PreparedStatement pstmt = conn.prepareStatement(sql)) {

int count = 0;
Expand Down Expand Up @@ -258,7 +256,6 @@ public void updateLinesOfAccountingCodes(List<LineOfAccounting> codes) throws SQ
pstmt.setTimestamp(57, java.sql.Timestamp.valueOf(LocalDateTime.now()));
pstmt.setObject(58, code.getId());


pstmt.addBatch();

// Execute every 10000 rows or when finished with the provided LOAs
Expand Down Expand Up @@ -411,7 +408,8 @@ public ArrayList<LineOfAccounting> dbLoasToModel(ResultSet rs) throws SQLExcepti

ArrayList<LineOfAccounting> loas = new ArrayList<LineOfAccounting>();

// Loas created times are in 2 different formats. some have 25 characters and some have 26. Based on their character length will choose the formatter
// Loas created times are in 2 different formats. some have 25 characters and
// some have 26. Based on their character length will choose the formatter
DateTimeFormatter timeFormatterLen25 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSS");
DateTimeFormatter timeFormatterLen26 = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSS");

Expand All @@ -422,9 +420,11 @@ public ArrayList<LineOfAccounting> dbLoasToModel(ResultSet rs) throws SQLExcepti
loa.setLoaDptID(rs.getString(LinesOfAccountingDatabaseColumns.loaDptId));

if (rs.getString(LinesOfAccountingDatabaseColumns.updatedAt).length() == 25) {
loa.setUpdatedAt(LocalDateTime.parse(rs.getString(LinesOfAccountingDatabaseColumns.updatedAt), timeFormatterLen25));
loa.setUpdatedAt(LocalDateTime.parse(rs.getString(LinesOfAccountingDatabaseColumns.updatedAt),
timeFormatterLen25));
} else if (rs.getString(LinesOfAccountingDatabaseColumns.updatedAt).length() == 26) {
loa.setUpdatedAt(LocalDateTime.parse(rs.getString(LinesOfAccountingDatabaseColumns.updatedAt), timeFormatterLen26));
loa.setUpdatedAt(LocalDateTime.parse(rs.getString(LinesOfAccountingDatabaseColumns.updatedAt),
timeFormatterLen26));
}

loas.add(loa);
Expand All @@ -449,10 +449,10 @@ public ArrayList<TransportationAccountingCode> getCurrentTacInformation() throws
TransportationAccountingCode tac = new TransportationAccountingCode();
tac.setId(UUID.fromString(rs.getString(TransportationAccountingCodesDatabaseColumns.id)));
if (rs.getString(TransportationAccountingCodesDatabaseColumns.loaId) != null
&& rs.getString(TransportationAccountingCodesDatabaseColumns.loaId) != "null") {
&& rs.getString(TransportationAccountingCodesDatabaseColumns.loaId) != "null") {
tac.setLoaID(UUID.fromString(rs.getString(TransportationAccountingCodesDatabaseColumns.loaId)));
} else {
UUID nilUUID = new UUID(0,0); // represents a nil UUID
UUID nilUUID = new UUID(0, 0); // represents a nil UUID
tac.setLoaID(nilUUID);
}
tac.setTac(rs.getString(TransportationAccountingCodesDatabaseColumns.tac));
Expand All @@ -464,4 +464,28 @@ public ArrayList<TransportationAccountingCode> getCurrentTacInformation() throws
}

}

// Get LOA loa_sys_ids with a count greater than 1
public ArrayList<String> getLoaSysIdCountGreaterThan1() throws SQLException {

logger.info("retrieving loa_sys_ids with a count greater than 1");

ArrayList<String> loaSysIds = new ArrayList<String>();

// Select loa_sys_ids with a count greater than 1
String sql = "SELECT loa_sys_id, COUNT(*) FROM lines_of_accounting GROUP BY loa_sys_id HAVING count(loa_sys_id) > 1";

try (Connection conn = this.getConnection(); PreparedStatement pstmt = conn.prepareStatement(sql)) {
ResultSet rs = pstmt.executeQuery();
while (rs.next()) {
loaSysIds.add(rs.getString(LinesOfAccountingDatabaseColumns.loaSysId));
}

logger.info("finished retrieving loa_sys_ids with a count greater than 1. " + loaSysIds.size()
+ " non-unique loaSysIds.");

return loaSysIds;
}

}
}
33 changes: 8 additions & 25 deletions src/main/java/com/milmove/trdmlambda/milmove/util/Trdm.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,11 @@ public void UpdateTGETData(XMLGregorianCalendar ourLastUpdate, String trdmTable,
// Get all loas
ArrayList<LineOfAccounting> currentLoas = databaseService.getCurrentLoaInformation();

// Get all loasSysIds that occur more than once
ArrayList<String> duplicateLoaSysIds = databaseService.getLoaSysIdCountGreaterThan1();

// Identify Loas to delete based on if their loaSysId is not unique, their id/primary key is not referenced in TACS loa_id and the loa is the latest
ArrayList<LineOfAccounting> loasToDelete = identifyDuplicateLoasToDelete(currentLoas, currentTacs);
ArrayList<LineOfAccounting> loasToDelete = identifyDuplicateLoasToDelete(currentLoas, currentTacs, duplicateLoaSysIds);

// Delete duplicate Loas
databaseService.deleteLoas(loasToDelete);
Expand Down Expand Up @@ -280,42 +283,22 @@ public List<LineOfAccounting> identifyLoasToCreate(List<LineOfAccounting> newLoa


// Identify Loas to delete based on if their loaSysId is not unique, their id/primary key is not referenced in TACS loa_id and the loa created_at is the latest
public ArrayList<LineOfAccounting> identifyDuplicateLoasToDelete(ArrayList<LineOfAccounting> loas, ArrayList<TransportationAccountingCode> tacs) throws SQLException {
public ArrayList<LineOfAccounting> identifyDuplicateLoasToDelete(ArrayList<LineOfAccounting> loas, ArrayList<TransportationAccountingCode> tacs, ArrayList<String> duplicateLoaSysIds) throws SQLException {
logger.info("identifying duplicate Line of Accounting codes to delete");
logger.info("LOA codes count: " + loas.size());
logger.info("TAC codes count: " + tacs.size());
logger.info("Duplicate LOA codes loa_sys_ids count: " + duplicateLoaSysIds.size());

// Store loas that needs to be checked for deletion
ArrayList<LineOfAccounting> duplicateLoas = new ArrayList<LineOfAccounting>();

// Store duplicate loa_sys_id
ArrayList<String> duplicateLoaSysIds = new ArrayList<String>();

logger.info("starting to identify LOA codes with non unique loa_sys_ids");
logger.info("starting to identify duplicate LOA codes");
for (LineOfAccounting loa : loas) {

// If already confirmed a duplicate then no need to check if it is
boolean alreadyConfirmedDupe = false;
if (duplicateLoaSysIds.contains(loa.getLoaSysID())) {
alreadyConfirmedDupe = true;
duplicateLoas.add(loa);
}

// If not already confirmed a duplicate check to see if it is a duplicate
if (!alreadyConfirmedDupe) {

// Check if there are more than 1 loa with this loa_sys_id if so enter if logic
List<LineOfAccounting> l1 = loas.stream().filter(l -> l.getLoaSysID().equals(loa.getLoaSysID())).collect(Collectors.toList());
if (l1.size() > 1) {

// Add loaSysId to duplicateLoaSysIds another loa with the same sysId is being checked it will not have to run through this logic
duplicateLoaSysIds.add(loa.getLoaSysID());
duplicateLoas.add(loa);
}
}
}

logger.info("finished identifying LOA codes with non unique loa_sys_ids. Count: " + duplicateLoas.size());
logger.info("finished identifying duplicate LOA codes. Count: " + duplicateLoas.size());

logger.info("starting to identify duplicate LOA codes that are not referenced by a TAC code");
// Duplicate loas not referenced in TACS
Expand Down
43 changes: 43 additions & 0 deletions src/test/java/com/milmove/trdmlambda/milmove/util/DatabaseServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,49 @@ void testUpdateTransportationAccountingCodes() throws Exception {
assertEquals(testTacs.get(0).getTac(), codes.get(0).getTac());
}

// Test that we get a list of loa_sys_ids that occur more than once
// DatabaseService.getLoaSysIdCountGreaterThan1()
@Test
void testGetLoaSysIdCountGreaterThan1() throws Exception {

setUpTests();

LocalDateTime time = LocalDateTime.now();
int mo = time.getMonthValue();
int day = time.getDayOfMonth();
int year = time.getYear();
int hr = time.getHour();
int min = time.getMinute();
int sec = time.getSecond();

String dayTime = Integer.toString(mo) + Integer.toString(day) + Integer.toString(year) + Integer.toString(hr) + Integer.toString(hr) + Integer.toString(min) + Integer.toString(sec);

String testLoaSysId = "dum" + dayTime;
String nonDupLoaSysId = testLoaSysId + "a";

ArrayList<LineOfAccounting> testLoas = createMockLoas(3);

testLoas.get(0).setLoaSysID(testLoaSysId);
testLoas.get(1).setLoaSysID(testLoaSysId);
testLoas.get(2).setLoaSysID(nonDupLoaSysId);

// Invoke insertLineOfAccountingCodes() with test LOA(s)
spyDatabaseService.insertLinesOfAccounting(testLoas);

Connection conn1 = createTestDbConnection();

// Mock the DatabaseService.getConnection() to return the test_db connection
doReturn(conn1).when(spyDatabaseService).getConnection();

// Make sure the test_db connection is returned when .getConnection is called
assertEquals(conn1, spyDatabaseService.getConnection());

ArrayList<String> loaSysIds = spyDatabaseService.getLoaSysIdCountGreaterThan1();

assertTrue(loaSysIds.contains(testLoaSysId));
assertFalse(loaSysIds.contains(nonDupLoaSysId));
}

// Test that we can delete a list of loas
// DatabaseService.deleteLoas()
@Test
Expand Down
12 changes: 10 additions & 2 deletions src/test/java/com/milmove/trdmlambda/milmove/util/TrdmTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,11 +292,19 @@ void identifyDuplicateLoasToDeleteTest() throws SQLException {
currentLoas.get(5).setLoaSysID("NoDupe" + rand.nextInt(1000) + rand.nextInt(1000)); // No delete
currentLoas.get(6).setLoaSysID("NoDupe" + rand.nextInt(1000) + rand.nextInt(1000)); // No Delete

// List of duplicateLoas
ArrayList<String> duplicateLoaSysIds = new ArrayList<String>();
duplicateLoaSysIds.add(currentLoas.get(0).getLoaSysID());
duplicateLoaSysIds.add(currentLoas.get(1).getLoaSysID());
duplicateLoaSysIds.add(currentLoas.get(2).getLoaSysID());
duplicateLoaSysIds.add(currentLoas.get(3).getLoaSysID());
duplicateLoaSysIds.add(currentLoas.get(4).getLoaSysID());

ArrayList<LineOfAccounting> loasToDelete = trdm.identifyDuplicateLoasToDelete(currentLoas, currentTacs);

ArrayList<LineOfAccounting> loasToDelete = trdm.identifyDuplicateLoasToDelete(currentLoas, currentTacs, duplicateLoaSysIds);
List<UUID> loaIds = loasToDelete.stream().map(loa -> loa.getId()).collect(Collectors.toList());

// Loas that should be deleted should be returned for deletetion
// Loas that should be deleted should be returned for deletion
assertTrue(loaIds.contains(currentLoas.get(2).getId()));
assertTrue(loaIds.contains(currentLoas.get(3).getId()));

Expand Down

0 comments on commit 9f138a0

Please sign in to comment.