Skip to content

Commit

Permalink
feat(re-index): recreate upload ranges each upload execution (#726)
Browse files Browse the repository at this point in the history
Closes: MSEARCH-934

(cherry picked from commit e6f3f90)
  • Loading branch information
psmagin committed Jan 8, 2025
1 parent 3a993ef commit ff6daa4
Show file tree
Hide file tree
Showing 6 changed files with 21 additions and 21 deletions.
4 changes: 4 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## v4.0.6 2025-01-08
### Tech Dept
* Recreate upload ranges each upload execution ([MSEARCH-934](https://folio-org.atlassian.net/browse/MSEARCH-934))

## v4.0.5 2025-01-03
### Bug fixes
* Implement scheduled indexing for instance sub-resources ([MSEARCH-922](https://folio-org.atlassian.net/browse/MSEARCH-922))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public void prepareAndSendIndexRanges(ReindexEntityType entityType) {
var repository = Optional.ofNullable(repositories.get(entityType))
.orElseThrow(() -> new UnsupportedOperationException("No repository found for entity type: " + entityType));

var uploadRanges = repository.getUploadRanges(true);
var uploadRanges = repository.createUploadRanges();
statusService.updateReindexUploadStarted(entityType, uploadRanges.size());
indexRangeEventProducer.sendMessages(prepareEvents(uploadRanges));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ ON CONFLICT (id)
""";

private static final String SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL = "SELECT * FROM %s WHERE entity_type = ?;";
private static final String DELETE_UPLOAD_RANGE_SQL = "DELETE FROM %s WHERE entity_type = ?;";
private static final TypeReference<LinkedHashSet<InstanceSubResource>> VALUE_TYPE_REF = new TypeReference<>() { };

protected final ReindexConfigurationProperties reindexConfig;
Expand All @@ -51,14 +52,19 @@ protected UploadRangeRepository(JdbcTemplate jdbcTemplate,
this.reindexConfig = reindexConfig;
}

public List<UploadRangeEntity> getUploadRanges(boolean populateIfNotExist) {
public List<UploadRangeEntity> getUploadRanges() {
var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE);
var sql = SELECT_UPLOAD_RANGE_BY_ENTITY_TYPE_SQL.formatted(fullTableName);
var uploadRanges = jdbcTemplate.query(sql, uploadRangeRowMapper(), entityType().getType());

return populateIfNotExist && uploadRanges.isEmpty()
? prepareAndSaveUploadRanges()
: uploadRanges;
return jdbcTemplate.query(sql, uploadRangeRowMapper(), entityType().getType());
}

public List<UploadRangeEntity> createUploadRanges() {
var fullTableName = getFullTableName(context, UPLOAD_RANGE_TABLE);
var deleteSql = DELETE_UPLOAD_RANGE_SQL.formatted(fullTableName);
jdbcTemplate.update(deleteSql, entityType().getType());

return prepareAndSaveUploadRanges();
}

public List<Map<String, Object>> fetchByIdRange(String lower, String upper) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ void setUp() {
@Test
void prepareAndSendIndexRanges_positive(@Random UploadRangeEntity uploadRange) {
// arrange
when(repository.getUploadRanges(true)).thenReturn(List.of(uploadRange));
when(repository.createUploadRanges()).thenReturn(List.of(uploadRange));

// act
service.prepareAndSendIndexRanges(ReindexEntityType.INSTANCE);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ void setIndexRangeFinishDate() {
var mergeRange = mergeRepository.getMergeRanges().stream()
.filter(range -> range.getEntityType().equals(ReindexEntityType.INSTANCE))
.findFirst();
var uploadRange = uploadRepository.getUploadRanges(false).stream()
var uploadRange = uploadRepository.getUploadRanges().stream()
.filter(range -> range.getEntityType().equals(ReindexEntityType.INSTANCE))
.findFirst();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,22 +71,12 @@ public String getDBSchemaName(String tenantId) {
}

@Test
void getUploadRanges_returnEmptyList_whenNoUploadRangesAndNotPopulate() {
// act
var ranges = repository.getUploadRanges(false);

// assert
assertThat(ranges).isEmpty();
}

@Test
@Sql("/sql/populate-subjects.sql")
void getUploadRanges_returnList_whenNoUploadRangesAndNotPopulate() {
void getUploadRanges_returnList() {
// arrange
properties.setUploadRangeLevel(1);

// act
var ranges = repository.getUploadRanges(true);
var ranges = repository.createUploadRanges();

// assert
assertThat(ranges)
Expand Down Expand Up @@ -139,7 +129,7 @@ void deleteByInstanceIds_oneSubjectRemovedAndOneInstanceCounterDecremented() {
.contains(
tuple("Sci-Fi", List.of(
Map.of("count", 1, "shared", true, "tenantId", "consortium"),
Map.of("count", 1, "shared", false, "tenantId", "member_tenant"))));
Map.of("count", 1, "shared", false, "tenantId", "member_tenant"))));
}

@Test
Expand Down

0 comments on commit ff6daa4

Please sign in to comment.