From 15e9b5007c2e46194c29caa1c888cfc3d0989277 Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Thu, 31 Oct 2024 08:53:12 +0100 Subject: [PATCH 1/3] use reference id for exporting data --- .../uio/ifi/localega/doa/model/Dataset.java | 28 ++++++++++++++++ .../localega/doa/model/DatasetReferences.java | 32 +++++++++++++++++++ .../doa/mq/ExportRequestsListener.java | 10 ++++-- .../DatasetReferencesRepository.java | 10 ++++++ .../doa/repositories/DatasetsRepository.java | 9 ++++++ .../doa/services/MetadataService.java | 21 ++++++++++-- 6 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 src/main/java/no/uio/ifi/localega/doa/model/Dataset.java create mode 100644 src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java create mode 100644 src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java create mode 100644 src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java diff --git a/src/main/java/no/uio/ifi/localega/doa/model/Dataset.java b/src/main/java/no/uio/ifi/localega/doa/model/Dataset.java new file mode 100644 index 0000000..8ec53f8 --- /dev/null +++ b/src/main/java/no/uio/ifi/localega/doa/model/Dataset.java @@ -0,0 +1,28 @@ +package no.uio.ifi.localega.doa.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.hibernate.annotations.CacheConcurrencyStrategy; +import org.hibernate.annotations.Immutable; + +@org.hibernate.annotations.Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL) +@Entity +@Immutable +@Getter +@Setter +@ToString +@RequiredArgsConstructor +@Table(name = "datasets", schema = "sda") +public class Dataset { + @Id + private Long id; + + @Column(name = "stable_id", unique = true) + private String stableId; +} diff --git a/src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java b/src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java new file mode 100644 index 0000000..b04c083 --- /dev/null +++ b/src/main/java/no/uio/ifi/localega/doa/model/DatasetReferences.java @@ -0,0 +1,32 @@ +package no.uio.ifi.localega.doa.model; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.*; +import org.hibernate.annotations.CacheConcurrencyStrategy; +import org.hibernate.annotations.Immutable; +import org.hibernate.proxy.HibernateProxy; + +import java.time.LocalDateTime; +import java.util.Objects; + +@org.hibernate.annotations.Cache(usage = CacheConcurrencyStrategy.TRANSACTIONAL) +@Entity +@Immutable +@Getter +@Setter +@ToString +@RequiredArgsConstructor +@Table(name = "dataset_references", schema = "sda") +public class DatasetReferences { + @Id + private Integer id; + + @Column(name = "dataset_id", nullable = false) + private Integer datasetId; + + @Column(name = "reference_id", nullable = false) + private String referenceId; +} diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index bd1706c..5c1f235 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -74,8 +74,14 @@ public void listen(String message) { String user = claims.get(Claims.SUBJECT).getAsString(); log.info("Export request received from user {}: {}", user, exportRequest); Collection datasetIds = aaiService.getDatasetIds(exportRequest.getJwtToken()); - if (StringUtils.isNotEmpty(exportRequest.getDatasetId())) { - exportDataset(user, datasetIds, exportRequest.getDatasetId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); + String datasetId = exportRequest.getDatasetId(); + if (StringUtils.isNotEmpty(datasetId)) { + if (metadataService.findByReferenceId(datasetId) != null) { + Integer id = metadataService.findByReferenceId(datasetId).getDatasetId(); + datasetId = metadataService.getDataset(id).getStableId(); + log.info("Reference id {} mapped to dataset id {}", exportRequest.getDatasetId(), datasetId); + } + exportDataset(user, datasetIds, datasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); } else if (StringUtils.isNotEmpty(exportRequest.getFileId())) { exportFile(user, datasetIds, exportRequest.getFileId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); } else { diff --git a/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java b/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java new file mode 100644 index 0000000..822fc81 --- /dev/null +++ b/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetReferencesRepository.java @@ -0,0 +1,10 @@ +package no.uio.ifi.localega.doa.repositories; + +import no.uio.ifi.localega.doa.model.DatasetReferences; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface DatasetReferencesRepository extends JpaRepository { + DatasetReferences findByReferenceId(String referenceId); +} diff --git a/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java b/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java new file mode 100644 index 0000000..9b08cce --- /dev/null +++ b/src/main/java/no/uio/ifi/localega/doa/repositories/DatasetsRepository.java @@ -0,0 +1,9 @@ +package no.uio.ifi.localega.doa.repositories; + +import no.uio.ifi.localega.doa.model.Dataset; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +@Repository +public interface DatasetsRepository extends JpaRepository { +} diff --git a/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java b/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java index 05f286a..20302c7 100644 --- a/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java +++ b/src/main/java/no/uio/ifi/localega/doa/services/MetadataService.java @@ -2,11 +2,11 @@ import lombok.extern.slf4j.Slf4j; import no.uio.ifi.localega.doa.dto.File; +import no.uio.ifi.localega.doa.model.Dataset; import no.uio.ifi.localega.doa.model.DatasetEventLog; +import no.uio.ifi.localega.doa.model.DatasetReferences; import no.uio.ifi.localega.doa.model.LEGADataset; -import no.uio.ifi.localega.doa.repositories.DatasetEventLogRepository; -import no.uio.ifi.localega.doa.repositories.DatasetRepository; -import no.uio.ifi.localega.doa.repositories.FileRepository; +import no.uio.ifi.localega.doa.repositories.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -32,6 +32,12 @@ public class MetadataService { @Autowired private DatasetEventLogRepository datasetEventLogRepository; + @Autowired + private DatasetReferencesRepository datasetReferencesRepository; + + @Autowired + private DatasetsRepository datasetsRepository; + /** * Returns collection of dataset IDs present in the databse. * @@ -88,4 +94,13 @@ public DatasetEventLog findLatestByDatasetId(String datasetId) { return optionalDatasetEventLog.orElse(null); } + public DatasetReferences findByReferenceId(String referenceId) { + Optional optionalDatasetReferences = Optional.ofNullable(datasetReferencesRepository.findByReferenceId(referenceId)); + return optionalDatasetReferences.orElse(null); + } + + public Dataset getDataset(Integer id) { + return datasetsRepository.findById(id).orElse(null); + } + } From 7790c03e9d0f14bff84b131c0c11c2c0edf6f4be Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Mon, 18 Nov 2024 09:43:57 +0100 Subject: [PATCH 2/3] rename vars ro increase readability --- .../doa/mq/ExportRequestsListener.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index 5c1f235..cf5320f 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -73,17 +73,18 @@ public void listen(String message) { String user = claims.get(Claims.SUBJECT).getAsString(); log.info("Export request received from user {}: {}", user, exportRequest); - Collection datasetIds = aaiService.getDatasetIds(exportRequest.getJwtToken()); - String datasetId = exportRequest.getDatasetId(); - if (StringUtils.isNotEmpty(datasetId)) { - if (metadataService.findByReferenceId(datasetId) != null) { - Integer id = metadataService.findByReferenceId(datasetId).getDatasetId(); - datasetId = metadataService.getDataset(id).getStableId(); - log.info("Reference id {} mapped to dataset id {}", exportRequest.getDatasetId(), datasetId); + Collection approvedDatasetIds = aaiService.getDatasetIds(exportRequest.getJwtToken()); + String requestedDatasetId = exportRequest.getDatasetId(); + if (StringUtils.isNotEmpty(requestedDatasetId)) { + if (metadataService.findByReferenceId(requestedDatasetId) != null) { + Integer datasetsDbTableId = metadataService.findByReferenceId(requestedDatasetId).getDatasetId(); + String stableDatasetId = metadataService.getDataset(datasetsDbTableId).getStableId(); + log.info("Reference id {} mapped to dataset id {}", requestedDatasetId, stableDatasetId); + requestedDatasetId = stableDatasetId; // use stable dataset id instead of reference to complete the export as normal } - exportDataset(user, datasetIds, datasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); + exportDataset(user, approvedDatasetIds, requestedDatasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); } else if (StringUtils.isNotEmpty(exportRequest.getFileId())) { - exportFile(user, datasetIds, exportRequest.getFileId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); + exportFile(user, approvedDatasetIds, exportRequest.getFileId(), exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); } else { throw new RuntimeException("Either Dataset ID or File ID should be specified"); } From 379bfaff310cfc6eebf8da4de9c7ac4db8917f31 Mon Sep 17 00:00:00 2001 From: Parisa Tejari Date: Wed, 20 Nov 2024 13:34:34 +0100 Subject: [PATCH 3/3] map the datasets with reference to datasets stable id --- .../java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java index cf5320f..4c1ae81 100644 --- a/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java +++ b/src/main/java/no/uio/ifi/localega/doa/mq/ExportRequestsListener.java @@ -81,6 +81,8 @@ public void listen(String message) { String stableDatasetId = metadataService.getDataset(datasetsDbTableId).getStableId(); log.info("Reference id {} mapped to dataset id {}", requestedDatasetId, stableDatasetId); requestedDatasetId = stableDatasetId; // use stable dataset id instead of reference to complete the export as normal + Collection approvedMappedDatasetIds = approvedDatasetIds.stream().map(x -> metadataService.getDataset(metadataService.findByReferenceId(x).getDatasetId()).getStableId()).toList(); + approvedDatasetIds = approvedMappedDatasetIds; } exportDataset(user, approvedDatasetIds, requestedDatasetId, exportRequest.getPublicKey(), exportRequest.getStartCoordinate(), exportRequest.getEndCoordinate()); } else if (StringUtils.isNotEmpty(exportRequest.getFileId())) {