Merge pull request #54 from FAIRDataTeam/release/1.5.0

Release 1.5.0

CHANGELOG.md

@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.5.0]
+
+### Added
+
+- Possibility to specify custom storage in frontend
+- History of created metadata in new dialog and stored using overlay model
+  per project
+
+### Changed
+
+- Saving project when overlay model updated (history)
+- Localizable audit table headers
+- Adjusted for compatibility with FAIR Data Point 1.5.0
+
+### Fixed
+
+- Virtuoso storage can use HTTP Basic Auth
+
 ## [1.4.0]
 
 ### Changed
@@ -86,9 +104,10 @@ Initial version based on reproducing functionality of deprecated [FAIRifier](htt
 - About dialog with basic information about the extension and its compatibility
 - Report a bug link to create a GitHub issue easily
 
-[Unreleased]: /../../compare/v1.4.0...develop
+[Unreleased]: /../../compare/v1.5.0...develop
 [1.0.0]: /../../tree/v1.0.0
 [1.1.0]: /../../tree/v1.1.0
 [1.2.0]: /../../tree/v1.2.0
 [1.3.0]: /../../tree/v1.3.0
 [1.4.0]: /../../tree/v1.4.0
+[1.5.0]: /../../tree/v1.5.0

pom.xml

@@ -28,7 +28,7 @@
 
     <groupId>solutions.fairdata.openrefine</groupId>
     <artifactId>metadata</artifactId>
-    <version>1.4.0</version>
+    <version>1.5.0</version>
     <packaging>jar</packaging>
 
     <name>OpenRefine - FAIR Metadata extension</name>

src/main/java/solutions/fairdata/openrefine/metadata/dto/dashboard/MembershipEntity.java → src/main/java/solutions/fairdata/openrefine/metadata/GenericUtils.java

@@ -20,8 +20,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-package solutions.fairdata.openrefine.metadata.dto.dashboard;
+package solutions.fairdata.openrefine.metadata;
 
-public enum MembershipEntity {
-    CATALOG, DATASET, DISTRIBUTION
+import java.time.OffsetDateTime;
+
+public class GenericUtils {
+
+    public static String currentTimestamp() {
+        return OffsetDateTime.now().toString();
+    }
 }

src/main/java/solutions/fairdata/openrefine/metadata/MetadataModuleImpl.java

@@ -25,12 +25,13 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.refine.io.ProjectUtilities;
 import com.google.refine.model.Project;
 import edu.mit.simile.butterfly.ButterflyModuleImpl;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import solutions.fairdata.openrefine.metadata.dto.ProjectInfoDTO;
-import solutions.fairdata.openrefine.metadata.dto.config.ProjectConfigDTO;
+import solutions.fairdata.openrefine.metadata.dto.project.ProjectHistoryDTO;
 import solutions.fairdata.openrefine.metadata.dto.config.SettingsConfigDTO;
 import solutions.fairdata.openrefine.metadata.dto.storage.StorageDTO;
 import solutions.fairdata.openrefine.metadata.model.MetadataOverlayModel;
@@ -150,23 +151,18 @@ public static MetadataOverlayModel getProjectModel(Project project) {
         MetadataOverlayModel metadataOverlayModel = (MetadataOverlayModel) project.overlayModels.get(OVERLAY_MODEL);
         if (metadataOverlayModel == null) {
             metadataOverlayModel = new MetadataOverlayModel();
-            metadataOverlayModel.setProjectData(new ProjectConfigDTO());
+            metadataOverlayModel.setProjectData(new ProjectHistoryDTO());
             project.overlayModels.put(OVERLAY_MODEL, metadataOverlayModel);
         }
         return metadataOverlayModel;
     }
 
-    public static ProjectConfigDTO getProjectConfigDTO(Project project) {
+    public static ProjectHistoryDTO getProjectHistoryDTO(Project project) {
         return getProjectModel(project).getProjectData();
     }
 
-    public static void setProjectConfigDTO(Project project, ProjectConfigDTO projectData) {
-        MetadataOverlayModel metadataOverlayModel = (MetadataOverlayModel) project.overlayModels.get(OVERLAY_MODEL);
-        if (metadataOverlayModel == null) {
-            metadataOverlayModel = new MetadataOverlayModel();
-            project.overlayModels.put(OVERLAY_MODEL, metadataOverlayModel);
-        }
-        metadataOverlayModel.setProjectData(projectData);
+    public static void forceSaveProject(Project project) throws IOException {
+        ProjectUtilities.save(project);
     }
 
     public static Logger getLogger() {

src/main/java/solutions/fairdata/openrefine/metadata/ProjectAudit.java

@@ -30,18 +30,13 @@
 
 import java.io.PrintWriter;
 import java.io.StringWriter;
-import java.time.OffsetDateTime;
 import java.util.ArrayList;
 
 public class ProjectAudit {
 
     private final AuditLogDTO log;
     private final int level;
 
-    private static String currentTimestamp() {
-        return OffsetDateTime.now().toString();
-    }
-
     private static String getStackTrace(Exception e) {
         StringWriter sw = new StringWriter();
         e.printStackTrace(new PrintWriter(sw));
@@ -62,7 +57,7 @@ public ProjectAudit(Project project) {
 
     public void report(EventType eventType, EventSource eventSource, String message) {
         if (eventType.getLevel() <= level && this.log != null) {
-            this.log.getEntries().add(new AuditEntryDTO(eventType, eventSource, message, currentTimestamp()));
+            this.log.getEntries().add(new AuditEntryDTO(eventType, eventSource, message, GenericUtils.currentTimestamp()));
         }
     }
 

src/main/java/solutions/fairdata/openrefine/metadata/commands/CatalogsMetadataCommand.java

@@ -23,6 +23,8 @@
 package solutions.fairdata.openrefine.metadata.commands;
 
 import com.google.refine.commands.Command;
+import com.google.refine.model.Project;
+import solutions.fairdata.openrefine.metadata.MetadataModuleImpl;
 import solutions.fairdata.openrefine.metadata.ProjectAudit;
 import solutions.fairdata.openrefine.metadata.commands.request.metadata.CatalogPostRequest;
 import solutions.fairdata.openrefine.metadata.commands.response.ErrorResponse;
@@ -31,6 +33,7 @@
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
 import solutions.fairdata.openrefine.metadata.dto.metadata.CatalogDTO;
 import solutions.fairdata.openrefine.metadata.dto.metadata.FDPMetadataDTO;
+import solutions.fairdata.openrefine.metadata.dto.project.ProjectMetadataRecordDTO;
 import solutions.fairdata.openrefine.metadata.fdp.FairDataPointClient;
 
 import javax.servlet.ServletException;
@@ -81,13 +84,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
     public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
         CatalogPostRequest catalogPostRequest = CommandUtils.objectMapper.readValue(request.getReader(), CatalogPostRequest.class);
         Writer w = CommandUtils.prepareWriter(response);
-        ProjectAudit pa = new ProjectAudit(getProject(request));
+        Project project = getProject(request);
+        ProjectAudit pa = new ProjectAudit(project);
         FairDataPointClient fdpClient = new FairDataPointClient(catalogPostRequest.getFdpUri(), catalogPostRequest.getToken(), pa);
 
         try {
             pa.reportInfo(EventSource.FDP_METADATA, "Creating catalog: " + catalogPostRequest.getFdpUri());
             CatalogDTO catalogDTO = fdpClient.postCatalog(catalogPostRequest.getCatalog());
-
+            MetadataModuleImpl.getProjectModel(project).addProjectMetadata(ProjectMetadataRecordDTO.createFor(catalogDTO));
+            MetadataModuleImpl.forceSaveProject(project);
             pa.reportDebug(EventSource.FDP_METADATA, "Catalog created: " + catalogDTO.getIri());
             CommandUtils.objectMapper.writeValue(w, new CatalogPostResponse(catalogDTO));
         } catch (Exception e) {

src/main/java/solutions/fairdata/openrefine/metadata/commands/DatasetsMetadataCommand.java

@@ -23,6 +23,8 @@
 package solutions.fairdata.openrefine.metadata.commands;
 
 import com.google.refine.commands.Command;
+import com.google.refine.model.Project;
+import solutions.fairdata.openrefine.metadata.MetadataModuleImpl;
 import solutions.fairdata.openrefine.metadata.ProjectAudit;
 import solutions.fairdata.openrefine.metadata.commands.request.metadata.DatasetPostRequest;
 import solutions.fairdata.openrefine.metadata.commands.response.ErrorResponse;
@@ -31,6 +33,7 @@
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
 import solutions.fairdata.openrefine.metadata.dto.metadata.CatalogDTO;
 import solutions.fairdata.openrefine.metadata.dto.metadata.DatasetDTO;
+import solutions.fairdata.openrefine.metadata.dto.project.ProjectMetadataRecordDTO;
 import solutions.fairdata.openrefine.metadata.fdp.FairDataPointClient;
 
 import javax.servlet.ServletException;
@@ -80,13 +83,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
     public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
         DatasetPostRequest datasetPostRequest = CommandUtils.objectMapper.readValue(request.getReader(), DatasetPostRequest.class);
         Writer w = CommandUtils.prepareWriter(response);
-        ProjectAudit pa = new ProjectAudit(getProject(request));
+        Project project = getProject(request);
+        ProjectAudit pa = new ProjectAudit(project);
         FairDataPointClient fdpClient = new FairDataPointClient(datasetPostRequest.getFdpUri(), datasetPostRequest.getToken(), pa);
 
         try {
             pa.reportInfo(EventSource.FDP_METADATA, "Creating dataset in catalog: " + datasetPostRequest.getDataset().getParent());
             DatasetDTO datasetDTO = fdpClient.postDataset(datasetPostRequest.getDataset());
-
+            MetadataModuleImpl.getProjectModel(project).addProjectMetadata(ProjectMetadataRecordDTO.createFor(datasetDTO));
+            MetadataModuleImpl.forceSaveProject(project);
             pa.reportDebug(EventSource.FDP_METADATA, "Dataset created: " + datasetDTO.getIri());
             CommandUtils.objectMapper.writeValue(w, new DatasetPostResponse(datasetDTO));
         } catch (Exception e) {

src/main/java/solutions/fairdata/openrefine/metadata/commands/DistributionsMetadataCommand.java

@@ -23,6 +23,8 @@
 package solutions.fairdata.openrefine.metadata.commands;
 
 import com.google.refine.commands.Command;
+import com.google.refine.model.Project;
+import solutions.fairdata.openrefine.metadata.MetadataModuleImpl;
 import solutions.fairdata.openrefine.metadata.ProjectAudit;
 import solutions.fairdata.openrefine.metadata.commands.request.metadata.DistributionPostRequest;
 import solutions.fairdata.openrefine.metadata.commands.response.ErrorResponse;
@@ -31,6 +33,7 @@
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
 import solutions.fairdata.openrefine.metadata.dto.metadata.DatasetDTO;
 import solutions.fairdata.openrefine.metadata.dto.metadata.DistributionDTO;
+import solutions.fairdata.openrefine.metadata.dto.project.ProjectMetadataRecordDTO;
 import solutions.fairdata.openrefine.metadata.fdp.FairDataPointClient;
 
 import javax.servlet.ServletException;
@@ -80,13 +83,15 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
     public void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
         DistributionPostRequest distributionPostRequest = CommandUtils.objectMapper.readValue(request.getReader(), DistributionPostRequest.class);
         Writer w = CommandUtils.prepareWriter(response);
-        ProjectAudit pa = new ProjectAudit(getProject(request));
+        Project project = getProject(request);
+        ProjectAudit pa = new ProjectAudit(project);
         FairDataPointClient fdpClient = new FairDataPointClient(distributionPostRequest.getFdpUri(), distributionPostRequest.getToken(), pa);
 
         try {
             pa.reportDebug(EventSource.FDP_METADATA, "Creating distribution in dataset: " + distributionPostRequest.getDistribution().getParent());
             DistributionDTO distributionDTO = fdpClient.postDistribution(distributionPostRequest.getDistribution());
-
+            MetadataModuleImpl.getProjectModel(project).addProjectMetadata(ProjectMetadataRecordDTO.createFor(distributionDTO));
+            MetadataModuleImpl.forceSaveProject(project);
             pa.reportDebug(EventSource.FDP_METADATA, "Distribution created: " + distributionDTO.getIri());
             CommandUtils.objectMapper.writeValue(w, new DistributionPostResponse(distributionDTO));
         } catch (Exception e) {

src/main/java/solutions/fairdata/openrefine/metadata/commands/ServiceCommand.java

@@ -29,7 +29,6 @@
 import solutions.fairdata.openrefine.metadata.commands.response.ServiceResponse;
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
 
-import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;

src/main/java/solutions/fairdata/openrefine/metadata/commands/SettingsCommand.java

@@ -32,7 +32,7 @@
 import solutions.fairdata.openrefine.metadata.commands.response.config.SettingsResponse;
 import solutions.fairdata.openrefine.metadata.dto.ProjectInfoDTO;
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
-import solutions.fairdata.openrefine.metadata.dto.config.ProjectConfigDTO;
+import solutions.fairdata.openrefine.metadata.dto.project.ProjectHistoryDTO;
 import solutions.fairdata.openrefine.metadata.dto.config.SettingsConfigDTO;
 
 import javax.servlet.ServletException;
@@ -48,7 +48,7 @@ public class SettingsCommand extends Command {
 
     private SettingsResponse createSettingsResponse(Project project) {
         SettingsConfigDTO settings = MetadataModuleImpl.getInstance().getSettingsDetails();
-        ProjectConfigDTO projectData = MetadataModuleImpl.getProjectConfigDTO(project);
+        ProjectHistoryDTO projectData = MetadataModuleImpl.getProjectHistoryDTO(project);
         ProjectInfoDTO projectInfo = MetadataModuleImpl.getInstance().getProjectInfo();
         return new SettingsResponse(settings, projectData, projectInfo);
     }
@@ -80,8 +80,35 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
         ProjectAudit pa = new ProjectAudit(getProject(request));
 
         try {
-            pa.reportDebug(EventSource.SETTINGS,"Persisting project configuration");
-            MetadataModuleImpl.setProjectConfigDTO(project, settingsRequest.getProjectData());
+            if (settingsRequest.getType().equalsIgnoreCase("catalog")) {
+                pa.reportDebug(EventSource.SETTINGS,"Persisting last catalog for repository");
+                MetadataModuleImpl.getProjectHistoryDTO(project).getLastCatalog().put(settingsRequest.getRepositoryUri(), settingsRequest.getLastUri());
+                MetadataModuleImpl.forceSaveProject(project);
+            } else if (settingsRequest.getType().equalsIgnoreCase("dataset")) {
+                MetadataModuleImpl.getProjectHistoryDTO(project).getLastDataset().put(settingsRequest.getRepositoryUri(), settingsRequest.getLastUri());
+                MetadataModuleImpl.forceSaveProject(project);
+            }
+            CommandUtils.objectMapper.writeValue(w, createSettingsResponse(project));
+        } catch (Exception e) {
+            pa.reportError(EventSource.SETTINGS,"Error occurred while persisting Settings");
+            pa.reportTrace(EventSource.SETTINGS, e);
+            CommandUtils.objectMapper.writeValue(w, new ErrorResponse("auth-fdp-command/error", e));
+        } finally {
+            w.flush();
+            w.close();
+        }
+    }
+
+    @Override
+    public void doDelete(HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+        Project project = getProject(request);
+        Writer w = CommandUtils.prepareWriter(response);
+        ProjectAudit pa = new ProjectAudit(getProject(request));
+
+        try {
+            pa.reportInfo(EventSource.SETTINGS,"Clearing project history");
+            MetadataModuleImpl.getProjectModel(project).setProjectData(new ProjectHistoryDTO());
+            MetadataModuleImpl.forceSaveProject(project);
             CommandUtils.objectMapper.writeValue(w, createSettingsResponse(project));
         } catch (Exception e) {
             pa.reportError(EventSource.SETTINGS,"Error occurred while persisting Settings");

src/main/java/solutions/fairdata/openrefine/metadata/commands/StoreDataCommand.java

@@ -38,8 +38,11 @@
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
 import solutions.fairdata.openrefine.metadata.dto.storage.ExportFormatDTO;
 import solutions.fairdata.openrefine.metadata.dto.storage.StorageDTO;
+import solutions.fairdata.openrefine.metadata.storage.FTPStorage;
 import solutions.fairdata.openrefine.metadata.storage.Storage;
 import solutions.fairdata.openrefine.metadata.storage.StorageRegistryUtil;
+import solutions.fairdata.openrefine.metadata.storage.TripleStoreHTTPStorage;
+import solutions.fairdata.openrefine.metadata.storage.VirtuosoStorage;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
@@ -108,6 +111,9 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) thro
         );
         storeDataInfoResponse.getDefaults().put("filename", defaultFilename);
         storeDataInfoResponse.getDefaults().put("baseURI", defaultBaseURI);
+        storeDataInfoResponse.getStorageTypes().put(FTPStorage.TYPE, FTPStorage.DETAILS);
+        storeDataInfoResponse.getStorageTypes().put(VirtuosoStorage.TYPE, VirtuosoStorage.DETAILS);
+        storeDataInfoResponse.getStorageTypes().put(TripleStoreHTTPStorage.TYPE, TripleStoreHTTPStorage.DETAILS);
         CommandUtils.objectMapper.writeValue(w, storeDataInfoResponse);
 
         w.flush();
@@ -128,7 +134,8 @@ public void doPost(HttpServletRequest request, HttpServletResponse response) thr
 
             ExportFormatDTO format = formats.get(storeDataRequest.getFormat());
             Exporter exporter = ExporterRegistry.getExporter(storeDataRequest.getFormat());
-            Storage storage = StorageRegistryUtil.getStorage(storeDataRequest.getStorage());
+            Storage storage = StorageRegistryUtil.getStorage(storeDataRequest.getStorage(), storeDataRequest.getCustom());
+
             if (exporter == null) {
                 pa.reportWarning(EventSource.STORAGE,"Unknown export format requested");
                 throw new MetadataCommandException("store-data-dialog/error/unknown-export-format");
@@ -178,7 +185,7 @@ else if (storage.forbidsContentType(exporter.getContentType())) {
                         pa.reportWarning(EventSource.STORAGE,"Filename violates storage requirements");
                         throw new MetadataCommandException("store-data-dialog/error/naming-violation");
                     }
-                    else if (storage.fordbidsByteSize(data.length)) {
+                    else if (storage.forbidsByteSize(data.length)) {
                         pa.reportWarning(EventSource.STORAGE,"File is too big for selected storage");
                         throw new MetadataCommandException("store-data-dialog/error/too-big");
                     }

src/main/java/solutions/fairdata/openrefine/metadata/commands/TypehintsCommand.java

@@ -28,7 +28,11 @@
 import solutions.fairdata.openrefine.metadata.commands.response.TypehintsResponse;
 import solutions.fairdata.openrefine.metadata.dto.TypehintDTO;
 import solutions.fairdata.openrefine.metadata.dto.audit.EventSource;
-import solutions.fairdata.openrefine.metadata.typehinting.*;
+import solutions.fairdata.openrefine.metadata.typehinting.LanguageTypehintService;
+import solutions.fairdata.openrefine.metadata.typehinting.LicenseTypehintService;
+import solutions.fairdata.openrefine.metadata.typehinting.MediaTypeTypehintService;
+import solutions.fairdata.openrefine.metadata.typehinting.ThemeTypehintService;
+import solutions.fairdata.openrefine.metadata.typehinting.TypehintService;
 
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;

src/main/java/solutions/fairdata/openrefine/metadata/commands/request/config/SettingsRequest.java

@@ -26,12 +26,13 @@
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
-import solutions.fairdata.openrefine.metadata.dto.config.ProjectConfigDTO;
 
 @NoArgsConstructor
 @AllArgsConstructor
 @Getter
 @Setter
 public class SettingsRequest {
-    private ProjectConfigDTO projectData;
+    private String type;
+    private String repositoryUri;
+    private String lastUri;
 }

src/main/java/solutions/fairdata/openrefine/metadata/commands/request/storage/StoreDataRequest.java

@@ -22,12 +22,13 @@
  */
 package solutions.fairdata.openrefine.metadata.commands.request.storage;
 
-import java.util.*;
-
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import solutions.fairdata.openrefine.metadata.dto.storage.CustomStorageDTO;
+
+import java.util.*;
 
 @NoArgsConstructor
 @AllArgsConstructor
@@ -41,5 +42,6 @@ public class StoreDataRequest {
     private String mode;
     private String format;
     private String storage;
+    private CustomStorageDTO custom;
     private HashMap<String, String> metadata;
 }