Datastore - improvements to allow customise the metadata data folder …

…names - allow to use different base folders for public / non-public metadata

core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStore.java

@@ -73,7 +73,7 @@ public class FilesystemStore extends AbstractStore {
     SettingManager settingManager;
 
     @Autowired
-    FilesystemStoreConfig filesystemStoreConfig;
+    StoreFolderConfig storeFolderConfig;
 
     @Override
     public List<MetadataResource> getResources(ServiceContext context, String metadataUuid, MetadataResourceVisibility visibility,
@@ -240,7 +240,7 @@ private Path getPath(ServiceContext context, int metadataId, MetadataResourceVis
     public void copyResources(ServiceContext context, String sourceUuid, String targetUuid,
                               MetadataResourceVisibility metadataResourceVisibility,
                               boolean sourceApproved, boolean targetApproved) throws Exception {
-       if (filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.NONE) &&
+       if (storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.NONE) &&
            metadataResourceVisibility.equals(MetadataResourceVisibility.PRIVATE)) {
            return;
        }
@@ -276,7 +276,7 @@ public String delResource(ServiceContext context, String metadataUuid, String re
     public String delResource(final ServiceContext context, final String metadataUuid, final MetadataResourceVisibility visibility,
                               final String resourceId, Boolean approved) throws Exception {
 
-        if (filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.NONE) &&
+        if (storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.NONE) &&
             visibility.equals(MetadataResourceVisibility.PRIVATE)) {
             return null;
         }
@@ -295,7 +295,7 @@ public String delResource(final ServiceContext context, final String metadataUui
     public MetadataResource patchResourceStatus(ServiceContext context, String metadataUuid, String resourceId,
                                                 MetadataResourceVisibility visibility, Boolean approved) throws Exception {
 
-        if (filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.NONE) &&
+        if (storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.NONE) &&
             visibility.equals(MetadataResourceVisibility.PRIVATE)) {
             return null;
         }
@@ -321,7 +321,7 @@ public List<MetadataResource> getResources(ServiceContext context, String metada
 
         List<MetadataResource> resourceList = new ArrayList<>(
             getResources(context, metadataUuid, MetadataResourceVisibility.PUBLIC, filter, approved));
-        if (canEdit && filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.DEFAULT)) {
+        if (canEdit && storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.DEFAULT)) {
             resourceList.addAll(getResources(context, metadataUuid, MetadataResourceVisibility.PRIVATE, filter, approved));
         }
 
@@ -356,7 +356,7 @@ private Path ensureDirectory(final ServiceContext context, final int metadataId,
     }
 
     private Path calculateMetadataDir(final Path metadataDir, final MetadataResourceVisibility visibility) {
-        if (filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.DEFAULT)) {
+        if (storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.DEFAULT)) {
             return metadataDir.resolve(visibility.toString());
         } else {
             return metadataDir;
@@ -368,7 +368,7 @@ private GeonetworkDataDirectory getDataDirectory(ServiceContext context) {
 
     private MetadataResourceVisibility calculateVisibilityToUse(MetadataResourceVisibility visibility) {
         // If the folder privileges strategy is NONE, use the PUBLIC visibility
-        if (filesystemStoreConfig.getFolderPrivilegesStrategy().equals(FilesystemStoreConfig.FolderPrivilegesStrategy.NONE) &&
+        if (storeFolderConfig.getFolderPrivilegesStrategy().equals(StoreFolderConfig.FolderPrivilegesStrategy.NONE) &&
             visibility.equals(MetadataResourceVisibility.PRIVATE)) {
             return MetadataResourceVisibility.PUBLIC;
         } else {

core/src/main/java/org/fao/geonet/api/records/attachments/FilesystemStoreConfig.java → core/src/main/java/org/fao/geonet/api/records/attachments/StoreFolderConfig.java

@@ -27,7 +27,7 @@
 
 import org.springframework.beans.factory.annotation.Value;
 
-public class FilesystemStoreConfig {
+public class StoreFolderConfig {
     public enum FolderStructureType {
         DEFAULT, CUSTOM
     }
@@ -48,13 +48,13 @@ public enum FolderPrivilegesStrategy {
     @Value("${datastore.folderStructureFallback}")
     private String folderStructureFallback;
 
-    @Value("${datastore.folderStructureDraft}")
-    private String folderStructureDraft;
+    @Value("${datastore.folderStructureNonPublic}")
+    private String folderStructureNonPublic;
 
-    @Value("${datastore.folderStructureFallbackDraft}")
-    private String folderStructureFallbackDraft;
+    @Value("${datastore.folderStructureFallbackNonPublic}")
+    private String folderStructureFallbackNonPublic;
 
-    FilesystemStoreConfig() {
+    StoreFolderConfig() {
 
     }
 
@@ -74,11 +74,11 @@ public String getFolderStructureFallback() {
         return folderStructureFallback;
     }
 
-    public String getFolderStructureDraft() {
-        return folderStructureDraft;
+    public String getFolderStructureNonPublic() {
+        return folderStructureNonPublic;
     }
 
-    public String getFolderStructureFallbackDraft() {
-        return folderStructureFallbackDraft;
+    public String getFolderStructureFallbackNonPublic() {
+        return folderStructureFallbackNonPublic;
     }
 }

core/src/main/java/org/fao/geonet/kernel/datamanager/base/BaseMetadataManager.java

@@ -34,7 +34,6 @@
 import jeeves.xlink.Processor;
 import org.apache.commons.lang.StringUtils;
 import org.fao.geonet.ApplicationContextHolder;
-import org.fao.geonet.api.records.attachments.FilesystemStoreConfig;
 import org.fao.geonet.api.records.attachments.Store;
 import org.fao.geonet.constants.Edit;
 import org.fao.geonet.constants.Geonet;

core/src/main/java/org/fao/geonet/lib/ResourceLib.java

@@ -32,7 +32,7 @@
 import org.fao.geonet.ApplicationContextHolder;
 import org.fao.geonet.GeonetContext;
 import org.fao.geonet.api.exception.ResourceNotFoundException;
-import org.fao.geonet.api.records.attachments.FilesystemStoreConfig;
+import org.fao.geonet.api.records.attachments.StoreFolderConfig;
 import org.fao.geonet.constants.Geonet;
 import org.fao.geonet.constants.Params;
 import org.fao.geonet.domain.Operation;
@@ -67,9 +67,9 @@ public class ResourceLib {
      */
     public Path getDir(String access, int id) throws IOException {
         Path mdDir = getMetadataDir(ApplicationContextHolder.get().getBean(GeonetworkDataDirectory.class), id);
-        FilesystemStoreConfig filesystemStoreConfig = ApplicationContextHolder.get().getBean(FilesystemStoreConfig.class);
+        StoreFolderConfig storeFolderConfig = ApplicationContextHolder.get().getBean(StoreFolderConfig.class);
 
-        if (filesystemStoreConfig.getFolderPrivilegesStrategy() == FilesystemStoreConfig.FolderPrivilegesStrategy.DEFAULT) {
+        if (storeFolderConfig.getFolderPrivilegesStrategy() == StoreFolderConfig.FolderPrivilegesStrategy.DEFAULT) {
             String subDir = (access != null && access.equals(Params.Access.PUBLIC)) ? Params.Access.PUBLIC
                 : Params.Access.PRIVATE;
             return mdDir.resolve(subDir);
@@ -101,9 +101,9 @@ public Path getMetadataDir(GeonetworkDataDirectory dataDirectory, String id) thr
      * @return The metadata data directory
      */
     public Path getMetadataDir(Path dataDir, String id) throws IOException {
-        FilesystemStoreConfig filesystemStoreConfig = ApplicationContextHolder.get().getBean(FilesystemStoreConfig.class);
+        StoreFolderConfig storeFolderConfig = ApplicationContextHolder.get().getBean(StoreFolderConfig.class);
 
-        if (filesystemStoreConfig.getFolderStructureType().equals(FilesystemStoreConfig.FolderStructureType.DEFAULT)) {
+        if (storeFolderConfig.getFolderStructureType().equals(StoreFolderConfig.FolderStructureType.DEFAULT)) {
             IMetadataFolderProcessor metadataFolderProcessor = new DefaultMetadataFolderPathProcessor(dataDir, id);
             return metadataFolderProcessor.calculateFolderPath();
         } else {
@@ -184,14 +184,14 @@ private String pad(int group, int lenght) {
 
     private Path getCustomMetadataFolder(Path dataDir, String id) throws IOException {
         try {
-            FilesystemStoreConfig filesystemStoreConfig = ApplicationContextHolder.get().getBean(FilesystemStoreConfig.class);
+            StoreFolderConfig storeFolderConfig = ApplicationContextHolder.get().getBean(StoreFolderConfig.class);
 
             EsSearchManager esSearchManager = ApplicationContextHolder.get().getBean(EsSearchManager.class);
             SearchResponse searchResponse = esSearchManager.query(String.format("id:(%s)", id), null, 0, 10);
             if ((searchResponse.getHits().getTotalHits() != null) && (searchResponse.getHits().getTotalHits().value > 0)) {
                 SearchHit searchHit = searchResponse.getHits().getAt(0);
 
-                IMetadataFolderProcessor customMetadataFolderPathProcessor = new CustomMetadataFolderPathProcessor(dataDir, filesystemStoreConfig, searchHit);
+                IMetadataFolderProcessor customMetadataFolderPathProcessor = new CustomMetadataFolderPathProcessor(dataDir, storeFolderConfig, searchHit);
                 return customMetadataFolderPathProcessor.calculateFolderPath();
             } else {
                 throw new ResourceNotFoundException("Metadata not found");
@@ -242,38 +242,38 @@ public Path calculateFolderPath() {
      */
     private class CustomMetadataFolderPathProcessor implements IMetadataFolderProcessor {
         Path dataDir;
-        FilesystemStoreConfig filesystemStoreConfig;
+        StoreFolderConfig filesystemStoreConfig;
         SearchHit searchHit;
 
-        CustomMetadataFolderPathProcessor(Path dataDir, FilesystemStoreConfig filesystemStoreConfig, SearchHit searchHit) {
+        CustomMetadataFolderPathProcessor(Path dataDir, StoreFolderConfig storeFolderConfig, SearchHit searchHit) {
             this.dataDir = dataDir;
-            this.filesystemStoreConfig = filesystemStoreConfig;
+            this.filesystemStoreConfig = storeFolderConfig;
             this.searchHit = searchHit;
         }
 
         public Path calculateFolderPath() {
             DocumentContext jsonContext = JsonPath.parse(searchHit.getSourceAsString());
-            boolean isDraft = (searchHit.getSourceAsMap().get("draft") != null) &&
-                (!searchHit.getSourceAsMap().get("draft").equals("n"));
+            boolean isPublished = (searchHit.getSourceAsMap().get("isPublishedToAll") != null) &&
+                (searchHit.getSourceAsMap().get("isPublishedToAll").equals("true"));
 
             String path;
 
             try {
-                String folderStructure = !isDraft ?
-                    filesystemStoreConfig.getFolderStructure() : filesystemStoreConfig.getFolderStructureDraft();
-                path = replaceTokens(jsonContext, folderStructure.split(File.separator), isDraft);
+                String folderStructure = isPublished ?
+                    filesystemStoreConfig.getFolderStructure() : filesystemStoreConfig.getFolderStructureNonPublic();
+                path = replaceTokens(jsonContext, folderStructure.split(File.separator), isPublished);
 
             } catch (Exception ex) {
-                String folderStructure = !isDraft ?
-                    filesystemStoreConfig.getFolderStructureFallback() : filesystemStoreConfig.getFolderStructureFallbackDraft();
+                String folderStructure = isPublished ?
+                    filesystemStoreConfig.getFolderStructureFallback() : filesystemStoreConfig.getFolderStructureFallbackNonPublic();
 
-                path = replaceTokens(jsonContext, folderStructure.split(File.separator), isDraft);
+                path = replaceTokens(jsonContext, folderStructure.split(File.separator), isPublished);
             }
 
             return dataDir.resolve(path);
         }
 
-        private String replaceTokens(DocumentContext jsonContext, String[] tokens, boolean isWorkingCopy) {
+        private String replaceTokens(DocumentContext jsonContext, String[] tokens, boolean isPublished) {
             List<String> replacedTokens = new ArrayList<>();
             for (int i = 0; i < tokens.length; i++) {
                 String valueToAdd = "";
@@ -289,7 +289,7 @@ private String replaceTokens(DocumentContext jsonContext, String[] tokens, boole
                     valueToAdd = token;
                 }
 
-                if ((i == tokens.length - 1) && isWorkingCopy) {
+                if ((i == tokens.length - 1) && !isPublished) {
                     valueToAdd = valueToAdd + "-draft";
                 }
 

core/src/main/resources/config-store/config-default.xml

@@ -32,8 +32,8 @@
   <bean id="filesystemStore"
         class="org.fao.geonet.api.records.attachments.FilesystemStore"/>
 
-  <bean id="filesystemStoreConfig"
-        class="org.fao.geonet.api.records.attachments.FilesystemStoreConfig">
+  <bean id="storeFolderConfig"
+        class="org.fao.geonet.api.records.attachments.StoreFolderConfig">
   </bean>
 
   <bean id="resourceStore"

core/src/test/java/org/fao/geonet/api/records/attachments/FilesystemStoreTest.java

@@ -36,9 +36,7 @@
 import org.junit.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 
-import java.io.File;
 import java.io.InputStream;
-import java.nio.file.Files;
 import java.util.Date;
 
 import static org.junit.Assert.*;
@@ -50,7 +48,7 @@ public class FilesystemStoreTest extends AbstractCoreIntegrationTest {
     @Autowired
     protected SettingManager settingManager;
     @Autowired
-    protected FilesystemStoreConfig filesystemStoreConfig;
+    protected StoreFolderConfig storeFolderConfig;
 
     @Test
     public void getResourceDescription() throws Exception {
@@ -74,7 +72,7 @@ public void getResourceDescription() throws Exception {
             IndexingMode.none);
 
         FilesystemStore filesystemStore = new FilesystemStore();
-        filesystemStore.filesystemStoreConfig = filesystemStoreConfig;
+        filesystemStore.storeFolderConfig = storeFolderConfig;
         filesystemStore.settingManager = this.settingManager;
 
         MetadataResource resource = filesystemStore.getResourceDescription(context, "uuid", MetadataResourceVisibility.PUBLIC, "test.jpg", true);
@@ -100,7 +98,7 @@ public void testGetResourceDescriptionNonExistingUuid() throws Exception {
 
         FilesystemStore filesystemStore = new FilesystemStore();
         filesystemStore.settingManager = this.settingManager;
-        filesystemStore.filesystemStoreConfig = filesystemStoreConfig;
+        filesystemStore.storeFolderConfig = storeFolderConfig;
 
         // context, metadataUuid, visibility, path, approved)
         filesystemStore.getResourceDescription(context, "nonExistingUuid",

services/src/main/java/org/fao/geonet/api/records/MetadataSharingApi.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2001-2023 Food and Agriculture Organization of the
+ * Copyright (C) 2001-2024 Food and Agriculture Organization of the
  * United Nations (FAO-UN), United Nations World Food Programme (WFP)
  * and United Nations Environment Programme (UNEP)
  *
@@ -39,6 +39,7 @@
 import org.fao.geonet.api.exception.ResourceNotFoundException;
 import org.fao.geonet.api.processing.report.MetadataProcessingReport;
 import org.fao.geonet.api.processing.report.SimpleMetadataProcessingReport;
+import org.fao.geonet.api.records.attachments.Store;
 import org.fao.geonet.api.records.model.*;
 import org.fao.geonet.api.tools.i18n.LanguageUtils;
 import org.fao.geonet.config.IPublicationConfig;
@@ -50,10 +51,12 @@
 import org.fao.geonet.events.history.RecordPrivilegesChangeEvent;
 import org.fao.geonet.kernel.AccessManager;
 import org.fao.geonet.kernel.DataManager;
+import org.fao.geonet.kernel.GeonetworkDataDirectory;
 import org.fao.geonet.kernel.datamanager.*;
 import org.fao.geonet.kernel.search.IndexingMode;
 import org.fao.geonet.kernel.setting.SettingManager;
 import org.fao.geonet.kernel.setting.Settings;
+import org.fao.geonet.lib.Lib;
 import org.fao.geonet.repository.*;
 import org.fao.geonet.repository.specification.MetadataSpecs;
 import org.fao.geonet.repository.specification.MetadataValidationSpecs;
@@ -74,6 +77,7 @@
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpSession;
+import java.nio.file.Path;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -1150,11 +1154,21 @@ private void shareMetadataWithReservedGroup(String metadataUuid, boolean publish
         List<MetadataPublicationNotificationInfo> metadataListToNotifyPublication = new ArrayList<>();
         boolean notifyByEmail = StringUtils.isNoneEmpty(sm.getValue(SYSTEM_METADATAPRIVS_PUBLICATION_NOTIFICATIONLEVEL));
 
+        Path originalMetadataDataDir = Lib.resource.getMetadataDir(ApplicationContextHolder.get().getBean(GeonetworkDataDirectory.class), metadata.getId());
+
         setOperations(sharing, dataManager, context, appContext, metadata, operationMap, privileges,
             ApiUtils.getUserSession(session).getUserIdAsInt(), true, null, request,
             metadataListToNotifyPublication, notifyByEmail);
         metadataIndexer.indexMetadata(String.valueOf(metadata.getId()), true, IndexingMode.full);
 
+        // Rename the metadata data folder when required, for example when using a custom folder structure based on
+        // the metadata resource identifier, the folder can change.
+        Path newMetadataDataDir = Lib.resource.getMetadataDir(ApplicationContextHolder.get().getBean(GeonetworkDataDirectory.class), metadata.getId());
+        if (!originalMetadataDataDir.equals(newMetadataDataDir)) {
+            Store store = context.getBean("resourceStore", Store.class);
+            store.renameFolder(originalMetadataDataDir, newMetadataDataDir);
+        }
+
         java.util.Optional<PublicationOption> publicationOption = publicationConfig.getPublicationOptionConfiguration(publicationType);
         if (publicationOption.isPresent() &&
             publicationOption.get().hasPublicationTo(ReservedGroup.all) &&

web/src/main/webResources/WEB-INF/config.properties

@@ -66,12 +66,34 @@ metadata.extentApi.disableFullUrlBackgroundMapServices=true
 db.migration_onstartup=true
 
 # Values: DEFAULT, CUSTOM
+# - DEFAULT uses the default structure to store the metadata files
+#   datadir
+#   |-{{sequence_folder}}
+#   |    |-{{metadata_id}}
+#   |    |    |-private
+#   |    |    |-public
+#   |    |        |--doc.pdf
+# - CUSTOM allows to customise the naming of folders to store the metadata files, for example instead of using
+#   the internal metadata id, using the metadata uuid or the metadata resource identifier.
 datastore.folderStructureType=DEFAULT
+
 # Values: DEFAULT, NONE
+# - DEFAULT uses the PRIVATE / PUBLIC subfolder structure
+# - NONE stores all the files in the same metadata folder
 datastore.folderPrivilegesStrategy=DEFAULT
+
 # Required when datastore.folderStructureType has the value CUSTOM, use JSONPath expressions to use Elasticsearch index properties
+# -- folder structure to store published metadata
 #datastore.folderStructure=datastore/$.resourceIdentifier[0].code/
+# -- folder structure to store published metadata if the criteria for previous property doesn't match
+# -- (for example the metadata has no resource identifier)
 #datastore.folderStructureFallback=datastore/$.uuid/
-#datastore.folderStructureDraft=datastore/draft/$.resourceIdentifier[0].code/
-#datastore.folderStructureFallbackDraft=datastore/draft/$.uuid/
+# -- folder structure to store non published metadata (if don't want to store the files in the same folder structure
+# -- as datastore.folderStructure for non published metadata)
+# -- when not defined it is used datastore.folderStructure
+#datastore.folderStructureNonPublic=datastore/private/$.resourceIdentifier[0].code/
+# -- folder structure to store non published metadata if the criteria for previous property doesn't match
+# -- (for example the metadata has no resource identifier)
+# -- when not defined it is used datastore.folderStructureFallback
+#datastore.folderStructureFallbackNonPublic=datastore/private/$.uuid/