Update file upload so that it has better error support. (#8427)

- Add bilingual error message.
- Fix queue removal so that it only remove items that are complete instead of fully clearing the queue.
- Fix error message Uncaught TypeError: Cannot read properties of undefined (reading 'message')
- Catch status 0 which is generally a network error and display an error message
- Catch status 413 which may come from a proxy server with no messages - it now displays correct error message.
- Add humanizeFileSize so that users can see errors in nice format instead of long byte format.

core/src/main/java/org/fao/geonet/api/exception/GeonetMaxUploadSizeExceededException.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2001-2024 Food and Agriculture Organization of the
+ * United Nations (FAO-UN), United Nations World Food Programme (WFP)
+ * and United Nations Environment Programme (UNEP)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or (at
+ * your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
+ *
+ * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2,
+ * Rome - Italy. email: [email protected]
+ */
+
+package org.fao.geonet.api.exception;
+
+import java.util.Locale;
+
+import org.fao.geonet.exceptions.LocalizedException;
+
+public class GeonetMaxUploadSizeExceededException extends LocalizedException {
+
+    public GeonetMaxUploadSizeExceededException() {
+        super();
+    }
+
+    public GeonetMaxUploadSizeExceededException(String message) {
+        super(message);
+    }
+
+    public GeonetMaxUploadSizeExceededException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public GeonetMaxUploadSizeExceededException(Throwable cause) {
+        super(cause);
+    }
+
+    protected String getResourceBundleBeanQualifier() {
+        return "apiMessages";
+    }
+
+    @Override
+    public GeonetMaxUploadSizeExceededException withMessageKey(String messageKey) {
+        super.withMessageKey(messageKey);
+        return this;
+    }
+
+    @Override
+    public GeonetMaxUploadSizeExceededException withMessageKey(String messageKey, Object[] messageKeyArgs) {
+        super.withMessageKey(messageKey, messageKeyArgs);
+        return this;
+    }
+
+    @Override
+    public GeonetMaxUploadSizeExceededException withDescriptionKey(String descriptionKey) {
+        super.withDescriptionKey(descriptionKey);
+        return this;
+    }
+
+    @Override
+    public GeonetMaxUploadSizeExceededException withDescriptionKey(String descriptionKey, Object[] descriptionKeyArgs) {
+        super.withDescriptionKey(descriptionKey, descriptionKeyArgs);
+        return this;
+    }
+
+    @Override
+    public GeonetMaxUploadSizeExceededException withLocale(Locale locale) {
+        super.withLocale(locale);
+        return this;
+    }
+}

core/src/main/java/org/fao/geonet/util/FileUtil.java

@@ -80,4 +80,21 @@ public static String readLastLines(File file, int lines) {
                 }
         }
     }
+
+    /**
+     * Similar to https://commons.apache.org/proper/commons-io/apidocs/org/apache/commons/io/FileUtils.html#byteCountToDisplaySize(long)
+     * however the format is returned in 2 decimal precision.
+     *
+     * @param bytes to be converted into human-readable format.
+     * @return human-readable formated bytes.
+     */
+    public static String humanizeFileSize(long bytes) {
+        if (bytes == 0) return "0 Bytes";
+
+        String[] sizes = {"Bytes", "KB", "MB", "GB", "TB"};
+        int i = (int) Math.floor(Math.log(bytes) / Math.log(1024)); // Determine the index for sizes
+        double humanizedSize = bytes / Math.pow(1024, i);
+
+        return String.format("%.2f %s", humanizedSize, sizes[i]);
+    }
 }

core/src/test/resources/org/fao/geonet/api/Messages.properties

@@ -176,6 +176,8 @@ api.exception.resourceAlreadyExists=Resource already exists
 api.exception.resourceAlreadyExists.description=Resource already exists.
 api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter
 api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter.
+exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded.
+exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}).
 exception.resourceNotFound.metadata=Metadata not found
 exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found.
 exception.resourceNotFound.resource=Metadata resource ''{0}'' not found

core/src/test/resources/org/fao/geonet/api/Messages_fre.properties

@@ -170,6 +170,8 @@ api.exception.resourceAlreadyExists=La ressource existe d\u00E9j\u00E0
 api.exception.resourceAlreadyExists.description=La ressource existe d\u00E9j\u00E0.
 api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfait
 api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait.
+exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e.
+exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}).
 exception.resourceNotFound.metadata=Fiches introuvables
 exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable.
 exception.resourceNotFound.resource=Ressource ''{0}'' introuvable

services/src/main/java/org/fao/geonet/api/GlobalExceptionController.java

@@ -35,6 +35,7 @@
 import org.fao.geonet.exceptions.UserNotFoundEx;
 import org.fao.geonet.exceptions.XSDValidationErrorEx;
 import org.fao.geonet.inspire.validator.InspireValidatorException;
+import org.fao.geonet.util.FileUtil;
 import org.fao.geonet.utils.Log;
 import org.json.JSONException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -148,15 +149,29 @@ public Object securityHandler(final HttpServletRequest request, final Exception
     }
 
     @ResponseBody
-    @ResponseStatus(HttpStatus.BAD_REQUEST)
+    @ResponseStatus(HttpStatus.PAYLOAD_TOO_LARGE)
     @ApiResponse(content = {@Content(mediaType = APPLICATION_JSON_VALUE)})
     @ExceptionHandler({
         MaxUploadSizeExceededException.class
     })
-    public ApiError maxFileExceededHandler(final Exception exception) {
-        storeApiErrorCause(exception);
+    public ApiError maxFileExceededHandler(final Exception exception, final HttpServletRequest request) {
+        Exception ex;
+        long contentLength = request.getContentLengthLong();
+        // As MaxUploadSizeExceededException is a spring exception, we need to convert it to a localized exception so that it can be translated.
+        if (exception instanceof MaxUploadSizeExceededException) {
+            ex = new GeonetMaxUploadSizeExceededException("uploadedResourceSizeExceededException", exception)
+                .withMessageKey("exception.maxUploadSizeExceeded",
+                    new String[]{FileUtil.humanizeFileSize(((MaxUploadSizeExceededException) exception).getMaxUploadSize())})
+                .withDescriptionKey("exception.maxUploadSizeExceeded.description",
+                    new String[]{FileUtil.humanizeFileSize(contentLength),
+                        FileUtil.humanizeFileSize(((MaxUploadSizeExceededException) exception).getMaxUploadSize())});
+        } else {
+            ex = exception;
+        }
+
+        storeApiErrorCause(ex);
 
-        return new ApiError("max_file_exceeded", exception);
+        return new ApiError("max_file_exceeded", ex);
     }
 
     @ResponseBody

web-ui/src/main/resources/catalog/components/filestore/FileStoreDirective.js

@@ -91,30 +91,91 @@
               }
             });
 
+            var humanizeDataSize = function (bytes) {
+              if (bytes === 0) return "0 Bytes";
+              const sizes = ["Bytes", "KB", "MB", "GB", "TB"];
+              const i = Math.floor(Math.log(bytes) / Math.log(1024)); // Determine the index for sizes
+              return parseFloat((bytes / Math.pow(1024, i)).toFixed(2)) + " " + sizes[i]; // Format size
+            };
+
+            // Function to remove files from scope.queue that match data.files by $$hashKey
+            var removeUploadedFilesFromQueue = function (data) {
+              data.files.forEach(function (file) {
+                for (var i = 0; i < scope.queue.length; i++) {
+                  if (scope.queue[i].$$hashKey === file.$$hashKey) {
+                    scope.queue.splice(i, 1);
+                    break;
+                  }
+                }
+              });
+            };
+
             var uploadResourceSuccess = function (e, data) {
               $rootScope.$broadcast("gnFileStoreUploadDone");
               if (scope.afterUploadCb && angular.isFunction(scope.afterUploadCb())) {
                 scope.afterUploadCb()(data.response().jqXHR.responseJSON);
               }
-              scope.clear(scope.queue);
+              removeUploadedFilesFromQueue(data);
             };
 
             var uploadResourceFailed = function (e, data) {
+              var jqXHR = angular.isDefined(data.response().jqXHR)
+                ? data.response().jqXHR
+                : null;
               var message =
-                data.errorThrown +
-                angular.isDefined(data.response().jqXHR.responseJSON.message)
-                  ? data.response().jqXHR.responseJSON.message
+                jqXHR &&
+                angular.isDefined(jqXHR.responseJSON) &&
+                angular.isDefined(jqXHR.responseJSON.message)
+                  ? jqXHR.responseJSON.message
                   : "";
+              if (message === "" && jqXHR) {
+                if (jqXHR.status === 0) {
+                  // Catch 0 which is generally a network error
+                  message = "uploadNetworkErrorException";
+                } else if (jqXHR.status === 413) {
+                  // Catch 413 which may come from a proxy server with no messages.
+                  message = "uploadedResourceSizeExceededException";
+                }
+              }
+              if (message === "" && typeof data.errorThrown === "string") {
+                message = data.errorThrown;
+              }
 
               $rootScope.$broadcast("StatusUpdated", {
                 title: $translate.instant("resourceUploadError"),
                 error: {
-                  message:
-                    message === "ResourceAlreadyExistException"
-                      ? $translate.instant("uploadedResourceAlreadyExistException", {
+                  message: (function () {
+                    switch (message) {
+                      case "uploadNetworkErrorException":
+                        return $translate.instant("uploadNetworkErrorException", {
                           file: data.files[0].name
-                        })
-                      : message
+                        });
+                      case "ResourceAlreadyExistException":
+                        return $translate.instant(
+                          "uploadedResourceAlreadyExistException",
+                          {
+                            file: data.files[0].name
+                          }
+                        );
+                      case "uploadedResourceSizeExceededException":
+                        console.error(
+                          "File " +
+                            data.files[0].name +
+                            " too large (" +
+                            data.files[0].size +
+                            " bytes)."
+                        );
+                        return $translate.instant(
+                          "uploadedResourceSizeExceededException",
+                          {
+                            file: data.files[0].name,
+                            humanizedSize: humanizeDataSize(data.files[0].size)
+                          }
+                        );
+                      default:
+                        return message;
+                    }
+                  })()
                 },
                 timeout: 0,
                 type: "danger"
@@ -125,6 +186,7 @@
               ) {
                 scope.afterUploadErrorCb()(message);
               }
+              removeUploadedFilesFromQueue(data);
             };
           }
         };

web-ui/src/main/resources/catalog/locales/en-v4.json

@@ -401,6 +401,8 @@
   "setServiceConnectPoint": "Add service connect point",
   "mimeType": "Format",
   "uploadedResourceAlreadyExistException": "File {{file}} already exist in this record data store. Remove it first.",
+  "uploadedResourceSizeExceededException": "File {{file}} too large ({{humanizedSize}}).",
+  "uploadNetworkErrorException": "File {{file}} failed to upload due to network error or connection reset.",
   "qualityMeasures": "Quality",
   "measureType": "Type",
   "measureName": "Measure",

web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages.properties

@@ -181,6 +181,8 @@ api.exception.resourceAlreadyExists=Resource already exists
 api.exception.resourceAlreadyExists.description=Resource already exists.
 api.exception.unsatisfiedRequestParameter=Unsatisfied request parameter
 api.exception.unsatisfiedRequestParameter.description=Unsatisfied request parameter.
+exception.maxUploadSizeExceeded=Maximum upload size of {0} exceeded.
+exception.maxUploadSizeExceeded.description=The request was rejected because its size ({0}) exceeds the configured maximum ({1}).
 exception.resourceNotFound.metadata=Metadata not found
 exception.resourceNotFound.metadata.description=Metadata with UUID ''{0}'' not found.
 exception.resourceNotFound.resource=Metadata resource ''{0}'' not found

web/src/main/webapp/WEB-INF/classes/org/fao/geonet/api/Messages_fre.properties

@@ -175,6 +175,8 @@ api.exception.resourceAlreadyExists=La ressource existe d\u00E9j\u00E0
 api.exception.resourceAlreadyExists.description=La ressource existe d\u00E9j\u00E0.
 api.exception.unsatisfiedRequestParameter=Param\u00E8tre de demande non satisfait
 api.exception.unsatisfiedRequestParameter.description=Param\u00E8tre de demande non satisfait.
+exception.maxUploadSizeExceeded=La taille maximale du t\u00E9l\u00E9chargement de {0} a \u00E9t\u00E9 exc\u00E9d\u00E9e.
+exception.maxUploadSizeExceeded.description=La demande a \u00E9t\u00E9 refus\u00E9e car sa taille ({0}) exc\u00E8de le maximum configur\u00E9 ({1}).
 exception.resourceNotFound.metadata=Fiches introuvables
 exception.resourceNotFound.metadata.description=La fiche ''{0}'' est introuvable.
 exception.resourceNotFound.resource=Ressource ''{0}'' introuvable