Add logging to server from uploader (#212)

* Add logging to server from uploader

src/fmu/sumo/uploader/_fileondisk.py

@@ -190,6 +190,8 @@ def upload_to_sumo(self, sumo_parent_id, sumo_connection):
 
         backoff = [1, 3, 9]
 
+        # UPLOAD METADATA
+
         for i in backoff:
             logger.debug("backoff in outer loop is %s", str(i))
 
@@ -249,6 +251,7 @@ def upload_to_sumo(self, sumo_parent_id, sumo_connection):
 
         # UPLOAD BLOB
 
+
         _t0_blob = time.perf_counter()
         upload_response = {}
         for i in backoff:
@@ -275,7 +278,7 @@ def upload_to_sumo(self, sumo_parent_id, sumo_connection):
                             # Outer code expects and interprets http error codes
                             upload_response["status_code"] = 418  
                             upload_response["text"] = "FAILED SEGY upload as OpenVDS. " + cmd_result.stderr
-                else:                
+                else:
                     response = self._upload_byte_string(
                         sumo_connection=sumo_connection,
                         object_id=self.sumo_object_id,
@@ -321,7 +324,7 @@ def upload_to_sumo(self, sumo_parent_id, sumo_connection):
                 continue
 
             except AuthenticationError as err:
-                logger.debug("Upload failed: %s", upload_response["text"])
+                logger.debug("Upload failed: %s", upload_response.get("text"))
                 result["status"] = "rejected"
                 self._delete_metadata(sumo_connection, self.sumo_object_id)
                 return result
@@ -344,3 +347,4 @@ def upload_to_sumo(self, sumo_parent_id, sumo_connection):
             result["status"] = "ok"
 
         return result
+

src/fmu/sumo/uploader/caseondisk.py

@@ -91,6 +91,13 @@ def __init__(
         logger.debug("self._sumo_parent_id is %s", self._sumo_parent_id)
         self._files = []
 
+        self._sumo_logger = sumo_connection.api.getLogger("log_2_server_caseondisk")
+        self._sumo_logger.setLevel(logging.INFO)
+        # Avoid that logging to sumo-server also is visible in local logging:
+        self._sumo_logger.propagate = False 
+        self._sumo_logger.info("Upload init for sumo_parent_id: " 
+                              + str(self._sumo_parent_id))
+
     def __str__(self):
         s = f"{self.__class__}, {len(self._files)} files."
 
@@ -367,26 +374,11 @@ def upload(self, threads=4, max_attempts=1, register_case=False):
 
         _dt = time.perf_counter() - _t0
 
+        upload_statistics = ""
         if len(ok_uploads) > 0:
             upload_statistics = _calculate_upload_stats(ok_uploads)
             logger.info(upload_statistics)
 
-        if failed_uploads:
-            logger.info(f"{len(failed_uploads)} files failed to be uploaded")
-
-            for u in failed_uploads[0:4]:
-                logger.info("\n" + "=" * 50)
-
-                logger.info(f"Filepath: {u.get('blob_file_path')}")
-                logger.info(
-                    f"Metadata: [{u.get('metadata_upload_response_status_code')}] "
-                    f"{u.get('metadata_upload_response_text')}"
-                )
-                logger.info(
-                    f"Blob: [{u.get('blob_upload_response_status_code')}] "
-                    f"{u.get('blob_upload_response_status_text')}"
-                )
-
         if rejected_uploads:
             logger.info(
                 f"\n\n{len(rejected_uploads)} files rejected by Sumo. First 5 rejected files:"
@@ -404,10 +396,11 @@ def upload(self, threads=4, max_attempts=1, register_case=False):
                     f"Blob: [{u.get('blob_upload_response_status_code')}] "
                     f"{u.get('blob_upload_response_status_text')}"
                 )
+                self._sumo_logger.info(_get_log_msg(self.sumo_parent_id, u))
 
         if failed_uploads:
             logger.info(
-                f"\n\n{len(failed_uploads)} files rejected by Sumo. First 5 rejected files:"
+                f"\n\n{len(failed_uploads)} files failed by Sumo. First 5 failed files:"
             )
 
             for u in failed_uploads[0:4]:
@@ -422,6 +415,7 @@ def upload(self, threads=4, max_attempts=1, register_case=False):
                     f"Blob: [{u.get('blob_upload_response_status_code')}] "
                     f"{u.get('blob_upload_response_status_text')}"
                 )
+                self._sumo_logger.info(_get_log_msg(self.sumo_parent_id, u))
 
         logger.info("Summary:")
         logger.info("Total files count: %s", str(len(self.files)))
@@ -430,8 +424,40 @@ def upload(self, threads=4, max_attempts=1, register_case=False):
         logger.info("Rejected: %s", str(len(rejected_uploads)))
         logger.info("Wall time: %s sec", str(_dt))
 
+        summary = {
+            "upload_summary": {
+                "parent_id": self.sumo_parent_id,
+                "total_files_count": str(len(self.files)),
+                "ok_files": str(len(ok_uploads)),
+                "failed_files": str(len(failed_uploads)),
+                "rejected_files": str(len(rejected_uploads)),
+                "wall_time_seconds": str(_dt),
+                "upload_statistics": upload_statistics
+            }
+        }
+        self._sumo_logger.info(str(summary))
+
         return ok_uploads
 
+def _get_log_msg(sumo_parent_id, status):
+    """Return a suitable logging for upload issues."""
+
+    json = {
+        "upload_issue": {
+            "case_uuid": str(sumo_parent_id),
+            "filepath": str(status.get('blob_file_path')),
+            "metadata": {
+                "status_code": str(status.get('metadata_upload_response_status_code')),
+                "response_text": status.get('metadata_upload_response_text')
+            },
+            "blob": {
+                "status_code": str(status.get('blob_upload_response_status_code')),
+                "response_text": ((status.get('blob_upload_response_status_text')))
+            }
+        }
+    }
+    return json
+
 
 def _sanitize_datetimes(data):
     """Sanitize datetimes.
@@ -510,4 +536,4 @@ def _calculate_upload_stats(uploads):
         },
     }
 
-    return stats
+    return stats

src/fmu/sumo/uploader/scripts/sumo_upload.py

@@ -118,6 +118,9 @@ def sumo_upload_main(
         )
         warnings.warn(
             "Problem related to Sumo upload: " f"{err}")
+        _sumo_logger = sumo_connection.api.getLogger("log_2_server_sumo_upload")
+        _sumo_logger.propagate = False
+        _sumo_logger.warning("Problem related to Sumo upload for case: ", case_metadata_path, " ", err)
         return