Merge pull request #103 from openownership/fix_json_to_xlsx_conversion

cove_bods/process.py: Fix ConvertJSONIntoSpreadsheets error handling

cove_bods/process.py

@@ -1,3 +1,6 @@
+from django.core.files.base import ContentFile
+from django.core.files.storage import default_storage
+
 from libcovebods.schema import SchemaBODS
 from libcovebods.config import LibCoveBODSConfig
 from libcovebods.jsonschemavalidate import JSONSchemaValidator
@@ -21,6 +24,30 @@
 from libcoveweb2.utils import group_data_list_by
 
 
+def create_error_file(directory: str, name: str, data: dict):
+    """Create temporary error file"""
+    filename = os.path.join(directory, f"{name}-error.json")
+    return default_storage.save(filename, ContentFile(json.dumps(data).encode('utf-8')))
+
+
+def error_file_exists(directory: str, name: str) -> bool:
+    """Test if error file exists"""
+    filename = os.path.join(directory, f"{name}-error.json")
+    return default_storage.exists(filename)
+
+
+def read_error_file(directory: str, name: str) -> dict:
+    """Read data from error file"""
+    filename = os.path.join(directory, f"{name}-error.json")
+    return json.loads(default_storage.open(filename).read().decode('utf-8'))
+
+
+def delete_error_file(directory: str, name: str):
+    """Delete temporary error file"""
+    filename = os.path.join(directory, f"{name}-error.json")
+    default_storage.delete(filename)
+
+
 class Sample(ProcessDataTask):
     def is_processing_applicable(self) -> bool:
         return True
@@ -253,7 +280,11 @@ def is_processing_applicable(self) -> bool:
         return True
 
     def is_processing_needed(self) -> bool:
-        return not os.path.exists(self.xlsx_filename)
+        if os.path.exists(self.xlsx_filename):
+            return False
+        if error_file_exists(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets"):
+            return False
+        return True
 
     def process(self, process_data: dict) -> dict:
 
@@ -276,7 +307,9 @@ def process(self, process_data: dict) -> dict:
             flattentool.flatten(process_data["json_data_filename"], **flatten_kwargs)
         except Exception as err:
             capture_exception(err)
-            # TODO log and show to user. https://github.com/Open-Telecoms-Data/cove-ofds/issues/24
+            create_error_file(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets",
+                              {"type": type(err).__name__,
+                               "filename": process_data["json_data_filename"].split('/')[-1]})
 
         return process_data
 
@@ -293,6 +326,12 @@ def get_context(self):
             context["download_xlsx_size"] = os.stat(self.xlsx_filename).st_size
         else:
             context["can_download_xlsx"] = False
+            if error_file_exists(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets"):
+                context["xlsx_error"] = read_error_file(self.supplied_data.storage_dir(),
+                                                        "ConvertJSONIntoSpreadsheets")
+                delete_error_file(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets")
+            else:
+                context["xlsx_error"] = False
         # done!
         return context
 

cove_bods/templates/cove_bods/explore.html

@@ -84,6 +84,13 @@ <h4 class="panel-title">
                                 <span class="glyphicon glyphicon-download" aria-hidden="true"></span>
                                 <a href="{{ download_xlsx_url }}">Excel Spreadsheet <small>({% trans 'Converted from Original' %})</small> ({{ download_xlsx_size|filesizeformat }})</a>
                             </li>
+                        {% else %}
+                            {% if xlsx_error %}
+                            <li>
+                                <span class="glyphicon glyphicon-download" aria-hidden="true"></span>
+                                <span>Excel Spreadsheet - Error ({{ xlsx_error.type }}) converting from JSON ({{ xlsx_error.filename }})</span>
+                            </li>
+                            {% endif %}
                         {% endif %}
 
                         {% if original_format != 'json' and can_download_json %}