align 11x to master (#278)

* [Fixes #273] connect resource with execution request
* store spacial file always true for cloning
---------

Co-authored-by: Giovanni Allegri <[email protected]>

Dockerfile

@@ -1,7 +1,7 @@
 FROM geonode/geonode-base:latest-ubuntu-22.04
 RUN rm -rf /usr/src/geonode
 RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode
-RUN cd /usr/src/geonode && git fetch --all && git checkout master && cd -
+RUN cd /usr/src/geonode && git fetch --all && git checkout 4.4.x && cd -
 RUN mkdir -p /usr/src/importer
 
 RUN cd ..

README.md

@@ -1,6 +1,9 @@
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/geonode-importer) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GeoNode/geonode-importer/runtests.yml) ![GitHub top language](https://img.shields.io/github/languages/top/GeoNode/geonode-importer)
 
+# NOTE:
+GeoNode 5 (master branch) includes the importer in its core. This repository and the `geonode-importer` package will be maintained for GeoNode <= 4.4.x
 
+For more information https://github.com/GeoNode/geonode/issues/12368.
 
 # geonode-importer
 

importer/celery_tasks.py

@@ -367,6 +367,9 @@ def create_geonode_resource(
                 handler_module_path, resource, _exec, **kwargs
             )
 
+        # assign geonode resource to ExectionRequest
+        orchestrator.update_execution_request_obj(_exec, {"geonode_resource": resource})
+
         # at the end recall the import_orchestrator for the next step
         import_orchestrator.apply_async(
             (

importer/handlers/common/metadata.py

@@ -78,6 +78,8 @@ def import_resource(self, files: dict, execution_id: str, **kwargs):
         self.handle_metadata_resource(_exec, dataset, original_handler)
 
         dataset.refresh_from_db()
+        # assign the resource to the execution_obj
+        orchestrator.update_execution_request_obj(_exec, {"geonode_resource": dataset})
 
         orchestrator.evaluate_execution_progress(
             execution_id, handler_module_path=str(self)

importer/handlers/common/raster.py

@@ -104,7 +104,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),

importer/handlers/common/remote.py

@@ -84,7 +84,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "source": _data.pop("source", "upload"),

importer/handlers/common/vector.py

@@ -114,7 +114,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),
@@ -220,7 +220,7 @@ def perform_last_step(execution_id):
         that the execution is completed
         """
         _exec = BaseHandler.perform_last_step(execution_id=execution_id)
-        if _exec and not _exec.input_params.get("store_spatial_file", False):
+        if _exec and not _exec.input_params.get("store_spatial_file", True):
             resources = ResourceHandlerInfo.objects.filter(execution_request=_exec)
             # getting all assets list
             assets = filter(None, [get_default_asset(x.resource) for x in resources])
@@ -607,7 +607,7 @@ def create_geonode_resource(
         resource_manager.set_thumbnail(None, instance=saved_dataset)
 
         ResourceBase.objects.filter(alternate=alternate).update(dirty_state=False)
-
+        
         saved_dataset.refresh_from_db()
         return saved_dataset
 
@@ -805,13 +805,13 @@ def _import_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs
                     "Dynamic model does not exists, removing ogr2ogr table in progress"
                 )
                 if instance_name is None:
-                    logger.info("No table created, skipping...")
+                    logger.warning("No table created, skipping...")
                     return
                 db_name = os.getenv("DEFAULT_BACKEND_DATASTORE", "datastore")
                 with connections[db_name].cursor() as cursor:
                     cursor.execute(f"DROP TABLE {instance_name}")
             except Exception as e:
-                logger.info(e)
+                logger.warning(e)
                 pass
 
     def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs):

importer/handlers/shapefile/handler.py

@@ -85,7 +85,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         additional_params = {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),

importer/handlers/tiles3d/handler.py

@@ -138,7 +138,7 @@ def extract_params_from_data(_data, action=None):
         """
         if action == exa.COPY.value:
             title = json.loads(_data.get("defaults"))
-            return {"title": title.pop("title")}, _data
+            return {"title": title.pop("title"), "store_spatial_file": True}, _data
 
         return {
             "skip_existing_layers": _data.pop("skip_existing_layers", "False"),

importer/handlers/tiles3d/tests.py

@@ -91,7 +91,7 @@ def test_extract_params_from_data(self):
             action="copy",
         )
 
-        self.assertEqual(actual, {"title": "title_of_the_cloned_resource"})
+        self.assertEqual(actual, {'store_spatial_file': True, 'title': 'title_of_the_cloned_resource'})
 
     def test_is_valid_should_raise_exception_if_the_3dtiles_is_invalid(self):
         data = {

importer/orchestrator.py

@@ -332,6 +332,11 @@ def update_execution_request_status(
                 task_args=celery_task_request.args
             )
 
+    def update_execution_request_obj(self, _exec_obj, payload):
+        ExecutionRequest.objects.filter(pk=_exec_obj.pk).update(**payload)
+        _exec_obj.refresh_from_db()
+        return _exec_obj
+
     def _last_step(self, execution_id, handler_module_path):
         """
         Last hookable step for each handler before mark the execution as completed

importer/tests/unit/test_task.py

@@ -245,7 +245,7 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing(
         """
         try:
             with self.assertRaises(Exception):
-                get_resource.return_falue = True
+                get_resource.return_value = True
                 publish_resources.return_value = True
                 extract_resource_to_publish.return_value = [
                     {"crs": 4326, "name": "dataset3"}