From c4cfba47a9d1e1d63a7a58e3be7ab7a6a5d4f727 Mon Sep 17 00:00:00 2001 From: mattiagiupponi <51856725+mattiagiupponi@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:26:28 +0200 Subject: [PATCH] align 11x to master (#278) * [Fixes #273] connect resource with execution request * store spacial file always true for cloning --------- Co-authored-by: Giovanni Allegri --- Dockerfile | 2 +- README.md | 3 +++ importer/celery_tasks.py | 3 +++ importer/handlers/common/metadata.py | 2 ++ importer/handlers/common/raster.py | 2 +- importer/handlers/common/remote.py | 2 +- importer/handlers/common/vector.py | 10 +++++----- importer/handlers/shapefile/handler.py | 2 +- importer/handlers/tiles3d/handler.py | 2 +- importer/handlers/tiles3d/tests.py | 2 +- importer/orchestrator.py | 5 +++++ importer/tests/unit/test_task.py | 2 +- 12 files changed, 25 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index b40b28ce..86e2e59b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM geonode/geonode-base:latest-ubuntu-22.04 RUN rm -rf /usr/src/geonode RUN git clone https://github.com/GeoNode/geonode.git /usr/src/geonode -RUN cd /usr/src/geonode && git fetch --all && git checkout master && cd - +RUN cd /usr/src/geonode && git fetch --all && git checkout 4.4.x && cd - RUN mkdir -p /usr/src/importer RUN cd .. diff --git a/README.md b/README.md index b72f2d3a..27f784d0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ ![PyPI - Downloads](https://img.shields.io/pypi/dm/geonode-importer) ![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/GeoNode/geonode-importer/runtests.yml) ![GitHub top language](https://img.shields.io/github/languages/top/GeoNode/geonode-importer) +# NOTE: +GeoNode 5 (master branch) includes the importer in its core. This repository and the `geonode-importer` package will be maintained for GeoNode <= 4.4.x +For more information https://github.com/GeoNode/geonode/issues/12368. # geonode-importer diff --git a/importer/celery_tasks.py b/importer/celery_tasks.py index ac79fe8b..48885770 100644 --- a/importer/celery_tasks.py +++ b/importer/celery_tasks.py @@ -367,6 +367,9 @@ def create_geonode_resource( handler_module_path, resource, _exec, **kwargs ) + # assign geonode resource to ExectionRequest + orchestrator.update_execution_request_obj(_exec, {"geonode_resource": resource}) + # at the end recall the import_orchestrator for the next step import_orchestrator.apply_async( ( diff --git a/importer/handlers/common/metadata.py b/importer/handlers/common/metadata.py index 76fbbcf8..b0778cd4 100644 --- a/importer/handlers/common/metadata.py +++ b/importer/handlers/common/metadata.py @@ -78,6 +78,8 @@ def import_resource(self, files: dict, execution_id: str, **kwargs): self.handle_metadata_resource(_exec, dataset, original_handler) dataset.refresh_from_db() + # assign the resource to the execution_obj + orchestrator.update_execution_request_obj(_exec, {"geonode_resource": dataset}) orchestrator.evaluate_execution_progress( execution_id, handler_module_path=str(self) diff --git a/importer/handlers/common/raster.py b/importer/handlers/common/raster.py index 350d877f..1a02b3d4 100644 --- a/importer/handlers/common/raster.py +++ b/importer/handlers/common/raster.py @@ -104,7 +104,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/common/remote.py b/importer/handlers/common/remote.py index ad9f9975..59a1cfaf 100755 --- a/importer/handlers/common/remote.py +++ b/importer/handlers/common/remote.py @@ -84,7 +84,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "source": _data.pop("source", "upload"), diff --git a/importer/handlers/common/vector.py b/importer/handlers/common/vector.py index 13f49cf4..85998fa9 100644 --- a/importer/handlers/common/vector.py +++ b/importer/handlers/common/vector.py @@ -114,7 +114,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), @@ -220,7 +220,7 @@ def perform_last_step(execution_id): that the execution is completed """ _exec = BaseHandler.perform_last_step(execution_id=execution_id) - if _exec and not _exec.input_params.get("store_spatial_file", False): + if _exec and not _exec.input_params.get("store_spatial_file", True): resources = ResourceHandlerInfo.objects.filter(execution_request=_exec) # getting all assets list assets = filter(None, [get_default_asset(x.resource) for x in resources]) @@ -607,7 +607,7 @@ def create_geonode_resource( resource_manager.set_thumbnail(None, instance=saved_dataset) ResourceBase.objects.filter(alternate=alternate).update(dirty_state=False) - + saved_dataset.refresh_from_db() return saved_dataset @@ -805,13 +805,13 @@ def _import_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs "Dynamic model does not exists, removing ogr2ogr table in progress" ) if instance_name is None: - logger.info("No table created, skipping...") + logger.warning("No table created, skipping...") return db_name = os.getenv("DEFAULT_BACKEND_DATASTORE", "datastore") with connections[db_name].cursor() as cursor: cursor.execute(f"DROP TABLE {instance_name}") except Exception as e: - logger.info(e) + logger.warning(e) pass def _publish_resource_rollback(self, exec_id, instance_name=None, *args, **kwargs): diff --git a/importer/handlers/shapefile/handler.py b/importer/handlers/shapefile/handler.py index 2ace0bf0..7f4acfcd 100644 --- a/importer/handlers/shapefile/handler.py +++ b/importer/handlers/shapefile/handler.py @@ -85,7 +85,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data additional_params = { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/tiles3d/handler.py b/importer/handlers/tiles3d/handler.py index 54689415..f7e58fbf 100755 --- a/importer/handlers/tiles3d/handler.py +++ b/importer/handlers/tiles3d/handler.py @@ -138,7 +138,7 @@ def extract_params_from_data(_data, action=None): """ if action == exa.COPY.value: title = json.loads(_data.get("defaults")) - return {"title": title.pop("title")}, _data + return {"title": title.pop("title"), "store_spatial_file": True}, _data return { "skip_existing_layers": _data.pop("skip_existing_layers", "False"), diff --git a/importer/handlers/tiles3d/tests.py b/importer/handlers/tiles3d/tests.py index 8817b86b..aee5595a 100755 --- a/importer/handlers/tiles3d/tests.py +++ b/importer/handlers/tiles3d/tests.py @@ -91,7 +91,7 @@ def test_extract_params_from_data(self): action="copy", ) - self.assertEqual(actual, {"title": "title_of_the_cloned_resource"}) + self.assertEqual(actual, {'store_spatial_file': True, 'title': 'title_of_the_cloned_resource'}) def test_is_valid_should_raise_exception_if_the_3dtiles_is_invalid(self): data = { diff --git a/importer/orchestrator.py b/importer/orchestrator.py index fb170ac5..e87127e4 100644 --- a/importer/orchestrator.py +++ b/importer/orchestrator.py @@ -332,6 +332,11 @@ def update_execution_request_status( task_args=celery_task_request.args ) + def update_execution_request_obj(self, _exec_obj, payload): + ExecutionRequest.objects.filter(pk=_exec_obj.pk).update(**payload) + _exec_obj.refresh_from_db() + return _exec_obj + def _last_step(self, execution_id, handler_module_path): """ Last hookable step for each handler before mark the execution as completed diff --git a/importer/tests/unit/test_task.py b/importer/tests/unit/test_task.py index ae1d4361..3c80b852 100644 --- a/importer/tests/unit/test_task.py +++ b/importer/tests/unit/test_task.py @@ -245,7 +245,7 @@ def test_publish_resource_if_overwrite_should_not_call_the_publishing( """ try: with self.assertRaises(Exception): - get_resource.return_falue = True + get_resource.return_value = True publish_resources.return_value = True extract_resource_to_publish.return_value = [ {"crs": 4326, "name": "dataset3"}