diff --git a/examples/extraction_pipelines/S2_extraction_example.ipynb b/examples/extraction_pipelines/S2_extraction_example.ipynb new file mode 100644 index 0000000..934a84a --- /dev/null +++ b/examples/extraction_pipelines/S2_extraction_example.ipynb @@ -0,0 +1,1233 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Example of a GFMAP full-extraction pipeline\n", + "\n", + "Designing of GFMAP Job DataFrames and DataCube creators functions, as well as post job-actions.\n", + "\n", + "Those dataframe should be containing all the necessary infromation to run a job and know where to save it." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### First step: splitting the job\n", + "\n", + "Splitting the dataset of extraction in multiple job based on position is necessary to respect OpenEO limitations.\n", + "\n", + "This script performs a split with the H3 hexagonal grid, yielding a list of sub-geodataframes.\n", + "\n", + "A subtility here is that some polygons are not directly extracted (field with `extract=False`), but should be kept for post-job actions. This requirement is filled by removing sub-dataframes that do not contain any extractable polyons." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# Configuring the logging for the openeo_gfmap package\n", + "from openeo_gfmap.manager import _log\n", + "import logging\n", + "\n", + "_log.setLevel(logging.DEBUG)\n", + "\n", + "stream_handler = logging.StreamHandler()\n", + "_log.addHandler(stream_handler)\n", + "\n", + "formatter = logging.Formatter('%(asctime)s|%(name)s|%(levelname)s: %(message)s')\n", + "stream_handler.setFormatter(formatter)\n", + "\n", + "# Exclude the other loggers from other libraries\n", + "class MyLoggerFilter(logging.Filter):\n", + " def filter(self, record):\n", + " return record.name == _log.name\n", + "\n", + "stream_handler.addFilter(MyLoggerFilter())\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/data/users/Private/couchard/openeo-gfmap/src/openeo_gfmap/manager/job_splitters.py:53: UserWarning: Geometry is in a geographic CRS. Results from 'centroid' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " polygons[\"h3index\"] = polygons.geometry.centroid.apply(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "137 jobs before filtering empty one (no extraction)\n", + "93 jobs after filtering empty one (no extraction)\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import geopandas as gpd\n", + "from openeo_gfmap.manager.job_splitters import split_job_hex\n", + "\n", + "base_df_path = Path('/vitodata/worldcereal/tmp/kristof/GFMAP/2021_EUR_DEMO_POLY_110.gpkg')\n", + "base_df = gpd.read_file(base_df_path)\n", + "# Splits the job using GFMAP\n", + "split_jobs = split_job_hex(\n", + " base_df, max_points=500, grid_resolution=4\n", + ")\n", + "\n", + "print(f'{len(split_jobs)} jobs before filtering empty one (no extraction)')\n", + "\n", + "# Remove the geometry where there are no points with the \"extract\" flag\n", + "split_jobs = [\n", + " job for job in split_jobs if job.extract.any()\n", + "]\n", + "print(f'{len(split_jobs)} jobs after filtering empty one (no extraction)')\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Second step: creating a dataframe for the GFMAP Job Manager\n", + "\n", + "Implementing a function that yields a `pandas.DataFrame` where each row correponds to a job.\n", + "\n", + "The dataframe should contain the informations required by the GFMAP Job Manager, as well as additional information used by the datacube creation function and the post-job action function.\n", + "\n", + "The output dataframe should be savable as a .csv file.\n", + "\n", + "Note: the full information of a sub-geodataframe of polygons can be saved into a row of a `pandas.DataFrame` by storing it in a row as string implementing the `geojson.FeatureCollection` interface. To convert the `geopandas.GeoDataFrame` into a stirng, simply use the `.to_json()` function." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
backend_nameout_prefixout_extensionstart_dateend_dategeometry
0cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
1cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
2cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
3cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
4cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
.....................
88cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
89cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
90cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
91cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
92cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
\n", + "

93 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " backend_name out_prefix out_extension start_date end_date \\\n", + "0 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "1 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "2 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "3 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "4 cdse S2 .nc 2020-08-30 2022-03-03 \n", + ".. ... ... ... ... ... \n", + "88 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "89 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "90 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "91 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "92 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "\n", + " geometry \n", + "0 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "1 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "2 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "3 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "4 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + ".. ... \n", + "88 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "89 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "90 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "91 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "92 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "\n", + "[93 rows x 6 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from openeo_gfmap import Backend\n", + "from typing import List\n", + "import pandas as pd\n", + "\n", + "def create_job_dataframe_s2(backend: Backend, split_jobs: List[gpd.GeoDataFrame]) -> pd.DataFrame:\n", + " \"\"\"Create a dataframe from the split jobs, containg all the necessary information to run the job.\"\"\"\n", + " columns = ['backend_name', 'out_prefix', 'out_extension', 'start_date', 'end_date', 'geometry']\n", + " rows = []\n", + " for job in split_jobs:\n", + " # Compute the average in the valid date and make a buffer of 1.5 year around\n", + " median_time = pd.to_datetime(job.valid_date).mean()\n", + " start_date = median_time - pd.Timedelta(days=275) # A bit more than 9 months\n", + " end_date = median_time + pd.Timedelta(days=275) # A bit more than 9 months\n", + " \n", + " rows.append(\n", + " pd.Series(\n", + " dict(zip(columns, [backend.value, 'S2', '.nc', start_date.strftime('%Y-%m-%d'), end_date.strftime('%Y-%m-%d'), job.to_json()]))\n", + " )\n", + " )\n", + "\n", + " return pd.DataFrame(rows)\n", + "\n", + "job_df = create_job_dataframe_s2(Backend.CDSE, split_jobs)\n", + "\n", + "job_df" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
backend_nameout_prefixout_extensionstart_dateend_dategeometry
0cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
1cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
2cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
3cdseS2.nc2020-08-302022-03-03{\"type\": \"FeatureCollection\", \"features\": [{\"i...
\n", + "
" + ], + "text/plain": [ + " backend_name out_prefix out_extension start_date end_date \\\n", + "0 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "1 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "2 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "3 cdse S2 .nc 2020-08-30 2022-03-03 \n", + "\n", + " geometry \n", + "0 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "1 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "2 {\"type\": \"FeatureCollection\", \"features\": [{\"i... \n", + "3 {\"type\": \"FeatureCollection\", \"features\": [{\"i... " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Run a subset of the jobs to test the manager, the selected jobs have a fair amount of geometries to extract\n", + "job_df = job_df.iloc[[0, 2, 3, -6]].reset_index(drop=True)\n", + "job_df" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Third step: implement the datacube creator function.\n", + "\n", + "Implement a function to create, from the additional rows provided before, an `openeo.BatchJob` that will be used to run the job.\n", + "\n", + "In this case we extract Sentinel-2 data, and we remove the polygons with `extract=False` (although we keep them in the row for the post-job action.)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import openeo\n", + "\n", + "import pandas as pd\n", + "import geojson\n", + "\n", + "from openeo_gfmap import TemporalContext, Backend, BackendContext, FetchType\n", + "from openeo_gfmap.fetching import build_sentinel2_l2a_extractor\n", + "\n", + "\n", + "def create_datacube_s2(row: pd.Series, connection: openeo.DataCube, provider=None, connection_provider=None) -> openeo.BatchJob:\n", + "\n", + " def buffer_geometry(geometry: geojson.FeatureCollection, buffer: float) -> str:\n", + " gdf = gpd.GeoDataFrame.from_features(geometry).set_crs(epsg=4326)\n", + " utm = gdf.estimate_utm_crs()\n", + " gdf = gdf.to_crs(utm)\n", + " gdf['geometry'] = gdf.centroid.buffer(distance=buffer, cap_style=3)\n", + "\n", + " gdf = gdf.to_crs(epsg=4326)\n", + " return geojson.loads(gdf.to_json())\n", + "\n", + "\n", + " def filter_extractonly_geometries(collection: geojson.FeatureCollection):\n", + " # Filter out geometries that do not have the field extract=True\n", + " features = [f for f in collection.features if f.properties.get('extract', False)]\n", + " return geojson.FeatureCollection(features)\n", + "\n", + " start_date = row.start_date\n", + " end_date = row.end_date\n", + " temporal_context = TemporalContext(start_date, end_date)\n", + "\n", + " # Get the feature collection containing the geometry to the job\n", + " geometry = geojson.loads(row.geometry)\n", + " assert isinstance(geometry, geojson.FeatureCollection)\n", + "\n", + " # Filter the geometry to the rows with the extract only flag\n", + " geometry = filter_extractonly_geometries(geometry)\n", + " assert len(geometry.features) > 0, \"No geometries with the extract flag found\"\n", + "\n", + " # Performs a buffer of 64 px around the geometry\n", + " geometry = buffer_geometry(geometry, 319)\n", + "\n", + " # Backend name and fetching type\n", + " backend = Backend(row.backend_name)\n", + " backend_context = BackendContext(backend)\n", + "\n", + " fetch_type = FetchType.POLYGON\n", + " bands_to_download = ['S2-B01', 'S2-B02', 'S2-B03', 'S2-B04', 'S2-B05', 'S2-B06', 'S2-B07', 'S2-B08', 'S2-B8A', 'S2-B09', 'S2-B11', 'S2-B12', 'S2-SCL']\n", + "\n", + " # Create the job to extract S2\n", + " extraction_parameters = {\n", + " \"target_resolution\": 10\n", + " }\n", + " extractor = build_sentinel2_l2a_extractor(\n", + " backend_context, bands=bands_to_download, fetch_type=fetch_type.POLYGON, **extraction_parameters \n", + " )\n", + "\n", + " cube = extractor.get_cube(connection, geometry, temporal_context)\n", + "\n", + " # Get the h3index to use in the tile\n", + " h3index = geometry.features[0].properties['h3index']\n", + " valid_date = geometry.features[0].properties['valid_date']\n", + "\n", + " # Increase the memory of the jobs\n", + " job_options = {\n", + " \"executor-memory\": \"5G\",\n", + " \"executor-memoryOverhead\": \"2G\",\n", + " }\n", + "\n", + " return cube.create_job(\n", + " out_format=\"NetCDF\",\n", + " title=f\"GFMAP_Extraction_S2_{h3index}_{valid_date}\",\n", + " sample_by_feature=True,\n", + " job_options=job_options\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fourth step: create output paths\n", + "\n", + "Implement a function that from a temporary path containing a job result, from the job dataframe row and the root folder will choose the output path where to save that job result." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 4.62 s, sys: 40.8 ms, total: 4.67 s\n", + "Wall time: 4.67 s\n" + ] + } + ], + "source": [ + "%%time\n", + "\n", + "# Load the S2 grid\n", + "s2_grid = gpd.read_file('./s2grid_bounds.geojson')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import xarray as xr\n", + "from pyproj import Transformer, CRS\n", + "from shapely.geometry import box, Point\n", + "\n", + "def generate_output_path_s2(root_folder: Path, tmp_path: Path, geometry_index: int, row: pd.Series):\n", + " features = geojson.loads(row.geometry)\n", + " sample_id = features[geometry_index].properties['sample_id']\n", + " ref_id = features[geometry_index].properties['ref_id']\n", + " \n", + " # Loads the array lazily in-memory\n", + " try:\n", + " inds = xr.open_dataset(tmp_path, chunks='auto')\n", + " \n", + " source_crs = CRS.from_wkt(inds.crs.attrs['crs_wkt'])\n", + " dst_crs = CRS.from_epsg(4326)\n", + " \n", + " transformer = Transformer.from_crs(source_crs, dst_crs, always_xy=True)\n", + " bounds = inds.x.min().item(), inds.y.min().item(), inds.x.max().item(), inds.y.max().item()\n", + " bbox = box(*bounds)\n", + "\n", + " # Get the center of the box\n", + " centroid = bbox.centroid\n", + " lon, lat = transformer.transform(centroid.x, centroid.y)\n", + " centroid_pt = Point(lon, lat)\n", + "\n", + " # Intersecting with the s2 grid\n", + " intersecting = s2_grid.geometry.intersects(centroid_pt)\n", + "\n", + " # Select the intersecting cell that has a centroid the closest from the point\n", + " intersecting_cells = s2_grid[intersecting]\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + " s2_tile = intersecting_cells.iloc[0]\n", + "\n", + " s2_tile_id = s2_tile.tile\n", + "\n", + " subfolder = root_folder / ref_id / str(source_crs.to_epsg()) / s2_tile_id / sample_id\n", + " except Exception:\n", + " # TODO: _log.error('Could not find S2 tile for file, setting up a dummy path')\n", + " subfolder = root_folder / 'unsortable'\n", + "\n", + " return subfolder / f'{row.out_prefix}_{sample_id}_{source_crs.to_epsg()}_{row.start_date}_{row.end_date}{row.out_extension}'\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Fifth step: Define the post-job action\n", + "\n", + "The post-job action will be called once the job resut was downloaded and saved to a specific path.\n", + "\n", + "A post-job action function must receive 3 parameters:\n", + "* `result_paths`: Paths to the downloaded job result files.\n", + "* `row`: The current job dataframe row.\n", + "* `parameters`: User-defined parameters set in the `GFMAPJobManager` constructor.\n", + "\n", + "The post-job action must return a list of paths containing the results from that job. For example, if no file is created/deleted in the post-job action, then the user can simply return the list of paths it has received as input `result_paths`. If instead files are added or removed, then the user will need to modify this list accordingly before returning it." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from rasterio.features import rasterize\n", + "from rasterio.transform import from_bounds\n", + "import json\n", + "\n", + "def post_job_action(result_paths: list, row: pd.Series, parameters: dict = {}) -> list:\n", + " base_gpd = gpd.GeoDataFrame.from_features(json.loads(row.geometry)).set_crs(epsg=4326)\n", + " assert len(base_gpd[base_gpd.extract == True]) == len(result_paths), \"The number of result paths should be the same as the number of geometries\"\n", + " extracted_gpd = base_gpd[base_gpd.extract == True].reset_index(drop=True)\n", + " # In this case we want to burn the metadata in a new file in the same folder as the S2 product\n", + " for idx, result_path in enumerate(result_paths.copy()):\n", + " sample_id = extracted_gpd.iloc[idx].sample_id\n", + " ref_id = extracted_gpd.iloc[idx].ref_id\n", + " lc_label = extracted_gpd.iloc[idx].landcover_label\n", + " confidence = extracted_gpd.iloc[idx].confidence\n", + " valid_date = extracted_gpd.iloc[idx].valid_date\n", + "\n", + " result_ds = xr.open_dataset(result_path, chunks='auto')\n", + "\n", + " target_crs = CRS.from_wkt(result_ds.crs.attrs['crs_wkt'])\n", + "\n", + " # Get the surrounding polygons around our extracted center geometry to rastetize them\n", + " bounds = (result_ds.x.min().item(), result_ds.y.min().item(), result_ds.x.max().item(), result_ds.y.max().item())\n", + " bbox = box(*bounds)\n", + " surround_gpd = base_gpd.to_crs(target_crs).clip(bbox)\n", + "\n", + " # Burn the polygon croptypes\n", + " transform = from_bounds(*bounds, result_ds.x.size, result_ds.y.size)\n", + " croptype_shapes = list(zip(surround_gpd.geometry, surround_gpd.croptype_label))\n", + " croptype = rasterize(croptype_shapes, out_shape=(result_ds.y.size, result_ds.x.size), transform=transform, all_touched=True, fill=0, default_value=65535, dtype='uint16')\n", + "\n", + " # Create the attributes to add to the metadata\n", + " crs_layer = result_ds['crs']\n", + " attributes = {\n", + " 'ref_id': ref_id,\n", + " 'sample_id': sample_id,\n", + " 'landcover_label': lc_label,\n", + " 'confidence': str(confidence),\n", + " 'valid_date': valid_date\n", + " }\n", + " attributes.update(result_ds.attrs)\n", + "\n", + " aux_dataset = xr.Dataset({'CROPTYPE': (('y', 'x'), croptype)}, coords={'y': result_ds.y, 'x': result_ds.x}, attrs=attributes)\n", + "\n", + " # Include the CRS layer from OpenEO\n", + " aux_dataset['crs'] = crs_layer\n", + " aux_dataset.attrs.update(result_ds.attrs)\n", + "\n", + " # Save the metadata in the same folder as the S2 product\n", + " metadata_path = result_path.parent / f'AUX_{sample_id}_{target_crs.to_epsg()}_{valid_date}.nc'\n", + " aux_dataset.to_netcdf(metadata_path, format='NETCDF4', engine='h5netcdf')\n", + " result_paths.append(metadata_path)\n", + "\n", + " return result_paths\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Sixth and last step: Running the manager\n", + "\n", + "Let's initialize and execute the Job Manager as defined the GFMAP, and then run it using the functions defined previously" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from openeo_gfmap.manager.job_manager import GFMAPJobManager\n", + "from openeo_gfmap.backend import cdse_connection\n", + "\n", + "base_output_dir = Path('/data/users/Public/couchard/world_cereal/extractions/')\n", + "tracking_job_csv = base_output_dir / 'job_tracker.csv'\n", + "\n", + "manager = GFMAPJobManager(\n", + " output_dir=base_output_dir,\n", + " output_path_generator=generate_output_path_s2,\n", + " post_job_action=post_job_action,\n", + " poll_sleep=60,\n", + " n_threads=2,\n", + " post_job_params={}\n", + ")\n", + "\n", + "manager.add_backend(\n", + " Backend.CDSE.value, cdse_connection, parallel_jobs=2\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-12 09:37:32,563|openeo_gfmap.manager|INFO: Starting job manager using 2 worker threads.\n", + "2024-02-12 09:37:32,575|openeo_gfmap.manager|INFO: Workers started, creating and running jobs.\n", + "2024-02-12 09:37:32,610|openeo_gfmap.manager|DEBUG: Normalizing dataframe. Columns: Index(['backend_name', 'out_prefix', 'out_extension', 'start_date', 'end_date',\n", + " 'geometry', 'status', 'id', 'start_time', 'cpu', 'memory', 'duration',\n", + " 'description', 'costs'],\n", + " dtype='object')\n", + "2024-02-12 09:37:32,612|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Authenticated using refresh token.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-12 09:37:33,882|openeo_gfmap.manager|DEBUG: Status of job j-2402124f58244715803a335628079b9a is finished (on backend cdse).\n", + "2024-02-12 09:37:33,883|openeo_gfmap.manager|INFO: Job j-2402124f58244715803a335628079b9a finished successfully, queueing on_job_done...\n", + "2024-02-12 09:37:33,888|openeo_gfmap.manager|DEBUG: Worker thread Thread-4: polled finished job with status PostJobStatus.FINISHED.\n", + "2024-02-12 09:37:38,861|openeo_gfmap.manager|DEBUG: Status of job j-24021279e49745e9a3806584fe0e8c9a is finished (on backend cdse).\n", + "2024-02-12 09:37:38,863|openeo_gfmap.manager|INFO: Job j-24021279e49745e9a3806584fe0e8c9a finished successfully, queueing on_job_done...\n", + "2024-02-12 09:37:38,867|openeo_gfmap.manager|DEBUG: Worker thread Thread-5: polled finished job with status PostJobStatus.FINISHED.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DataCube()\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-12 09:37:43,664|openeo_gfmap.manager|DEBUG: Downloading asset openEO_0.nc from job j-2402124f58244715803a335628079b9a -> /tmp/tmp91ksct_n\n", + "2024-02-12 09:37:45,148|openeo_gfmap.manager|DEBUG: Downloading asset openEO_0.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /tmp/tmp_4v2t_f0\n", + "2024-02-12 09:37:52,138|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_0.nc from job j-2402124f58244715803a335628079b9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:37:52,672|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_0.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VMD/2021_LV_LPIS_POLY_110-12880341/S2_2021_LV_LPIS_POLY_110-12880341_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:53,429|openeo_gfmap.manager|INFO: Downloaded asset openEO_0.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VMD/2021_LV_LPIS_POLY_110-12880341/S2_2021_LV_LPIS_POLY_110-12880341_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:53,432|openeo_gfmap.manager|DEBUG: Downloading asset openEO_1.nc from job j-2402124f58244715803a335628079b9a -> /tmp/tmpw01mu9qf\n", + "2024-02-12 09:37:54,746|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_0.nc from job j-24021279e49745e9a3806584fe0e8c9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:37:54,905|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_0.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1551152/S2_at2021lpis1551152_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:55,842|openeo_gfmap.manager|INFO: Downloaded asset openEO_0.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1551152/S2_at2021lpis1551152_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:55,847|openeo_gfmap.manager|DEBUG: Downloading asset openEO_1.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /tmp/tmpgdkcbiqo\n", + "2024-02-12 09:37:58,579|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_1.nc from job j-2402124f58244715803a335628079b9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:37:58,765|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_1.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VMD/2021_LV_LPIS_POLY_110-12525751/S2_2021_LV_LPIS_POLY_110-12525751_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:59,373|openeo_gfmap.manager|INFO: Downloaded asset openEO_1.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VMD/2021_LV_LPIS_POLY_110-12525751/S2_2021_LV_LPIS_POLY_110-12525751_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:37:59,378|openeo_gfmap.manager|DEBUG: Downloading asset openEO_2.nc from job j-2402124f58244715803a335628079b9a -> /tmp/tmp9wfpr2qu\n", + "2024-02-12 09:38:05,003|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_1.nc from job j-24021279e49745e9a3806584fe0e8c9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:38:05,194|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_1.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1313057/S2_at2021lpis1313057_32633_2020-08-30_2022-03-03.nc\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "DataCube()\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-12 09:38:05,385|openeo_gfmap.manager|INFO: Downloaded asset openEO_1.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1313057/S2_at2021lpis1313057_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:05,390|openeo_gfmap.manager|DEBUG: Downloading asset openEO_2.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /tmp/tmpkew_9hyq\n", + "2024-02-12 09:38:08,255|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_2.nc from job j-2402124f58244715803a335628079b9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:38:08,439|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_2.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VND/2021_LV_LPIS_POLY_110-12440159/S2_2021_LV_LPIS_POLY_110-12440159_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:08,747|openeo_gfmap.manager|INFO: Downloaded asset openEO_2.nc from job j-2402124f58244715803a335628079b9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32635/35VND/2021_LV_LPIS_POLY_110-12440159/S2_2021_LV_LPIS_POLY_110-12440159_32635_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:08,749|openeo_gfmap.manager|DEBUG: Calling post job action for job j-2402124f58244715803a335628079b9a...\n", + "2024-02-12 09:38:09,559|openeo_gfmap.manager|INFO: Job j-2402124f58244715803a335628079b9a and post job action finished successfully.\n", + "2024-02-12 09:38:15,092|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_2.nc from job j-24021279e49745e9a3806584fe0e8c9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:38:15,307|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_2.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1561389/S2_at2021lpis1561389_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:15,516|openeo_gfmap.manager|INFO: Downloaded asset openEO_2.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1561389/S2_at2021lpis1561389_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:15,519|openeo_gfmap.manager|DEBUG: Downloading asset openEO_3.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /tmp/tmpvahpjkqn\n", + "2024-02-12 09:38:22,615|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_3.nc from job j-24021279e49745e9a3806584fe0e8c9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:38:22,788|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_3.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1222569/S2_at2021lpis1222569_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:22,969|openeo_gfmap.manager|INFO: Downloaded asset openEO_3.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis1222569/S2_at2021lpis1222569_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:22,973|openeo_gfmap.manager|DEBUG: Downloading asset openEO_4.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /tmp/tmp9qq2f6m9\n", + "2024-02-12 09:38:29,854|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_4.nc from job j-24021279e49745e9a3806584fe0e8c9a...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 09:38:30,027|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_4.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis809720/S2_at2021lpis809720_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:30,148|openeo_gfmap.manager|INFO: Downloaded asset openEO_4.nc from job j-24021279e49745e9a3806584fe0e8c9a -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32633/33TXN/at2021lpis809720/S2_at2021lpis809720_32633_2020-08-30_2022-03-03.nc\n", + "2024-02-12 09:38:30,150|openeo_gfmap.manager|DEBUG: Calling post job action for job j-24021279e49745e9a3806584fe0e8c9a...\n", + "2024-02-12 09:38:31,352|openeo_gfmap.manager|INFO: Job j-24021279e49745e9a3806584fe0e8c9a and post job action finished successfully.\n", + "2024-02-12 09:39:29,274|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:39:29,492|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:39:30,816|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:40:30,968|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:40:33,929|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:40:34,269|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:41:34,440|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:41:34,695|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:41:35,280|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:42:35,462|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:42:36,041|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:42:36,485|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:43:36,626|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:43:36,946|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:43:38,638|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:44:38,795|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:44:39,048|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:44:41,675|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:45:41,861|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:45:42,175|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:45:44,476|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:46:44,671|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:46:44,946|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:46:47,272|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:47:47,411|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:47:47,719|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:47:48,098|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:48:48,231|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:48:48,650|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:48:50,196|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:49:50,520|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:49:50,827|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:49:52,613|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:50:52,803|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:50:54,202|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:50:54,794|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:51:54,958|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:51:56,063|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:51:56,289|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:52:56,486|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:52:56,866|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:52:57,592|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:53:57,737|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:53:58,210|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:54:01,194|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:55:01,369|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:55:05,199|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:55:05,437|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:56:05,605|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:56:06,025|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:56:08,011|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:57:08,220|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:57:08,685|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:57:12,111|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:58:12,249|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:58:12,658|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:58:14,530|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 09:59:14,656|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 09:59:14,954|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 09:59:15,242|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 10:00:15,353|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 10:00:15,947|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:00:29,066|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 10:01:29,230|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 10:01:29,528|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:01:30,633|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is running (on backend cdse).\n", + "2024-02-12 10:02:30,836|openeo_gfmap.manager|DEBUG: Updating status. 2 on 4 active jobs...\n", + "2024-02-12 10:02:32,780|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:02:46,801|openeo_gfmap.manager|DEBUG: Status of job j-2402124d03ab4e718bef6ce4e8185348 is finished (on backend cdse).\n", + "2024-02-12 10:02:46,803|openeo_gfmap.manager|INFO: Job j-2402124d03ab4e718bef6ce4e8185348 finished successfully, queueing on_job_done...\n", + "2024-02-12 10:02:46,809|openeo_gfmap.manager|DEBUG: Worker thread Thread-5: polled finished job with status PostJobStatus.FINISHED.\n", + "2024-02-12 10:02:51,114|openeo_gfmap.manager|DEBUG: Downloading asset openEO_0.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmp_gibcv9i\n", + "2024-02-12 10:02:58,022|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_0.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:02:58,509|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_0.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12461808/S2_2021_LV_LPIS_POLY_110-12461808_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:02:58,693|openeo_gfmap.manager|INFO: Downloaded asset openEO_0.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12461808/S2_2021_LV_LPIS_POLY_110-12461808_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:02:58,697|openeo_gfmap.manager|DEBUG: Downloading asset openEO_1.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmp710760kb\n", + "2024-02-12 10:03:05,950|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_1.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:06,320|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_1.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12484602/S2_2021_LV_LPIS_POLY_110-12484602_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:06,478|openeo_gfmap.manager|INFO: Downloaded asset openEO_1.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12484602/S2_2021_LV_LPIS_POLY_110-12484602_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:06,483|openeo_gfmap.manager|DEBUG: Downloading asset openEO_2.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmpx_uhw6jb\n", + "2024-02-12 10:03:13,646|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_2.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:14,059|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_2.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12710543/S2_2021_LV_LPIS_POLY_110-12710543_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:14,188|openeo_gfmap.manager|INFO: Downloaded asset openEO_2.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12710543/S2_2021_LV_LPIS_POLY_110-12710543_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:14,193|openeo_gfmap.manager|DEBUG: Downloading asset openEO_3.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmpnmy5v_v0\n", + "2024-02-12 10:03:18,924|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_3.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:19,242|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_3.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12668906/S2_2021_LV_LPIS_POLY_110-12668906_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:19,358|openeo_gfmap.manager|INFO: Downloaded asset openEO_3.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12668906/S2_2021_LV_LPIS_POLY_110-12668906_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:19,363|openeo_gfmap.manager|DEBUG: Downloading asset openEO_4.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmp2sgwj7iq\n", + "2024-02-12 10:03:24,044|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_4.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:24,459|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_4.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12842675/S2_2021_LV_LPIS_POLY_110-12842675_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:24,666|openeo_gfmap.manager|INFO: Downloaded asset openEO_4.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12842675/S2_2021_LV_LPIS_POLY_110-12842675_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:24,670|openeo_gfmap.manager|DEBUG: Downloading asset openEO_5.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmpt61g6d34\n", + "2024-02-12 10:03:30,997|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_5.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:31,312|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_5.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12579669/S2_2021_LV_LPIS_POLY_110-12579669_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:31,513|openeo_gfmap.manager|INFO: Downloaded asset openEO_5.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12579669/S2_2021_LV_LPIS_POLY_110-12579669_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:31,518|openeo_gfmap.manager|DEBUG: Downloading asset openEO_6.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmp9viznwne\n", + "2024-02-12 10:03:38,362|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_6.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:38,880|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_6.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12483998/S2_2021_LV_LPIS_POLY_110-12483998_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:38,989|openeo_gfmap.manager|INFO: Downloaded asset openEO_6.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12483998/S2_2021_LV_LPIS_POLY_110-12483998_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:38,993|openeo_gfmap.manager|DEBUG: Downloading asset openEO_7.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /tmp/tmpvp3hzxc5\n", + "2024-02-12 10:03:44,408|openeo_gfmap.manager|DEBUG: Generating output path for asset openEO_7.nc from job j-2402124d03ab4e718bef6ce4e8185348...\n", + "/tmp/ipykernel_23450/1596858655.py:32: UserWarning: Geometry is in a geographic CRS. Results from 'distance' are likely incorrect. Use 'GeoSeries.to_crs()' to re-project geometries to a projected CRS before this operation.\n", + "\n", + " intersecting_cells['distance'] = intersecting_cells.distance(centroid_pt)\n", + "/home/couchard/miniconda3/envs/gfmap/lib/python3.9/site-packages/geopandas/geodataframe.py:1543: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame.\n", + "Try using .loc[row_indexer,col_indexer] = value instead\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " super().__setitem__(key, value)\n", + "/tmp/ipykernel_23450/1596858655.py:33: SettingWithCopyWarning: \n", + "A value is trying to be set on a copy of a slice from a DataFrame\n", + "\n", + "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", + " intersecting_cells.sort_values('distance', inplace=True)\n", + "2024-02-12 10:03:44,747|openeo_gfmap.manager|DEBUG: Generated path for asset openEO_7.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12308263/S2_2021_LV_LPIS_POLY_110-12308263_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:44,946|openeo_gfmap.manager|INFO: Downloaded asset openEO_7.nc from job j-2402124d03ab4e718bef6ce4e8185348 -> /data/users/Public/couchard/world_cereal/extractions/2021_EUR_DEMO_POLY_110/32634/34VEH/2021_LV_LPIS_POLY_110-12308263/S2_2021_LV_LPIS_POLY_110-12308263_32634_2020-08-30_2022-03-03.nc\n", + "2024-02-12 10:03:44,948|openeo_gfmap.manager|DEBUG: Calling post job action for job j-2402124d03ab4e718bef6ce4e8185348...\n", + "2024-02-12 10:03:46,982|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:03:47,304|openeo_gfmap.manager|INFO: Job j-2402124d03ab4e718bef6ce4e8185348 and post job action finished successfully.\n", + "2024-02-12 10:03:47,374|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:04:47,471|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:04:48,400|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:05:48,513|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:05:48,775|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:06:48,987|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:06:49,580|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:07:49,782|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:07:50,229|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:08:50,385|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:08:50,823|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:09:50,944|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:09:51,455|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:10:51,598|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:11:17,145|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:12:17,315|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:12:17,688|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:13:17,805|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:13:18,499|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:14:18,608|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:14:18,986|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:15:19,116|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:15:19,564|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:16:19,716|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:16:19,975|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:17:20,142|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:17:20,525|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:18:20,697|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:18:20,959|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:19:21,184|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:19:21,544|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:20:21,695|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:20:26,767|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:21:26,916|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:21:27,696|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:22:27,831|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:22:28,087|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:23:28,237|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:23:28,481|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:24:28,627|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:24:29,027|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:25:29,137|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:25:29,855|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:26:30,003|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:26:31,146|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:27:31,306|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:27:31,531|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:28:31,696|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:28:32,167|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:29:32,318|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:29:32,643|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:30:32,852|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:30:33,161|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:31:33,313|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:31:34,064|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:32:34,293|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:32:34,630|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:33:34,773|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:33:34,997|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:34:35,180|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:34:35,697|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is running (on backend cdse).\n", + "2024-02-12 10:35:35,827|openeo_gfmap.manager|DEBUG: Updating status. 1 on 4 active jobs...\n", + "2024-02-12 10:35:36,174|openeo_gfmap.manager|DEBUG: Status of job j-2402128d377942dfa41b8b075936fa95 is error (on backend cdse).\n", + "2024-02-12 10:35:36,176|openeo_gfmap.manager|INFO: Job j-2402128d377942dfa41b8b075936fa95 finished with error, queueing on_job_error...\n", + "2024-02-12 10:35:36,179|openeo_gfmap.manager|DEBUG: Worker thread Thread-4: polled finished job with status PostJobStatus.ERROR.\n", + "Exception in thread Thread-4:\n", + "Traceback (most recent call last):\n", + " File \"/home/couchard/miniconda3/envs/gfmap/lib/python3.9/threading.py\", line 980, in _bootstrap_inner\n", + " self.run()\n", + " File \"/home/couchard/miniconda3/envs/gfmap/lib/python3.9/threading.py\", line 917, in run\n", + " self._target(*self._args, **self._kwargs)\n", + " File \"/data/users/Private/couchard/openeo-gfmap/src/openeo_gfmap/manager/job_manager.py\", line 64, in _post_job_worker\n", + " if status == PostJobStatus.ERROR:\n", + " File \"/data/users/Private/couchard/openeo-gfmap/src/openeo_gfmap/manager/job_manager.py\", line 137, in on_job_error\n", + " output_log_path = (\n", + "TypeError: '>' not supported between instances of 'list' and 'int'\n" + ] + } + ], + "source": [ + "manager.run_jobs(job_df, create_datacube_s2, tracking_job_csv)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "gfmap", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}