Access from python script

.github/workflows/ci.yml

@@ -52,6 +52,7 @@ jobs:
           pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=ldndctools tests/ | tee pytest-coverage.txt
 
     - name: Pytest coverage comment
+      continue-on-error: true
       uses: MishaKav/pytest-coverage-comment@main
       if: ${{ matrix.python-version == '3.10' }}
       with:

data/catalog.yml

@@ -21,7 +21,8 @@ sources:
           anon: true
           default_fill_cache: false
           client_kwargs:
-            endpoint_url: 'https://s3.imk-ifu.kit.edu:8082'
+            endpoint_url: 'https://s3.imk-ifu.kit.edu:10443'
+            verify: False
         simplecache:
           cache_storage: '.cache'
           same_names: true
@@ -63,7 +64,7 @@ sources:
           anon: true
           default_fill_cache: false
           client_kwargs:
-            endpoint_url: 'https://s3.imk-ifu.kit.edu:8082'
+            endpoint_url: 'https://s3.imk-ifu.kit.edu:10443'
         simplecache:
           cache_storage: '.cache'
           same_names: true
@@ -73,12 +74,12 @@ sources:
     driver: zarr
     args:
       urlpath: 's3://era5land-zarr/data.zarr'
-      consolidated: True
+      consolidated: False
       storage_options:
-        use_ssl: False
+        use_ssl: True
         anon: True
         client_kwargs:
-          endpoint_url: 'https://s3.imk-ifu.kit.edu:8082'
+          endpoint_url: 'https://s3.imk-ifu.kit.edu:10443'
           verify: False
 
 
@@ -100,7 +101,7 @@ sources:
           anon: true
           default_fill_cache: false
           client_kwargs:
-            endpoint_url: 'https://s3.imk-ifu.kit.edu:8082'
+            endpoint_url: 'https://s3.imk-ifu.kit.edu:10443'
         simplecache:
           cache_storage: '.cache'
           same_names: true

ldndctools/cdgen.py

@@ -161,7 +161,7 @@ def inner_func(x, lat, lon):
 def geohash_xr(mask: xr.DataArray) -> xr.DataArray:
     lon_xr = mask.lon.broadcast_like(mask)
     lat_xr = mask.lat.broadcast_like(mask)
-    data = xr.apply_ufunc(inner_func, mask, lat_xr, lon_xr, output_dtypes=[np.int64])
+    data = xr.apply_ufunc(inner_func, mask, lat_xr, lon_xr, output_dtypes=[np.int64], dask="allowed")
     assert data.dtype == np.int64
     return data
 

ldndctools/cli/selector.py

@@ -220,14 +220,18 @@ def ask(self):
 
 
 class CoordinateSelection:
-    def __init__(self, infile, lon_col="lon", lat_col="lat", id_col="ID"):
-        df = pd.read_csv(infile, delim_whitespace=True)
-
-        self.lons = df[lon_col].values
-        self.lats = df[lat_col].values
-        self.ids = (
-            df[id_col].values if id_col in list(df.columns) else range(len(self.lats))
-        )
+
+    def __init__(self, infile=None, lon="lon", lat="lat", cid="ID"):
+        if infile:
+            df = pd.read_csv(infile, delim_whitespace=True)
+
+            self.lons = df[lon].values
+            self.lats = df[lat].values
+            self.ids  = df[cid].values if cid in list(df.columns) else range(len(self.lats))
+        else:
+            self.lons = [lon]
+            self.lats = [lat]
+            self.ids  = [cid]
 
     @property
     def selected(self):

ldndctools/dlsc.py

@@ -30,8 +30,7 @@
 from ldndctools.sources.soil.soil_iscricwise import ISRICWISE_SoilDataset
 
 log = logging.getLogger(__name__)
-
-log.setLevel("DEBUG")
+log.setLevel("INFO")
 
 NODATA = "-99.99"
 
@@ -41,12 +40,17 @@
     DPATH = Path(dpath)
 
 
-def main():
+def main( **kwargs):
     # parse args
     args = cli()
 
     # read config
-    cfg = get_config(args.config)
+    if 'config' in kwargs:
+        cfg = kwargs['config']
+        for k,v in cfg.items():
+            setattr(args, k, v)
+    else:
+        cfg = get_config(args.config)
 
     # write config
     if args.storeconfig:
@@ -80,14 +84,21 @@ def main():
     res = RES[args.resolution]
 
     bbox = None
+
+    if ('lat' in args) and ('lon' in args):
+       setattr(args, 'bbox', [float(args.lon-0.5), float(args.lat-0.5), float(args.lon+0.5), float(args.lat+0.5)]) 
+       log.info("Creating bounding box for coordinates specified.")
     if args.bbox:
-        x1, y1, x2, y2 = [float(x) for x in args.bbox.split(",")]
+        if type( args.bbox) == list:
+            x1, y1, x2, y2 = args.bbox
+        else: 
+            x1, y1, x2, y2 = [float(x) for x in args.bbox.split(",")]
         try:
             bbox = BoundingBox(x1=x1, y1=y1, x2=x2, y2=y2)
         except ValidationError:
-            print("Ilegal bounding box coordinates specified.")
-            print("required: x1,y1,x2,y2 [cond: x1<x2, y1<y2]")
-            print(f"given:    x1={x1},y1={y1},x2={x2},y2={y2}")
+            log.info("Ilegal bounding box coordinates specified.")
+            log.info("required: x1,y1,x2,y2 [cond: x1<x2, y1<y2]")
+            log.info(f"given:    x1={x1},y1={y1},x2={x2},y2={y2}")
             exit(1)
 
     if not args.outfile:
@@ -144,7 +155,7 @@ def main():
     log.info(selector.selected)
 
     with tqdm(total=1) as progressbar:
-        xml, nc = create_dataset(soil, selector, res, progressbar)
+        xml, nc = create_dataset(soil, selector, res, args, progressbar)
 
     open(cfg["outname"], "w").write(xml)
     ENCODING = {

ldndctools/extra.py

@@ -91,7 +91,7 @@ def cfgfile_exists(cfgFile):
         cfgFile = _find_config()
 
         if not cfgfile_exists(cfgFile):
-            log.info("Copying config file")
+            log.info("Copying default config file")
             _copy_default_config()
             cfgFile = _find_config()
 

ldndctools/io/xmlwriter.py

@@ -71,9 +71,9 @@ def write(
         if status_widget:
             status_widget.warning("Preparing data")
 
-        if not all(v is None for v in [sample, id_selection, coords]):
-            # options currently not implemented
-            raise NotImplementedError
+        #if not all(v is None for v in [sample, id_selection, coords]):
+        #    # options currently not implemented
+        #    raise NotImplementedError
 
         ids: xr.DataArray = xr.zeros_like(self.mask, dtype=np.int32)
 
@@ -91,7 +91,10 @@ def write(
                         lat=self.soil.coords["lat"][j].values.item(),
                         lon=self.soil.coords["lon"][i].values.item(),
                     )
-                # print(cid)
+                if id_selection:
+                    if cid not in id_selection:
+                        continue
+
                 ids[j, i] = cid
                 Lcids.append(cid)
                 Lix.append(i)

ldndctools/misc/create_data.py

@@ -1,29 +1,35 @@
 from typing import Any, Optional, Union
 
+import logging
 import numpy as np
 import rioxarray  # noqa
 import xarray as xr
 
+from ldndctools.misc.geohash import coords2geohash_dec
 from ldndctools.cli.selector import CoordinateSelection, Selector
 from ldndctools.io.xmlwriter import SiteXmlWriter
 from ldndctools.misc.types import RES
 from ldndctools.sources.soil.soil_base import SoilDataset
 
+log = logging.getLogger(__name__)
+
+log.setLevel("DEBUG")
 
 def create_dataset(
     soil: SoilDataset,
     selector: Union[Selector, CoordinateSelection],
     res: RES,
+    args,
     progressbar: Optional[Any] = None,
     status_widget: Optional[Any] = None,
-):
+    ):
     # soil = soil.load()
     # soil = soil.rio.write_crs(4326)
 
     if isinstance(selector, Selector):
-        print("Using Selector")
+        log.info("Using Selector")
         if selector.gdf_mask is None:
-            print("No valid data to process for this region/ bbox request.")
+            log.info("No valid data to process for this region/ bbox request.")
             exit(1)
 
         # clip region selection
@@ -36,7 +42,12 @@ def create_dataset(
         soil.clip_mask(selector.gdf_mask.geometry, all_touched=True)
 
         xmlwriter = SiteXmlWriter(soil, res=res)
-        site_xml = xmlwriter.write(progressbar=progressbar, status_widget=status_widget)
+        if ('lat' in args) and ('lon' in args):
+            sel = soil._soil.sel(lat=args.lat, lon=args.lon, method='nearest')
+            cid = coords2geohash_dec( lat=sel["lat"].values.item(), lon=sel["lon"].values.item())
+            site_xml = xmlwriter.write(progressbar=progressbar, status_widget=status_widget, id_selection=[cid])
+        else:
+            site_xml = xmlwriter.write(progressbar=progressbar, status_widget=status_widget)
 
     else:
         # WARNING: THIS BRANCH IS DEFUNCT!!!

ldndctools/misc/types.py

@@ -2,7 +2,7 @@
 from enum import Enum
 from typing import Optional
 
-from pydantic import BaseModel, confloat, conint, root_validator, ValidationError
+from pydantic import BaseModel, confloat, conint, model_validator, ValidationError
 from pydantic.dataclasses import dataclass
 from pydantic.json import pydantic_encoder
 
@@ -40,17 +40,15 @@ class BoundingBox:
     y1: confloat(ge=-90, le=90) = -90
     y2: confloat(ge=-90, le=90) = 90
 
-    @root_validator(allow_reuse=True)
+    @model_validator(mode='after')
     def check_x1_smaller_x2(cls, values):
-        x1, x2 = values.get("x1"), values.get("x2")
-        if x1 >= x2:
+        if values.x1 >= values.x2:
             raise ValidationError("x1 must be smaller x2")
         return values
 
-    @root_validator(allow_reuse=True)
+    @model_validator(mode='after')
     def check_y1_smaller_y2(cls, values):
-        y1, y2 = values.get("y1"), values.get("y2")
-        if y1 >= y2:
+        if values.y1 >= values.y2:
             raise ValidationError("y1 must be smaller y2")
         return values
 
@@ -97,15 +95,15 @@ class LayerData(BaseModel):
     class Config:
         validate_assignment = True
 
-    # @root_validator
+    # @model_validator
     # def check_wcmin_smaller_wcmax(cls, values):
     #     wcmin, wcmax = values.get("wcmin"), values.get("wcmax")
     #     if None not in [wcmin, wcmax]:
     #         if wcmin >= wcmax:
     #             raise ValidationError("wcmin must be smaller wcmax")
     #     return values
 
-    @root_validator(allow_reuse=True)
+    #@model_validator(mode='after')
     def check_texture_is_plausible(cls, values):
         sand, silt, clay = values.get("sand"), values.get("silt"), values.get("clay")
         args = [a for a in [sand, silt, clay] if a is not None]

requirements.txt

@@ -2,10 +2,10 @@ aiohttp >= 3.8.1
 boto3 >= 1.20.49
 dask[distributed] >= 2022.2.0
 geopandas >= 0.10.2
-git+https://github.com/rasterio/rasterio.git
+rasterio
 #cython >= 0.29.22
 #cytoolz >= 0.11.2
-git+https://github.com/Toblerity/Fiona.git  # required for 3.10 compat
+fiona
 intake >= 0.6.5
 intake-geopandas >= 0.4.0
 intake-xarray >= 0.6.0