Merge pull request #84 from stac-utils/swap_recursion

Swap out pystac recursion, lint recursively
stac-utils · Jul 31, 2022 · 0c0e9b3 · 0c0e9b3
2 parents 51d5add + 3b0ae30
commit 0c0e9b3
Show file tree

Hide file tree

Showing 7 changed files with 228 additions and 53 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ The format is (loosely) based on [Keep a Changelog](http://keepachangelog.com/)
 
 
 ## [Unreleased]
+### Added
+- recursive mode lints assets https://github.com/stac-utils/stac-check/pull/84
+### Changed
+- recursive mode swaps pystac for stac-validator https://github.com/stac-utils/stac-check/pull/84
 ### Fixed
 - fix catalog file name check https://github.com/stac-utils/stac-check/pull/83
 

diff --git a/README.md b/README.md
@@ -17,11 +17,14 @@ or for local development
 Usage: stac-check [OPTIONS] FILE
 
 Options:
-  --version        Show the version and exit.
-  -l, --links      Validate links for format and response.
-  -a, --assets     Validate assets for format and response.
-  -r, --recursive  Validate all assets in a collection or catalog.
-  --help           Show this message and exit.
+  --version                Show the version and exit.
+  -l, --links              Validate links for format and response.
+  -a, --assets             Validate assets for format and response.
+  -m, --max-depth INTEGER  Maximum depth to traverse when recursing. Omit this
+                           argument to get full recursion. Ignored if
+                           `recursive == False`.
+  -r, --recursive          Recursively validate all related stac objects.
+  --help                   Show this message and exit.               Show this message and exit.
 ```
 ---
 ### Docker  
@@ -44,8 +47,13 @@ stac-check: STAC spec validaton and linting tool
 
 Please upgrade from version 0.9.0 to version 1.0.0!
 
-Validator: pystac 1.1.0
-    Recursive: Validate all assets in a collection or catalog
+Validator: stac-validator 3.1.0
+
+
+Recursive: Validate all assets in a collection or catalog
+Max-depth = None
+-------------------------
+Asset 1 Validated: https://raw.githubusercontent.com/stac-utils/pystac/main/tests/data-files/examples/0.9.0/collection-spec/examples/landsat-collection.json
 
 Valid COLLECTION: True
 
@@ -60,12 +68,17 @@ STAC Best Practices:
 
     Links in catalogs and collections should always have a 'title' field
 
+This object has 4 links
 
-Recursive validation has failed!
-Recursive validation error message: 
-    Exception Could not read uri https://landsat-stac.s3.amazonaws.com/landsat-8-l1/paths/catalog.json
+-------------------------
+Asset 2 Validated: https://landsat-stac.s3.amazonaws.com/landsat-8-l1/paths/catalog.json
 
-This object has 4 links
+Valid: False
+Schemas validated: 
+    https://cdn.staclint.com/v0.9.0/collection.json
+Error Type: JSONDecodeError
+Error Message: Expecting value: line 1 column 1 (char 0)
+-------------------------
 ```
 
 ``` stac-check sample_files/0.9.0/landsat8-sample.json```

diff --git a/sample_files/1.0.0/catalog-with-bad-item.json b/sample_files/1.0.0/catalog-with-bad-item.json
@@ -0,0 +1,31 @@
+{
+  "id": "examples",
+  "type": "Catalog",
+  "title": "Example Catalog",
+  "stac_version": "1.0.0",
+  "description": "This is a valid catalog with links to one valid item and one invalid item.",
+  "links": [
+    {
+      "rel": "root",
+      "href": "./catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "item",
+      "href": "./bad-item.json",
+      "type": "application/json",
+      "title": "Invalid item"
+    },
+    {
+      "rel": "item",
+      "href": "./collectionless-item.json",
+      "type": "application/json",
+      "title": "Collection with no items (standalone)"
+    },
+    {
+      "rel": "self",
+      "href": "https://raw.githubusercontent.com/radiantearth/stac-spec/v1.0.0/examples/catalog.json",
+      "type": "application/json"
+    }
+  ]
+}
diff --git a/setup.py b/setup.py
@@ -15,14 +15,14 @@
     packages=find_packages(exclude=("tests",)),
     include_package_data=True,
     install_requires=[
-        "pystac[validation]>=1.1.0",
-        "click>=7.1.2",
+        "click>=8.0.0",
         "requests>=2.19.1",
         "jsonschema>=3.1.2b0",
         "pytest",
         "stac-validator>=3.0.0",
         "PyYAML",
         "python-dotenv",
+        "types-setuptools",
     ],
     entry_points={
         'console_scripts': ['stac-check=stac_check.cli:main']

diff --git a/stac_check/cli.py b/stac_check/cli.py
@@ -1,5 +1,6 @@
 import click
 from .lint import Linter
+import pkg_resources
 
 def link_asset_message(link_list:list, type: str, format: str):
     if len(link_list) > 0:
@@ -9,7 +10,27 @@ def link_asset_message(link_list:list, type: str, format: str):
     else:
         click.secho(f"No {type.upper()} {format} errors!", fg="green")
 
-def cli_message(linter):
+def recursive_message(linter):
+    click.secho()
+    click.secho(f"Recursive: Validate all assets in a collection or catalog", bold=True)
+    click.secho(f"Max-depth = {linter.max_depth}")
+    click.secho("-------------------------")
+    for count, msg in enumerate(linter.validate_all):
+        click.secho(f"Asset {count+1} Validated: {msg['path']}", bg="white", fg="black")
+        click.secho()
+        if msg['valid_stac'] == True:
+            recursive_linter = Linter(msg["path"], recursive=0)
+            cli_message(recursive_linter)
+        else:
+            click.secho(f"Valid: {msg['valid_stac']}", fg='red')
+            click.secho("Schemas validated: ", fg="blue")
+            for schema in msg["schema"]:
+                click.secho(f"    {schema}")
+            click.secho(f"Error Type: {msg['error_type']}", fg='red')
+            click.secho(f"Error Message: {msg['error_message']}", fg='red')
+        click.secho("-------------------------")
+
+def intro_message(linter):
     click.secho("""
  ____  ____  __    ___       ___  _  _  ____  ___  __ _ 
 / ___)(_  _)/ _\  / __)___  / __)/ )( \(  __)/ __)(  / )
@@ -28,15 +49,11 @@ def cli_message(linter):
 
     click.secho()
 
-    ''' validator used - pystac if recursive otherwise stac-validator '''
-    if linter.recursive == True:
-        click.secho(f"Validator: pystac 1.1.0", bg="blue", fg="white")
-        click.secho(f"    Recursive: Validate all assets in a collection or catalog")
-    else:
-        click.secho(f"Validator: stac-validator {linter.validator_version}", bg="blue", fg="white")
+    click.secho(f"Validator: stac-validator {linter.validator_version}", bg="blue", fg="white")
 
     click.secho()
-
+
+def cli_message(linter):
     ''' valid stac object message - true or false '''
     if linter.valid_stac == True:
         click.secho(f"Valid {linter.asset_type}: {linter.valid_stac}", fg='green')
@@ -61,7 +78,7 @@ def cli_message(linter):
     if linter.validate_all == True:
         click.secho()
         click.secho(f"Recursive validation has passed!", fg='blue')
-    elif linter.validate_all == False and linter.recursive == True:
+    elif linter.validate_all == False and linter.recursive:
         click.secho()
         click.secho(f"Recursive validation has failed!", fg='red')
 
@@ -89,12 +106,6 @@ def cli_message(linter):
         click.secho(f"Validation error message: ", fg='red')
         click.secho(f"    {linter.error_msg}")
 
-    if linter.recursive_error_msg != "":
-        click.secho(f"Recursive validation error message: ", fg='red')
-        click.secho(f"    {linter.recursive_error_msg}")
-
-    click.secho()
-
     click.secho(f"This object has {len(linter.data['links'])} links")
 
     click.secho()
@@ -103,7 +114,16 @@ def cli_message(linter):
     # click.secho(json.dumps(linter.message, indent=4))
 
 @click.option(
-    "-r", "--recursive", is_flag=True, help="Validate all assets in a collection or catalog."
+    "--recursive",
+    "-r",
+    is_flag=True,
+    help="Recursively validate all related stac objects.",
+)
+@click.option(
+    "--max-depth",
+    "-m",
+    type=int,
+    help="Maximum depth to traverse when recursing. Omit this argument to get full recursion. Ignored if `recursive == False`.",
 )
 @click.option(
     "-a", "--assets", is_flag=True, help="Validate assets for format and response."
@@ -113,7 +133,11 @@ def cli_message(linter):
 )
 @click.command()
 @click.argument('file')
-@click.version_option(version="1.2.0")
-def main(file, assets, links, recursive):
-    linter = Linter(file, assets, links, recursive)
-    cli_message(linter)
+@click.version_option(version=pkg_resources.require("stac-check")[0].version)
+def main(file, recursive, max_depth, assets, links):
+    linter = Linter(file, assets=assets, links=links, recursive=recursive, max_depth=max_depth)
+    intro_message(linter)
+    if recursive > 0:
+        recursive_message(linter)
+    else:
+        cli_message(linter)
diff --git a/stac_check/lint.py b/stac_check/lint.py
@@ -5,10 +5,10 @@
 import yaml
 import os
 from dataclasses import dataclass
-import pystac
 import requests
 from typing import Optional
 from dotenv import load_dotenv
+import pkg_resources
 
 load_dotenv()
 
@@ -19,14 +19,16 @@ class Linter:
     assets: bool = False
     links: bool = False
     recursive: bool = False
+    max_depth: Optional[int] = None
 
     def __post_init__(self):
         self.data = self.load_data(self.item)
         self.message = self.validate_file(self.item)
         self.config = self.parse_config(self.config_file)
         self.asset_type = self.message["asset_type"] if "asset_type" in self.message else ""
         self.version = self.message["version"] if "version" in self.message else ""
-        self.validator_version = "2.3.0"
+        self.validator_version = pkg_resources.require("stac-validator")[0].version
+        self.validate_all = self.recursive_validation(self.item)
         self.valid_stac = self.message["valid_stac"]
         self.error_type = self.check_error_type()
         self.error_msg = self.check_error_message()
@@ -35,8 +37,6 @@ def __post_init__(self):
         self.invalid_link_format = self.check_links_assets(10, "links", "format") if self.links else None
         self.invalid_link_request = self.check_links_assets(10, "links", "request") if self.links else None
         self.schema = self.message["schema"] if "schema" in self.message else []
-        self.recursive_error_msg = ""
-        self.validate_all = self.recursive_validation(self.load_data(self.item))
         self.object_id = self.data["id"] if "id" in self.data else ""
         self.file_name = os.path.basename(self.item).split('.')[0]
         self.best_practices_msg = self.create_best_practices_msg()
@@ -73,13 +73,9 @@ def validate_file(self, file):
 
     def recursive_validation(self, file):
         if self.recursive:
-            try:
-                catalog = pystac.read_dict(file)
-                catalog.validate_all()
-                return True
-            except Exception as e:
-                self.recursive_error_msg = f"Exception {str(e)}"
-                return False
+            stac = StacValidate(file, recursive=True, max_depth=self.max_depth)
+            stac.run()
+            return stac.message
 
     def set_update_message(self):
         if self.version != "1.0.0":