Initial Push

mypy.ini

@@ -2,3 +2,6 @@
 mypy_path = src
 explicit_package_bases = True
 namespace_packages = True
+
+[mypy-netCDF4.*]
+ignore_missing_imports = True

scripts/lint

@@ -13,7 +13,7 @@ Execute project linters.
 "
 }
 
-EC_EXCLUDE="(__pycache__|.git|.coverage|coverage.xml|.*\.egg-info|.mypy_cache|.tif|.tiff|.npy)"
+EC_EXCLUDE="(__pycache__|.git|.coverage|coverage.xml|.*\.egg-info|.mypy_cache|.tif|.tiff|*.nc)"
 
 DIRS_TO_CHECK=("src" "tests" "scripts")
 

setup.cfg

@@ -28,6 +28,7 @@ package_dir =
 packages = find_namespace:
 install_requires =
     stactools == 0.2.1
+    netCDF4 ~= 1.5.7
 
 [options.packages.find]
 where = src

src/stactools/noaa_sst/cog.py

@@ -0,0 +1,30 @@
+import os.path
+import urllib.request
+import urllib.error
+from stactools.core.utils.convert import cogify
+
+
+def retrieve_nc(url):
+    fileName = url.split('/')[-1]
+    try:
+        urllib.request.urlretrieve(url, fileName)
+    except (urllib.error.URLError, urllib.error.HTTPError,
+            urllib.error.ContentTooShortError):
+        print("File could not be downloaded")
+        return (-1)
+    return (fileName)
+
+
+def create_cog(nc_href: str, cog_href: str) -> None:
+    sst_str = "analysed_sst"
+    sif_str = "sea_ice_fraction"
+    sst_output = os.path.join(
+        os.path.split(cog_href)[0], ('sst_' + os.path.split(cog_href)[-1]))
+    sif_output = os.path.join(
+        os.path.split(cog_href)[0], ('sif_' + os.path.split(cog_href)[-1]))
+    cogify(f'NETCDF:"{nc_href}":{sst_str}', sst_output,
+           ["-co", "compress=LZW"])
+    cogify(f'NETCDF:"{nc_href}":{sif_str}', sif_output,
+           ["-co", "compress=LZW"])
+
+    print('Done')

src/stactools/noaa_sst/commands.py

@@ -2,6 +2,7 @@
 import logging
 
 from stactools.noaa_sst import stac
+from stactools.noaa_sst import cog
 
 logger = logging.getLogger(__name__)
 
@@ -15,39 +16,67 @@ def create_noaasst_command(cli):
     def noaasst():
         pass
 
+    @noaasst.command(
+        "retrieve-nc",
+        short_help="Downloads a NOAA CoralTemp netcdf file",
+    )
+    @click.argument("source")
+    def retrieve_nc_command(source: str) -> None:
+        """Downloads a NOAA CoralTemp netcdf file given its FTP address
+        Args:
+            source (str): The ftp address of the netcdf
+        """
+        cog.retrieve_nc(source)
+
+    @noaasst.command(
+        "create-cog",
+        short_help="Creates 2 COGs from a NOAA CoralTemp netcdf file",
+    )
+    @click.argument("source")
+    @click.argument("destination")
+    def create_cog_command(source: str, destination: str) -> None:
+        """Creates 2 Cogs
+        Args:
+            source (str): An HREF for the NOAA CoralTemp netcdf file
+            destination (str): An HREF for the Collection JSON
+        """
+        cog.create_cog(source, destination)
+
     @noaasst.command(
         "create-collection",
         short_help="Creates a STAC collection",
     )
     @click.argument("destination")
     def create_collection_command(destination: str):
         """Creates a STAC Collection
-
         Args:
             destination (str): An HREF for the Collection JSON
         """
         collection = stac.create_collection()
 
         collection.set_self_href(destination)
+        collection.validate()
 
         collection.save_object()
 
         return None
 
     @noaasst.command("create-item", short_help="Create a STAC item")
-    @click.argument("source")
+    @click.argument("nc_href")
+    @click.argument("sst_cog_href")
+    @click.argument("sif_cog_href")
     @click.argument("destination")
-    def create_item_command(source: str, destination: str):
+    def create_item_command(nc_href: str, sst_cog_href: str, sif_cog_href: str,
+                            destination: str) -> None:
         """Creates a STAC Item
-
         Args:
-            source (str): HREF of the Asset associated with the Item
+            nc_href (str): HREF of the netcdf associated with the Item
+            sst_cog_href (str): An HREF for the associated sea surface temp COG asset
+            sif_cog_href (str): An HREF for the associated sea ice fraction COG asset
             destination (str): An HREF for the STAC Collection
         """
-        item = stac.create_item(source)
-
+        item = stac.create_item(nc_href, sst_cog_href, sif_cog_href)
+        item.validate()
         item.save_object(dest_href=destination)
 
-        return None
-
     return noaasst

src/stactools/noaa_sst/constants.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from pyproj import CRS
+from pystac import Provider, ProviderRole, Link
+
+NOAA_SST_ID = "noaa-sst"
+SST_EPSG = 4326
+SST_CRS = CRS.from_epsg(SST_EPSG)
+LICENSE = "proprietary"
+lic_link = """"https://github.com/stactools-packages/noaa-sst/tree/main/src/stactools/noaa_sst/license
+/data-license.txt"""
+LICENSE_LINK = Link(rel="license",
+                    target=lic_link,
+                    title="Public Domain License - NOAA")
+SPATIAL_EXTENT = [-180.0, -90.0, 180.0, 90.0]
+TEMPORAL_EXTENT = [
+    datetime(1985, 1, 1),
+    None,
+]
+DESCRIPTION = """The NOAA Coral Reef Watch (CRW) daily global 5km Sea Surface Temperature (SST)
+product, also known as CoralTemp, shows the nighttime ocean temperature measured at the surface.
+The SST scale ranges from -2 to 35 °C. The product is updated each afternoon at about
+12:00pm U.S. Eastern Time."""
+TITLE = 'NOAA 5km Sea Surface Temperature (SST)'
+SST_PROVIDER = Provider(
+    name="NOAA Coral Reef Watch Program",
+    roles=[ProviderRole.PRODUCER, ProviderRole.PROCESSOR, ProviderRole.HOST],
+    url="https://coralreefwatch.noaa.gov/")

src/stactools/noaa_sst/license/data-license.txt

@@ -0,0 +1 @@
+OSTIA Usage Statement (1985-2002): IMPORTANT usage statement. Unless otherwise agreed in writing, these data may be used for pure academic research only, with no commercial or other application and all usage must meet the Met Office Standard Terms and Conditions, which may be found here: http://www.metoffice.gov.uk/corporate/legal/tandc.html. The data may be used for a maximum period of 5 years. Reproduction of the data is permitted provided the following copyright statement is included: (C) Crown Copyright 2010, published by the Met Office. You must submit a completed reproduction license application form (here http://www.metoffice.gov.uk/corporate/legal/repro_licence.html) before using the data. This only needs to be completed once for each user. WARNING Some applications are unable to properly handle signed byte values. If values are encountered > 127, please subtract 256 from this reported value. GHRSST statement (2002-present): GHRSST protocol describes data use as free and open. Coral Reef Watch program statement: The data produced by Coral Reef Watch are available for use without restriction, but Coral Reef Watch relies on the ethics and integrity of the user to ensure that the source of the data and products is appropriately cited and credited. When using these data and products, credit and courtesy should be given to NOAA Coral Reef Watch. Please include the appropriate DOI associated with this dataset in the citation. For more information, visit the NOAA Coral Reef Watch website: https://coralreefwatch.noaa.gov. Recommendations for citing and providing credit are provided at https://coralreefwatch.noaa.gov/satellite/docs/recommendations_crw_citation.php. Users are referred to the footer section of Coral Reef Watch's website (https://coralreefwatch.noaa.gov/index.php) for disclaimers, policies, notices pertaining to the use of the data.

src/stactools/noaa_sst/stac.py

@@ -1,19 +1,31 @@
-from datetime import datetime, timezone
+from datetime import datetime
 import logging
+from pystac.extensions.item_assets import AssetDefinition, ItemAssetsExtension
+from pystac.extensions.projection import ProjectionExtension
 
 from pystac import (
     Collection,
     Item,
     Asset,
-    Provider,
-    ProviderRole,
     Extent,
     SpatialExtent,
     TemporalExtent,
     CatalogType,
     MediaType,
 )
-from pystac.extensions.projection import ProjectionExtension
+
+from stactools.noaa_sst.constants import (
+    NOAA_SST_ID,
+    SPATIAL_EXTENT,
+    TEMPORAL_EXTENT,
+    SST_PROVIDER,
+    TITLE,
+    DESCRIPTION,
+    LICENSE,
+    LICENSE_LINK,
+    SST_EPSG,
+)
+from netCDF4 import Dataset
 
 logger = logging.getLogger(__name__)
 
@@ -30,93 +42,115 @@ def create_collection() -> Collection:
     Returns:
         Collection: STAC Collection object
     """
-    providers = [
-        Provider(
-            name="The OS Community",
-            roles=[
-                ProviderRole.PRODUCER, ProviderRole.PROCESSOR,
-                ProviderRole.HOST
-            ],
-            url="https://github.com/stac-utils/stactools",
-        )
-    ]
-
-    # Time must be in UTC
-    demo_time = datetime.now(tz=timezone.utc)
-
     extent = Extent(
-        SpatialExtent([[-180., 90., 180., -90.]]),
-        TemporalExtent([demo_time, None]),
+        SpatialExtent([SPATIAL_EXTENT]),
+        TemporalExtent(TEMPORAL_EXTENT),
     )
 
     collection = Collection(
-        id="my-collection-id",
-        title="A dummy STAC Collection",
-        description="Used for demonstration purposes",
-        license="CC-0",
-        providers=providers,
+        id=NOAA_SST_ID,
+        title=TITLE,
+        description=DESCRIPTION,
+        license=LICENSE,
+        providers=[SST_PROVIDER],
         extent=extent,
         catalog_type=CatalogType.RELATIVE_PUBLISHED,
     )
 
-    return collection
-
+    item_assets = ItemAssetsExtension.ext(collection, add_if_missing=True)
+
+    item_assets.item_assets = {
+        "sst_cog":
+        AssetDefinition({
+            "type":
+            MediaType.COG,
+            "roles": ["data"],
+            "description":
+            "COG containing 5km Sea Surface Temperature information"
+        }),
+        "sif_cog":
+        AssetDefinition({
+            "type":
+            MediaType.COG,
+            "roles": ["data"],
+            "description":
+            "COG containing 5km Sea Ice Fraction information"
+        })
+    }
 
-def create_item(asset_href: str) -> Item:
-    """Create a STAC Item
+    collection.add_link(LICENSE_LINK)
 
-    This function should include logic to extract all relevant metadata from an
-    asset, metadata asset, and/or a constants.py file.
+    return collection
 
-    See `Item<https://pystac.readthedocs.io/en/latest/api.html#item>`_.
 
+def create_item(nc_href: str, sst_cog_href: str, sif_cog_href: str) -> Item:
+    """Creates a STAC Item
     Args:
-        asset_href (str): The HREF pointing to an asset associated with the item
-
+        nc_href (str): HREF of the netcdf associated with the Item
+        sst_cog_href (str): An HREF for the associated sea surface temp COG asset
+        sif_cog_href (str): An HREF for the associated sea ice fraction COG asset
+        The COG should be created in advance using `cog.create_cog`
     Returns:
         Item: STAC Item object
     """
-
-    properties = {
-        "title": "A dummy STAC Item",
-        "description": "Used for demonstration purposes",
-    }
-
-    demo_geom = {
+    with Dataset(nc_href) as ds:
+        properties = {
+            "title": ds.title,
+            "noaa-sst:institution": ds.institution,
+            "noaa-sst:source": ds.source,
+            "noaa-sst:history": ds.history,
+            "noaa-sst:comment": ds.comment,
+        }
+        item_datetime = datetime.strptime(ds.time_coverage_start,
+                                          '%Y%m%dT%H%M%SZ')
+
+        dims = ds.dimensions
+        ds_shape = [dims["lon"].size, dims["lat"].size]
+        x_cellsize = 360.0 / float(dims["lon"].size)
+        y_cellsize = 180.0 / float(dims["lat"].size)
+
+    global_geom = {
         "type":
         "Polygon",
-        "coordinates": [[[-180, -90], [180, -90], [180, 90], [-180, 90],
-                         [-180, -90]]],
+        "coordinates": [[[-180.0, -90.0], [180.0, -90.0], [180.0, 90.0],
+                         [-180.0, 90.0], [-180.0, -90.0]]],
     }
 
-    # Time must be in UTC
-    demo_time = datetime.now(tz=timezone.utc)
-
-    item = Item(
-        id="my-item-id",
-        properties=properties,
-        geometry=demo_geom,
-        bbox=[-180, 90, 180, -90],
-        datetime=demo_time,
-        stac_extensions=[],
-    )
+    item = Item(id=f"{NOAA_SST_ID}-noaa-{item_datetime}",
+                properties=properties,
+                geometry=global_geom,
+                bbox=SPATIAL_EXTENT,
+                datetime=item_datetime,
+                stac_extensions=[])
 
-    # It is a good idea to include proj attributes to optimize for libs like stac-vrt
     proj_attrs = ProjectionExtension.ext(item, add_if_missing=True)
-    proj_attrs.epsg = 4326
-    proj_attrs.bbox = [-180, 90, 180, -90]
-    proj_attrs.shape = [1, 1]  # Raster shape
-    proj_attrs.transform = [-180, 360, 0, 90, 0, 180]  # Raster GeoTransform
+    proj_attrs.epsg = SST_EPSG
+    proj_attrs.bbox = SPATIAL_EXTENT
+    proj_attrs.shape = ds_shape
+    proj_attrs.transform = [
+        SPATIAL_EXTENT[0],
+        x_cellsize,
+        0.0,
+        SPATIAL_EXTENT[1],
+        0.0,
+        -y_cellsize,
+    ]
 
     # Add an asset to the item (COG for example)
     item.add_asset(
-        "image",
+        "sst_cog",
         Asset(
-            href=asset_href,
+            href=sst_cog_href,
+            media_type=MediaType.COG,
+            roles=["data"],
+        ),
+    )
+    item.add_asset(
+        "sif_cog",
+        Asset(
+            href=sif_cog_href,
             media_type=MediaType.COG,
             roles=["data"],
-            title="A dummy STAC Item COG",
         ),
     )
-
     return item

tests/__init__.py

@@ -0,0 +1,3 @@
+from stactools.testing import TestData
+
+test_data = TestData(__file__)