Skip to content

Commit

Permalink
Adding pydml writing #6
Browse files Browse the repository at this point in the history
  • Loading branch information
sam-laptop-vm committed Dec 23, 2022
1 parent 8eac151 commit d9f92db
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 20 deletions.
36 changes: 36 additions & 0 deletions build_catalog/configuration-pytdml-tds.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
dataset_type: EOTrainingDataset
id: spacenet_7
name: SpaceNet 7 Multi-Temporal Urban Development Challenge
description: The SpaceNet 7 Multi-Temporal Urban Development Challenge aims to help address this deficit and develop novel computer vision methods for non-video time series data.
version: 1.0
created_time: 2020
updated_time: 2020
license: CC-BY-SA-4.0
providers:
- SpaceNet LLC
keywords:
- Building Footprints
classes:
- Building
tasks:
- description: Track buildings in satellite imagery time series
task_type: Building Object Change Detection
data_sources:
- id: mosaic-image
data_type: Optical Image
format: tif
bands:
- red
- green
- blue
image_size: 1024x1023
data:
task_type: ObjectDetection
label_type: ObjectLabel
data_path:
- type: image
format: stac
root_path: D:\TrainingDatasets\SpaceNet7\sn7_train_source
- type: label
format: stac
root_path: D:\TrainingDatasets\SpaceNet7\sn7_train_labels
33 changes: 25 additions & 8 deletions build_catalog/configuration-tds.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,35 @@

# Testing url
url: "https://pixalytics-ogc-api.s3.eu-west-2.amazonaws.com/TDS/input/20200831T101156_rgb_classification.tif"

# Output director.
output_dir: "./ogcapi/CATALOG"

# Catalog Id
catalog_id: "tds-catalog"

# Catalog description
catalog_title: "TDS Catalog"

# Catalog description
catalog_desc: "TDS catalog"

# TDS url
url: "https://pixalytics-ogc-api.s3.eu-west-2.amazonaws.com/TDS/input/"

# TDS data files to include in catalog
files: "tile49_rgb.tif"

# TDS label files to include in catalog
label_files: "tile49_label.tif"

# Image Id
image_id: "image_id"

# Spatial resolution
gsd: "10.0"

# Output director.
output_dir: "./ogcapi/CATALOG"

# YAML for API Records
yaml_file: "./tds-record.yml"
yaml_file: "./tds-record.yml"

# Provider details
provider_name: "Pixalytics Ltd"
provider_url: "https://www.pixalytics.com/"


47 changes: 35 additions & 12 deletions build_catalog/create_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,14 @@ def pull_s3bucket(logger, tmp_dir, url, catalog_id, catalog_desc):
return img_path


def add_item(footprint, bbox, epsg, gsd, img_path, image_id):
fdate = image_id.split("_")[0]
dateval = datetime(int(fdate[0:4]), int(fdate[4:6]), int(fdate[6:8]), int(fdate[9:11]), int(fdate[11:13]),
int(fdate[13:15]))
def add_item(logger, footprint, bbox, epsg, gsd, img_path, image_id):
try:
fdate = image_id.split("_")[0]
dateval = datetime(int(fdate[0:4]), int(fdate[4:6]), int(fdate[6:8]), int(fdate[9:11]), int(fdate[11:13]),
int(fdate[13:15]))
except:
dateval = datetime.utcnow()
logger.warning("Failed to extract date from {}, using today's date".format(image_id, dateval))

# Add item to catalog and apply timestamp
item = pystac.Item(id=image_id.split(".")[0],
Expand Down Expand Up @@ -117,6 +121,10 @@ def add_item(footprint, bbox, epsg, gsd, img_path, image_id):
return item


def write_pytdml(logger,pytdml_yaml, pytdml_json):
cmd = "pytdml/yaml_to_tdml.py --config={} --output={}".format(pytdml_yaml, pytdml_json)


def main():
parser = ArgumentParser(
description="Creates STAC Catalog (as Collection or Catalog) or OGC Records Catalog",
Expand Down Expand Up @@ -196,9 +204,10 @@ def main():

# Configuration to be loaded from main directory
if args.test:
CONFIGURATION_FILE_PATH = os.path.join(code_dir, "test-configuration.yaml")
CONFIGURATION_FILE_PATH = os.path.join(code_dir, "configuration-test.yaml")
elif args.tds:
CONFIGURATION_FILE_PATH = os.path.join(code_dir, "tds-configuration.yaml")
CONFIGURATION_FILE_PATH = os.path.join(code_dir, "configuration-tds.yaml")
CONFIGURATION_PYTDML = os.path.join(code_dir, "configuration-tds-pytdml.yaml")
elif args.netcdf:
CONFIGURATION_FILE_PATH = os.path.join(code_dir, "configuration-nc.yaml")
elif args.netcdfsingle:
Expand All @@ -215,6 +224,12 @@ def main():
url = config["url"]
temp = config["files"]
files = temp.split(",")

# Additional files for a TDS dataset
if "tds" in CONFIGURATION_FILE_PATH:
temp = config["label_files"]
label_files = temp.split(",")

out_default = config["output_dir"]
gsd = config["gsd"]
yaml_file = config["yaml_file"]
Expand Down Expand Up @@ -256,7 +271,7 @@ def main():

# Date range
# Add item to catalog
if args.test:
if args.test or args.tds:
dateval = datetime.utcnow()
end_dateval = dateval
else:
Expand Down Expand Up @@ -284,6 +299,7 @@ def main():
cat_folder = os.path.join(outdir, "{}-stac{}-v{}".format(catalog_id, netcdf, version))
elif args.tds:
cat_folder = os.path.join(outdir, "{}-tds{}-v{}".format(catalog_id, netcdf, version))
pytdml_folder = os.path.join(outdir, "{}-tds-pytdml-v{}".format(catalog_id, version))
else:
cat_folder = os.path.join(outdir, "{}-records{}-v{}".format(catalog_id, netcdf, version))

Expand Down Expand Up @@ -317,13 +333,13 @@ def main():
catalog = pystac.Catalog(id=catalog_id, title=catalog_title, description=catalog_desc)

for count, file in enumerate(files):
item = add_item(footprint, bbox, src_crs.split(":")[1], gsd, url, file)
item = add_item(logger, footprint, bbox, src_crs.split(":")[1], gsd, url, file)
catalog.add_item(item)

if count == 0:
# JSON dump item
logger.debug(json.dumps(item.to_dict(), indent=4))

catalog.add_item(item)

# Update extents in catalog from items
if args.collection:
catalog.update_extent_from_items()
Expand All @@ -345,12 +361,15 @@ def main():
catalog = pystac.Catalog(id=catalog_id, title=catalog_title, description=catalog_desc)

for count, file in enumerate(files):
item = add_item(footprint, bbox, src_crs.split(":")[1], gsd, url, file)
item = add_item(logger, footprint, bbox, src_crs.split(":")[1], gsd, url, file)
catalog.add_item(item)
if count == 0:
# JSON dump item
logger.debug(json.dumps(item.to_dict(), indent=4))

catalog.add_item(item)
logger.info("Adding label file")
item = add_item(logger, footprint, bbox, src_crs.split(":")[1], gsd, url, label_files[count])
catalog.add_item(item)

# Set HREFs
catalog.normalize_hrefs(cat_folder)
Expand All @@ -365,6 +384,10 @@ def main():
with open(catalog.get_self_href()) as f:
print(f.read())

# Also create pytdml catalog
pytdml_json = os.path.join(pytdml_folder, "{}.gson".format(catalog_id))
write_pytdml(logger, CONFIGURATION_PYTDML, pytdml_json)

else: # OGC Records
logger.info("Creating OGC Records Catalog")

Expand Down

0 comments on commit d9f92db

Please sign in to comment.