Skip to content

Commit

Permalink
Update BRDF parser for metadata change on the 11th
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeremyh
jeremyh committed Apr 4, 2024
1 parent f2117dd commit 668e842
Show file tree
Hide file tree
Showing 4 changed files with 413 additions and 38 deletions.
96 changes: 58 additions & 38 deletions eodatasets3/prepare/nasa_c_m_mcd43a1_6_prepare.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,28 @@ def parse_xml(filepath: Path):
root = ElementTree.parse(str(filepath), forbid_dtd=False).getroot()

granule_id = root.find("*//ECSDataGranule/LocalGranuleID").text
instrument = root.find("*//Platform/Instrument/InstrumentShortName").text
platform = "+".join(
sorted(
(ele.text for ele in root.findall("*//Platform/PlatformShortName")),
reverse=True,

collection_name = root.find("*//CollectionMetaData/ShortName").text
collection_version = root.find("*//CollectionMetaData/VersionID").text

instrument_node = root.find("*//Platform/Instrument/InstrumentShortName")

if instrument_node is not None:
instrument = instrument_node.text
platform = "+".join(
sorted(
(ele.text for ele in root.findall("*//Platform/PlatformShortName")),
reverse=True,
)
)
)
elif collection_name.startswith("MCD43"):
instrument = "MODIS"
platform = "Terra+Aqua"
else:
raise ValueError(
f"Could not determine instrument and platform from collection name {collection_name}"
)

start_date = root.find("*//RangeDateTime/RangeBeginningDate").text
start_time = root.find("*//RangeDateTime/RangeBeginningTime").text
end_date = root.find("*//RangeDateTime/RangeEndingDate").text
Expand All @@ -57,6 +72,7 @@ def parse_xml(filepath: Path):
creation_dt = root.find("*//InsertTime").text

return {
"collection_version": collection_version,
"granule_id": granule_id,
"instrument": instrument,
"platform": platform,
Expand Down Expand Up @@ -131,38 +147,42 @@ def process_datasets(input_path: Path, xml_file: Path) -> Iterable[Dict]:
xml_md = parse_xml(xml_file)
ds_props = _get_dataset_properties(datasets[0])

md = {}
md["id"] = str(uuid.uuid5(MCD43A1_NS, xml_md["granule_id"]))
md["product"] = {"href": "https://collections.dea.ga.gov.au/nasa_c_m_mcd43a1_6"}
md["crs"] = ds_props.pop("crs")
md["geometry"] = valid_region(datasets)
md["grids"] = ds_props.pop("grids")
md["lineage"] = {}
md["measurements"] = band_info
md["properties"] = {
"dtr:start_datetime": xml_md["from_dt"].isoformat(),
"dtr:end_datetime": xml_md["to_dt"].isoformat(),
"eo:instrument": xml_md["instrument"],
"eo:platform": xml_md["platform"],
"eo:gsd": ds_props.pop("eo:gsd"),
"eo:epsg": None,
"item:providers": [
{
"name": "National Aeronautics and Space Administration",
"roles": [ItemProvider.PRODUCER.value, ItemProvider.PROCESSOR.value],
"url": "https://modis.gsfc.nasa.gov/data/dataprod/mod43.php",
},
{
"name": "United States Geological Society",
"roles": [ItemProvider.PROCESSOR.value],
"url": "https://lpdaac.usgs.gov/products/mcd43a1v006/",
},
],
"odc:creation_datetime": xml_md["creation_dt"].isoformat(),
"odc:file_format": "HDF4_EOS:EOS_GRID",
"odc:region_code": "h{}v{}".format(
xml_md["horizontal_tile"], xml_md["vertical_tile"]
),
md = {
"id": str(uuid.uuid5(MCD43A1_NS, xml_md["granule_id"])),
"product": {"href": "https://collections.dea.ga.gov.au/nasa_c_m_mcd43a1_6"},
"crs": ds_props.pop("crs"),
"geometry": valid_region(datasets),
"grids": ds_props.pop("grids"),
"lineage": {},
"measurements": band_info,
"properties": {
"dtr:start_datetime": xml_md["from_dt"].isoformat(),
"dtr:end_datetime": xml_md["to_dt"].isoformat(),
"eo:instrument": xml_md["instrument"],
"eo:platform": xml_md["platform"],
"eo:gsd": ds_props.pop("eo:gsd"),
"eo:epsg": None,
"item:providers": [
{
"name": "National Aeronautics and Space Administration",
"roles": [
ItemProvider.PRODUCER.value,
ItemProvider.PROCESSOR.value,
],
"url": "https://modis.gsfc.nasa.gov/data/dataprod/mod43.php",
},
{
"name": "United States Geological Society",
"roles": [ItemProvider.PROCESSOR.value],
"url": "https://lpdaac.usgs.gov/products/mcd43a1v006/",
},
],
"odc:creation_datetime": xml_md["creation_dt"].isoformat(),
"odc:file_format": "HDF4_EOS:EOS_GRID",
"odc:region_code": "h{}v{}".format(
xml_md["horizontal_tile"], xml_md["vertical_tile"]
),
},
}

return [md]
Expand Down
213 changes: 213 additions & 0 deletions tests/integration/prepare/MCD43A1.A2024070.h19v16.061.2024079033215.hdf.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE GranuleMetaDataFile SYSTEM "http://ecsinfo.gsfc.nasa.gov/ECSInfo/ecsmetadata/dtds/DPL/ECS/ScienceGranuleMetadata.dtd">
<GranuleMetaDataFile>
<DTDVersion>1.0</DTDVersion>
<DataCenterId>EDC</DataCenterId>
<GranuleURMetaData>
<GranuleUR>SC:MCD43A1.061:2679459485</GranuleUR>
<DbID>2679459485</DbID>
<InsertTime>2024-03-18 22:52:25.228</InsertTime>
<LastUpdate>2024-03-18 22:52:25.228</LastUpdate>
<CollectionMetaData>
<ShortName>MCD43A1</ShortName>
<VersionID>61</VersionID>
</CollectionMetaData>
<DataFiles>
<DataFileContainer>
<DistributedFileName>MCD43A1.A2024070.h19v16.061.2024079033215.hdf</DistributedFileName>
<FileSize>166175778</FileSize>
<ChecksumType>MD5</ChecksumType>
<Checksum>564edefa30e95cd6d7b5f682d30c8aa0</Checksum>
<ChecksumOrigin>DPLIngst</ChecksumOrigin>
</DataFileContainer>
</DataFiles>
<ECSDataGranule>
<SizeMBECSDataGranule>158.478</SizeMBECSDataGranule>
<ReprocessingPlanned>further update is anticipated</ReprocessingPlanned>
<ReprocessingActual>processed once</ReprocessingActual>
<LocalGranuleID>MCD43A1.A2024070.h19v16.061.2024079033215.hdf</LocalGranuleID>
<DayNightFlag>Day</DayNightFlag>
<ProductionDateTime>2024-03-19 03:39:40.000</ProductionDateTime>
<LocalVersionID>6.1.34</LocalVersionID>
</ECSDataGranule>
<PGEVersionClass>
<PGEVersion>6.1.13</PGEVersion>
</PGEVersionClass>
<RangeDateTime>
<RangeEndingTime>23:59:59.999999</RangeEndingTime>
<RangeEndingDate>2024-03-17</RangeEndingDate>
<RangeBeginningTime>00:00:00.000000</RangeBeginningTime>
<RangeBeginningDate>2024-03-02</RangeBeginningDate>
</RangeDateTime>
<SpatialDomainContainer>
<HorizontalSpatialDomainContainer>
<GPolygon>
<Boundary>
<Point>
<PointLongitude>58.0861219214408</PointLongitude>
<PointLatitude>-80.1214779421689</PointLatitude>
</Point>
<Point>
<PointLongitude>28.4645734312001</PointLongitude>
<PointLatitude>-69.3533470211469</PointLatitude>
</Point>
<Point>
<PointLongitude>61.2685062171821</PointLongitude>
<PointLatitude>-69.0789424401956</PointLatitude>
</Point>
<Point>
<PointLongitude>125.319334242246</PointLongitude>
<PointLatitude>-78.7435199798004</PointLatitude>
</Point>
</Boundary>
</GPolygon>
</HorizontalSpatialDomainContainer>
</SpatialDomainContainer>
<MeasuredParameter>
<MeasuredParameterContainer>
<ParameterName>NOT SET</ParameterName>
<QAFlags>
<AutomaticQualityFlag>Passed</AutomaticQualityFlag>
<AutomaticQualityFlagExplanation>Passed was set as a default value. More algorithm will be developed</AutomaticQualityFlagExplanation>
<OperationalQualityFlag>Passed</OperationalQualityFlag>
<OperationalQualityFlagExplanation>Passed</OperationalQualityFlagExplanation>
<ScienceQualityFlag>Not Investigated</ScienceQualityFlag>
<ScienceQualityFlagExplanation>See http://landweb.nascom/nasa.gov/cgi-bin/QA_WWW/qaFlagPage.cgi?sat=aqua the product Science Quality status.</ScienceQualityFlagExplanation>
</QAFlags>
</MeasuredParameterContainer>
</MeasuredParameter>
<Platform>
<PlatformShortName>Terra</PlatformShortName>
<Instrument>
<InstrumentShortName>MODIS</InstrumentShortName>
<Sensor>
<SensorShortName>MODIS</SensorShortName>
</Sensor>
</Instrument>
</Platform>
<Platform>
<PlatformShortName>Aqua</PlatformShortName>
<Instrument>
<InstrumentShortName>MODIS</InstrumentShortName>
<Sensor>
<SensorShortName>MODIS</SensorShortName>
</Sensor>
</Instrument>
</Platform>
<PSAs>
<PSA>
<PSAName>QAPERCENTGOODQUALITY</PSAName>
<PSAValue>19</PSAValue>
</PSA>
<PSA>
<PSAName>QAPERCENTOTHERQUALITY</PSAName>
<PSAValue>68</PSAValue>
</PSA>
<PSA>
<PSAName>QAPERCENTNOTPRODUCEDCLOUD</PSAName>
<PSAValue>11</PSAValue>
</PSA>
<PSA>
<PSAName>QAPERCENTNOTPRODUCEDOTHER</PSAName>
<PSAValue>0</PSAValue>
</PSA>
<PSA>
<PSAName>HORIZONTALTILENUMBER</PSAName>
<PSAValue>19</PSAValue>
</PSA>
<PSA>
<PSAName>VERTICALTILENUMBER</PSAName>
<PSAValue>16</PSAValue>
</PSA>
<PSA>
<PSAName>TileID</PSAName>
<PSAValue>51019016</PSAValue>
</PSA>
<PSA>
<PSAName>BRDFCODEID</PSAName>
<PSAValue>AMBRALS_V4.0R1</PSAValue>
</PSA>
<PSA>
<PSAName>SETUPFILEID</PSAName>
<PSAValue>06121997</PSAValue>
</PSA>
<PSA>
<PSAName>ALBEDOFILEID</PSAName>
<PSAValue>06121997</PSAValue>
</PSA>
<PSA>
<PSAName>BRDFDATABASEVERSION</PSAName>
<PSAValue>v1.0(500m)</PSAValue>
</PSA>
<PSA>
<PSAName>PERCENTLANDINTILE</PSAName>
<PSAValue>92</PSAValue>
</PSA>
<PSA>
<PSAName>PERCENTPROCESSEDINTILE</PSAName>
<PSAValue>90</PSAValue>
</PSA>
<PSA>
<PSAName>PERCENTNEWBRDFS</PSAName>
<PSAValue>22</PSAValue>
</PSA>
<PSA>
<PSAName>PERCENTSHAPEFIXEDBRDFS</PSAName>
<PSAValue>77</PSAValue>
</PSA>
<PSA>
<PSAName>PERCENTSUBSTITUTEBRDFS</PSAName>
<PSAValue>0</PSAValue>
</PSA>
<PSA>
<PSAName>AVERAGENUMBEROBS</PSAName>
<PSAValue>17</PSAValue>
</PSA>
<PSA>
<PSAName>identifier_product_doi</PSAName>
<PSAValue>10.5067/MODIS/MCD43A1.061</PSAValue>
</PSA>
<PSA>
<PSAName>identifier_product_doi_authority</PSAName>
<PSAValue>http://dx.doi.org</PSAValue>
</PSA>
</PSAs>
<InputGranule>
<InputPointer>MYD09GA.A2024062.h19v16.061.2024064054054.hdf</InputPointer>
<InputPointer>MYD09GA.A2024063.h19v16.061.2024065123316.hdf</InputPointer>
<InputPointer>MYD09GA.A2024064.h19v16.061.2024066070025.hdf</InputPointer>
<InputPointer>MYD09GA.A2024065.h19v16.061.2024067045627.hdf</InputPointer>
<InputPointer>MYD09GA.A2024066.h19v16.061.2024068054232.hdf</InputPointer>
<InputPointer>MYD09GA.A2024067.h19v16.061.2024069033053.hdf</InputPointer>
<InputPointer>MYD09GA.A2024068.h19v16.061.2024070032630.hdf</InputPointer>
<InputPointer>MYD09GA.A2024069.h19v16.061.2024071031737.hdf</InputPointer>
<InputPointer>MYD09GA.A2024070.h19v16.061.2024072044438.hdf</InputPointer>
<InputPointer>MYD09GA.A2024071.h19v16.061.2024073030417.hdf</InputPointer>
<InputPointer>MYD09GA.A2024072.h19v16.061.2024074033105.hdf</InputPointer>
<InputPointer>MYD09GA.A2024073.h19v16.061.2024075035410.hdf</InputPointer>
<InputPointer>MYD09GA.A2024074.h19v16.061.2024076071755.hdf</InputPointer>
<InputPointer>MYD09GA.A2024075.h19v16.061.2024077061041.hdf</InputPointer>
<InputPointer>MYD09GA.A2024076.h19v16.061.2024078230426.hdf</InputPointer>
<InputPointer>MYD09GA.A2024077.h19v16.061.2024079031724.hdf</InputPointer>
<InputPointer>MOD09GA.A2024062.h19v16.061.2024064043216.hdf</InputPointer>
<InputPointer>MOD09GA.A2024063.h19v16.061.2024065104338.hdf</InputPointer>
<InputPointer>MOD09GA.A2024064.h19v16.061.2024066062058.hdf</InputPointer>
<InputPointer>MOD09GA.A2024065.h19v16.061.2024067035936.hdf</InputPointer>
<InputPointer>MOD09GA.A2024066.h19v16.061.2024068063239.hdf</InputPointer>
<InputPointer>MOD09GA.A2024067.h19v16.061.2024069030844.hdf</InputPointer>
<InputPointer>MOD09GA.A2024068.h19v16.061.2024070031548.hdf</InputPointer>
<InputPointer>MOD09GA.A2024069.h19v16.061.2024071030858.hdf</InputPointer>
<InputPointer>MOD09GA.A2024070.h19v16.061.2024073195456.hdf</InputPointer>
<InputPointer>MOD09GA.A2024071.h19v16.061.2024073193826.hdf</InputPointer>
<InputPointer>MOD09GA.A2024072.h19v16.061.2024074025416.hdf</InputPointer>
<InputPointer>MOD09GA.A2024073.h19v16.061.2024075033805.hdf</InputPointer>
<InputPointer>MOD09GA.A2024074.h19v16.061.2024076064841.hdf</InputPointer>
<InputPointer>MOD09GA.A2024075.h19v16.061.2024077061417.hdf</InputPointer>
<InputPointer>MOD09GA.A2024076.h19v16.061.2024078060346.hdf</InputPointer>
<InputPointer>MOD09GA.A2024077.h19v16.061.2024079031018.hdf</InputPointer>
<InputPointer>MCD43DB.A2024069.61.h19v16.hdf</InputPointer>
</InputGranule>
<BrowseProduct>
<BrowseGranuleId>UR:10:DsShESDTUR:UR:15:DsShSciServerUR:13:[EDC:DSSDSRV]:24:BR:Browse.001:2679459492</BrowseGranuleId>
</BrowseProduct>
</GranuleURMetaData>
</GranuleMetaDataFile>
Loading

0 comments on commit 668e842

Please sign in to comment.