Cleaned scripts and added additional metadata field

NERC-CEH · Oct 11, 2024 · 739f8fc · 739f8fc
1 parent 4e11bd5
commit 739f8fc
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 12 deletions.
diff --git a/dvc.lock b/dvc.lock
@@ -5,26 +5,26 @@ stages:
     deps:
     - path: scripts/fetch_eidc_metadata.py
       hash: md5
-      md5: ff336062c921e5e8f95bd569cd064e22
-      size: 664
+      md5: 43a63d91a3d66caa03738a000c841406
+      size: 674
     outs:
     - path: data/eidc_metadata.json
       hash: md5
-      md5: 3c4cd23bfc699358c955d4e7c68e8c9d
-      size: 8968593
+      md5: 423dc3a61ede72e1d5c818d74277c0b4
+      size: 12140491
   prepare:
     cmd: python scripts/prepare_data.py data/eidc_metadata.json data/prepared_eidc_metadata.json
     deps:
     - path: data/eidc_metadata.json
       hash: md5
-      md5: 3c4cd23bfc699358c955d4e7c68e8c9d
-      size: 8968593
+      md5: 423dc3a61ede72e1d5c818d74277c0b4
+      size: 12140491
     - path: scripts/prepare_data.py
       hash: md5
-      md5: bdab13adab508052f1d16ab0967b428b
-      size: 1215
+      md5: bcbf4413aeee83928054d9c6c6c2bacc
+      size: 1224
     outs:
     - path: data/prepared_eidc_metadata.json
       hash: md5
-      md5: ffd8914e46ffba8c47cdca6eb6ae0140
-      size: 2120475
+      md5: 0b4ca8c49da450bc8fec0e92d577466c
+      size: 411936
diff --git a/scripts/fetch_eidc_metadata.py b/scripts/fetch_eidc_metadata.py
@@ -15,7 +15,7 @@ def main(output_file: str) -> None:
         },
     )
     with open(output_file, "w") as f:
-        json.dump(res.json(), f)
+        json.dump(res.json(), f, indent=4)
 
 
 if __name__ == "__main__":

diff --git a/scripts/prepare_data.py b/scripts/prepare_data.py
@@ -3,7 +3,7 @@
 from argparse import ArgumentParser
 
 
-METADATA_FIELDS = ["title", "description", "lineage"]
+METADATA_FIELDS = ["title", "description", "lineage", "title"]
 
 
 def extact_eidc_metadata_fields(json_data: Dict, fields: List[str] = METADATA_FIELDS) -> Dict[str,str]: