diff --git a/data/.gitignore b/data/.gitignore index d703ed1..88737f3 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -3,3 +3,4 @@ /eidc_metadata.json /prepared_data.json /prepared_eidc_metadata.json +/extracted_metadata.json diff --git a/dvc.lock b/dvc.lock index 81cd8e2..bf2dea4 100644 --- a/dvc.lock +++ b/dvc.lock @@ -13,18 +13,18 @@ stages: md5: 423dc3a61ede72e1d5c818d74277c0b4 size: 12140491 prepare: - cmd: python scripts/prepare_data.py data/eidc_metadata.json data/prepared_eidc_metadata.json + cmd: python scripts/extract_metadata.py data/eidc_metadata.json data/extracted_metadata.json deps: - path: data/eidc_metadata.json hash: md5 md5: 423dc3a61ede72e1d5c818d74277c0b4 size: 12140491 - - path: scripts/prepare_data.py + - path: scripts/extract_metadata.py hash: md5 - md5: bcbf4413aeee83928054d9c6c6c2bacc - size: 1224 + md5: c2fa7d2c4b8f28a6e24536ce0df244fd + size: 1296 outs: - - path: data/prepared_eidc_metadata.json + - path: data/extracted_metadata.json hash: md5 - md5: 0b4ca8c49da450bc8fec0e92d577466c - size: 411936 + md5: 7d2ae8d6a41a960592f30496eb498af7 + size: 4578493 diff --git a/dvc.yaml b/dvc.yaml index dc7a55c..517a69d 100644 --- a/dvc.yaml +++ b/dvc.yaml @@ -6,9 +6,9 @@ stages: outs: - data/eidc_metadata.json prepare: - cmd: python scripts/prepare_data.py data/eidc_metadata.json data/prepared_eidc_metadata.json + cmd: python scripts/extract_metadata.py data/eidc_metadata.json data/extracted_metadata.json deps: - data/eidc_metadata.json - - scripts/prepare_data.py + - scripts/extract_metadata.py outs: - - data/prepared_eidc_metadata.json + - data/extracted_metadata.json diff --git a/scripts/prepare_data.py b/scripts/extract_metadata.py similarity index 76% rename from scripts/prepare_data.py rename to scripts/extract_metadata.py index 4adffdc..241bc1a 100644 --- a/scripts/prepare_data.py +++ b/scripts/extract_metadata.py @@ -3,17 +3,19 @@ from argparse import ArgumentParser -METADATA_FIELDS = ["title", "description", "lineage", "title"] +METADATA_FIELDS = ["title", "description", "lineage"] -def extact_eidc_metadata_fields(json_data: Dict, fields: List[str] = METADATA_FIELDS) -> Dict[str,str]: - metadata = {} - metadata["id"] = json_data["identifier"] +def extact_eidc_metadata_fields(json_data: Dict, fields: List[str] = METADATA_FIELDS) -> List[Dict[str,str]]: + metadatas = [] for field in fields: if json_data[field]: + metadata = {} + metadata["id"] = json_data["identifier"] metadata["field"] = field metadata["value"] = json_data[field] - return metadata + metadatas.append(metadata) + return metadatas def parse_eidc_metadata(file_path: str) -> List[Dict[str,str]]: @@ -22,7 +24,7 @@ def parse_eidc_metadata(file_path: str) -> List[Dict[str,str]]: json_data = json.load(f) for dataset in json_data["results"]: dataset_metadata = extact_eidc_metadata_fields(dataset) - data.append(dataset_metadata) + data.extend(dataset_metadata) return data