refactor: introduce recursive function for getting field list #15

ICIJ · Jan 11, 2023 · e6fe6a1 · e6fe6a1
1 parent 4f9bd2b
commit e6fe6a1
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 14 deletions.
diff --git a/tarentula/datashare_client.py b/tarentula/datashare_client.py
@@ -193,26 +193,26 @@ def temporary_project(self, source=DATASHARE_DEFAULT_PROJECT, delete=True):
         finally:
             if delete and project is not None:
                 self.delete_index(project)
-        return project      
+        return project
 
-    def metadata_fields(self, index=DATASHARE_DEFAULT_PROJECT, documentType='Document'):
+    def metadata_fields(self, index=DATASHARE_DEFAULT_PROJECT, document_type='Document'):
         url = urljoin(self.elasticsearch_host, index)
         mapping = requests.get(url, cookies=self.cookies, headers=self.headers).json()
         results = []
+        self.get_fields(document_type, index, mapping, results, [])
+        return results
+
+    def get_fields(self, document_type, index, mapping, results, field_stack):
         for field, properties in mapping[index]['mappings']['properties'].items():
+            complete_field_name = '.'.join(field_stack + [field])
             count = self.count(index, query={
-                "query": {"bool": {"must": {"match": {"type": documentType}}, "filter": {"exists": {"field": field}}}}
+                "query": {"bool": {"must": {"match": {"type": document_type}},
+                                   "filter": {"exists": {"field": complete_field_name}}}}
             })
             if 'type' in properties:
                 if count["count"] > 0:
-                    results.append({"field": field, "type": properties["type"], "count": count["count"]})
+                    results.append({"field": complete_field_name, "type": properties["type"], "count": count["count"]})
             elif 'properties' in properties:
-                for f, p in mapping[index]['mappings']['properties'][field]['properties'].items():
-                    inner_field = f'{field}.{f}'
-                    inner_count = self.count(index, query={
-                        "query": {
-                            "bool": {"must": {"match": {"type": documentType}}, "filter": {"exists": {"field": inner_field}}}}
-                    })
-                    if inner_count["count"] > 0:
-                        results.append({"field": inner_field, "type": p["type"], "count": inner_count["count"]})
-        return results
+                self.get_fields(document_type, index,
+                                {index: {"mappings": mapping[index]['mappings']['properties'][field]}}, results,
+                                field_stack + [field])
diff --git a/tarentula/metadata_fields.py b/tarentula/metadata_fields.py
@@ -32,5 +32,5 @@ def __init__(self,
             exit()
 
     def start(self):
-        fields = self.datashare_client.metadata_fields(self.datashare_project, documentType=self.type)
+        fields = self.datashare_client.metadata_fields(self.datashare_project, document_type=self.type)
         print(dumps(fields))