WIP

helpers/ckan.py

@@ -1,30 +1,29 @@
 import json
 import os
 from collections import defaultdict
-from glob import glob
 
 out = defaultdict(dict)
 licenses = {}
 
+# The ckan directory is the one produced by https://github.com/Bjwebb/IATI-Registry-Refresher/tree/save_ckan_json
 
-for filepath in glob(os.path.join('metadata', '*', '*')):
-    publisher = filepath.split('/', 2)[1]
-    with open(filepath) as fp:
+for publisher in os.listdir('ckan'):
+    with open(os.path.join('ckan', publisher)) as fp:
         try:
-            package = json.load(fp)
-            if package['resources']:
-                extras = dict((x['key'], x['value']) for x in package['extras'])
-                out[publisher][package['name']] = {
-                    'title': package['title'],
-                    'extras': extras,
-                    'license_id': package['license_id'],
-                    'resource': package['resources'][0],
-                }
-                if package['license_id']:
-                    licenses[package['license_id']] = {
-                        'name': package['license_title'],
-                        'url': package.get('license_url'),
+            for package in json.load(fp)['result']['results']:
+                if package['resources']:
+                    extras = dict((x['key'], x['value']) for x in package['extras'])
+                    out[publisher][package['name']] = {
+                        'title': package['title'],
+                        'extras': extras,
+                        'license_id': package['license_id'],
+                        'resource': package['resources'][0],
                     }
+                    if package['license_id']:
+                        licenses[package['license_id']] = {
+                            'name': package['license_title'],
+                            'url': package.get('license_url'),
+                        }
         except ValueError:
             print('{0} is not valid JSON'.format(publisher))