Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Groundwork for removing the vocabulary notices. #135

Closed
wants to merge 6 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 44 additions & 18 deletions stats/dashboard.py
Original file line number Diff line number Diff line change
@@ -86,18 +86,40 @@ def convert_to_float(x):


# Import codelists
## In order to test whether or not correct codelist values are being used in the data
## we need to pull in data about how codelists map to elements
# In order to test whether or not correct codelist values are being
# used in the data we need to pull in data about how codelists map
# to elements
def get_codelist_mapping(major_version):
codelist_mapping_xml = etree.parse('helpers/mapping-{}.xml'.format(major_version))
codelist_mappings = [ x.text for x in codelist_mapping_xml.xpath('mapping/path') ]
codelist_mappings = [ re.sub('^\/\/iati-activity', './',path) for path in codelist_mappings]
codelist_mappings = [ re.sub('^\/\/', './/', path) for path in codelist_mappings ]
return codelist_mappings
codelist_mapping_xml = etree.parse('helpers/mapping-{}.xml'.format(
major_version))
codelist_mappings = codelist_mapping_xml.xpath('mapping')
codelist_conditions = [x.find('condition').text
if x.find('condition') is not None else None
for x in codelist_mappings]
codelist_paths = [x.find('path').text for x in codelist_mappings]
codelist_paths = [re.sub(r'^\/\/iati-activity', './', path)
for path in codelist_paths]
codelist_paths = [re.sub(r'^\/\/', './/', path)
for path in codelist_paths]
codelist_condition_paths = []
for path, mapping in zip(codelist_paths, codelist_mappings):
condition = mapping.find('condition')
if condition is not None:
condition = condition.text
pref, attr = path.rsplit('/', 1)
condition_path = '{0}[{1}]/{2}'.format(pref, condition, attr)
else:
condition_path = None
codelist_condition_paths.append((path, condition_path))

return codelist_condition_paths


codelist_mappings = {major_version: get_codelist_mapping(major_version)
for major_version in ['1', '2']}

codelist_mappings = { major_version: get_codelist_mapping(major_version) for major_version in ['1', '2'] }

CODELISTS = {'1':{}, '2':{}}
CODELISTS = {'1': {}, '2': {}}
for major_version in ['1', '2']:
for codelist_name in ['Version', 'ActivityStatus', 'Currency', 'Sector', 'SectorCategory', 'DocumentCategory', 'AidType', 'BudgetNotProvided']:
CODELISTS[major_version][codelist_name] = set(c['code'] for c in json.load(open('helpers/codelists/{}/{}.json'.format(major_version, codelist_name)))['data'])
@@ -520,21 +542,25 @@ def elements(self):
def elements_total(self):
return element_to_count_dict(self.element, 'iati-activity', defaultdict(int), True)

@returns_numberdictdict
def codelist_values(self):
def _codelist_values(self):
out = defaultdict(lambda: defaultdict(int))
for path in codelist_mappings[self._major_version()]:
for value in self.element.xpath(path):
for path, condition_path in codelist_mappings[self._major_version()]:
if condition_path is not None:
values = self.element.xpath(condition_path)
else:
values = self.element.xpath(path)
for value in values:
out[path][value] += 1
return out

@returns_numberdictdict
def codelist_values(self):
return self._codelist_values()

@returns_numberdictdict
def codelist_values_by_major_version(self):
out = defaultdict(lambda: defaultdict(int))
for path in codelist_mappings[self._major_version()]:
for value in self.element.xpath(path):
out[path][value] += 1
return { self._major_version(): out }
out = self._codelist_values()
return {self._major_version(): out}

@returns_numberdictdict
def boolean_values(self):