diff --git a/git.sh b/git.sh index 4cc9704bf2a..243c80fe3aa 100755 --- a/git.sh +++ b/git.sh @@ -17,23 +17,23 @@ mkdir -p $GITOUT_DIR/gitaggregate-dated -cd helpers -# Update codelist mapping, codelists and schemas -echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelist mapping" -./get_codelist_mapping.sh -echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelists" -./get_codelists.sh -echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update schemas" -./get_schemas.sh - -wget -q https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv -wget -q https://codeforiati.org/imf-exchangerates/imf_exchangerates_A_ENDA_USD.csv -O currency_conversion/exchange_rates.csv - -# Build a JSON file of metadata for each CKAN publisher, and for each dataset published. -# This is based on the data from the CKAN API -echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running ckan.py" -python ckan.py -cd .. +# cd helpers +# # Update codelist mapping, codelists and schemas +# echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelist mapping" +# ./get_codelist_mapping.sh +# echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update codelists" +# ./get_codelists.sh +# echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Update schemas" +# ./get_schemas.sh +# +# wget -q https://raw.githubusercontent.com/codeforIATI/IATI-Dashboard/main/registry_id_relationships.csv +# wget -q https://codeforiati.org/imf-exchangerates/imf_exchangerates_A_ENDA_USD.csv -O currency_conversion/exchange_rates.csv +# +# # Build a JSON file of metadata for each CKAN publisher, and for each dataset published. +# # This is based on the data from the CKAN API +# echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Running ckan.py" +# python ckan.py +# cd .. echo "LOG: `date '+%Y-%m-%d %H:%M:%S'` - Copying ckan.json" cp helpers/ckan.json $GITOUT_DIR cp helpers/licenses.json $GITOUT_DIR diff --git a/stats/analytics.py b/stats/analytics.py index fa278566265..46ab28cb5cc 100644 --- a/stats/analytics.py +++ b/stats/analytics.py @@ -7,6 +7,7 @@ import copy import csv +import glob import json import os import re @@ -14,6 +15,7 @@ from datetime import date, datetime, timedelta from decimal import Decimal, InvalidOperation +from bdd_tester import BDDTester import iatirulesets from dateutil.relativedelta import relativedelta from helpers.currency_conversion import get_USD_value @@ -39,6 +41,38 @@ ) +INDEX_INDICATOR_DEFINTIONS_PATH = './2024-Index-indicator-definitions' + + +def load_ati_tests(): + """Load the index tests.""" + base_path = os.path.join(INDEX_INDICATOR_DEFINTIONS_PATH, "test_definitions") + step_definitions = os.path.join(base_path, "step_definitions.py") + feature_filepaths = glob.glob(os.path.join(base_path, "*", "*.feature")) + tester = BDDTester(step_definitions) + all_tests = [t for feature_filepath in feature_filepaths for t in tester.load_feature(feature_filepath).tests] + + # Remove the current data condition from tests. + for test in all_tests: + test.steps = [x for x in test.steps if not (x.step_type == "given" and x.text == "the activity is current")] + + return all_tests + + +ati_tests = load_ati_tests() + + +def load_ati_current_data_test(): + """Load the current data test.""" + base_path = os.path.join(INDEX_INDICATOR_DEFINTIONS_PATH, "test_definitions") + step_definitions = os.path.join(base_path, "step_definitions.py") + tester = BDDTester(step_definitions) + return tester.load_feature(os.path.join(base_path, "current_data.feature")).tests[0] + + +ati_current_data_test = load_ati_current_data_test() + + def add_years(d, years): """Return a date that's `years` years before/after the date (or datetime) object `d`. Return the same calendar date (month and day) in the @@ -2034,6 +2068,29 @@ def transaction_total(self): out += 1 return out + @returns_numberdict + @memoize + def ati_tests(self): + out = {} + for test in ati_tests: + result = test(self.element) + result = int(bool(result)) + out[f"{test.feature.name}: {test.name}"] = result + return out + + @returns_number + @memoize + def ati_current(self): + return int(bool(ati_current_data_test(self.element))) + + @returns_numberdict + def ati_tests_current(self): + if self.ati_current(): + return self.ati_tests() + else: + return {} + + ckan = json.load(open("helpers/ckan.json")) publisher_re = re.compile(r"(.*)\-[^\-]") diff --git a/stats/tests/test_ati.py b/stats/tests/test_ati.py new file mode 100644 index 00000000000..1c7c8244793 --- /dev/null +++ b/stats/tests/test_ati.py @@ -0,0 +1,50 @@ +# coding=utf-8 +import datetime + +import pytest +from lxml import etree + +from stats.analytics import ActivityStats + + +class MockActivityStats(ActivityStats): + def __init__(self, major_version): + self.major_version = major_version + return super(MockActivityStats, self).__init__() + + def _major_version(self): + return self.major_version + + +@pytest.mark.parametrize("major_version", ["1", "2"]) +def test_comprehensiveness_is_current(major_version): + activity_stats = MockActivityStats(major_version) + + activity_stats.element = etree.fromstring( + """ + + + <narrative>Title</narrative> + + + """ + ) + ati_dict = activity_stats.ati_tests() + assert type(ati_dict) is dict + ati_dict["Title: Title is present"] == 1 + ati_dict["Title: Title has at least 10 characters"] == 0 + + activity_stats.element = etree.fromstring( + """ + + + <narrative>Title with at least 10 characters</narrative> + + + """ + ) + + ati_dict = activity_stats.ati_tests() + assert type(ati_dict) is dict + ati_dict["Title: Title is present"] == 1 + ati_dict["Title: Title has at least 10 characters"] == 0