diff --git a/cloudbuild.py.yaml b/cloudbuild.py.yaml index 09c63c99ca..503e74dcf2 100644 --- a/cloudbuild.py.yaml +++ b/cloudbuild.py.yaml @@ -10,13 +10,6 @@ steps: - | ./run_tests.sh -r -# We can't just test everything under our Cloud Build working directory because -# that's where python installs all of its packages also, and we don't really -# want to test all of that other unrelated code everytime. -# -# The unittest module only supports one -s path at a time, so we need multiple rules -# to get both places where we submit code. -# - id: python_test name: python:3.11 entrypoint: /bin/sh @@ -25,9 +18,7 @@ steps: args: - -c - | - ./run_tests.sh -p util/ - ./run_tests.sh -p import-automation/executor - ./run_tests.sh -p scripts/ + ./run_tests.sh -a - id: python_format_check name: python:3.11 diff --git a/import-automation/executor/test/integration_test.py b/import-automation/executor/test/integration_test.py index 7fd562f53f..25fc32664c 100644 --- a/import-automation/executor/test/integration_test.py +++ b/import-automation/executor/test/integration_test.py @@ -27,10 +27,10 @@ # The GitHub params belong to the public Data Commons gmail account. # Auth tokens, user name and other details can be found in the inbox # and in the inbox of teammates. - 'github_repo_owner_username': os.environ['_GITHUB_REPO_OWNER_USERNAME'], + 'github_repo_owner_username': os.getenv('_GITHUB_REPO_OWNER_USERNAME', ''), 'github_repo_name': 'data-demo', - 'github_auth_username': os.environ['_GITHUB_AUTH_USERNAME'], - 'github_auth_access_token': os.environ['_GITHUB_AUTH_ACCESS_TOKEN'] + 'github_auth_username': os.getenv('_GITHUB_AUTH_USERNAME', ''), + 'github_auth_access_token': os.getenv('_GITHUB_AUTH_ACCESS_TOKEN', '') } diff --git a/tools/import_differ/import_differ.py b/tools/import_differ/import_differ.py index 21e659ba74..00c9ab43b9 100644 --- a/tools/import_differ/import_differ.py +++ b/tools/import_differ/import_differ.py @@ -21,7 +21,7 @@ from absl import flags from absl import logging -import differ_utils +from tools.import_differ import differ_utils SAMPLE_COUNT = 3 GROUPBY_COLUMNS = 'variableMeasured,observationAbout,observationDate,measurementMethod,unit,observationPeriod' diff --git a/tools/import_differ/import_differ_test.py b/tools/import_differ/import_differ_test.py index 60137119d8..46bbefcf8a 100644 --- a/tools/import_differ/import_differ_test.py +++ b/tools/import_differ/import_differ_test.py @@ -17,9 +17,8 @@ import unittest from pandas.testing import assert_frame_equal -from import_differ import ImportDiffer - -import differ_utils +from tools.import_differ import import_differ +from tools.import_differ import differ_utils module_dir = os.path.dirname(__file__) @@ -37,8 +36,9 @@ def test_diff_analysis(self): previous_data = os.path.join(module_dir, 'test', 'previous.mcf') output_location = os.path.join(module_dir, 'test') - differ = ImportDiffer(current_data, previous_data, output_location, - groupby_columns, value_columns) + differ = import_differ.ImportDiffer(current_data, previous_data, + output_location, groupby_columns, + value_columns) current = differ_utils.load_mcf_file(current_data) previous = differ_utils.load_mcf_file(previous_data) diff --git a/tools/import_validation/__init__.py b/tools/import_validation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/import_validation/import_validation.py b/tools/import_validation/import_validation.py index 8dee351aeb..a83f6dc5cb 100644 --- a/tools/import_validation/import_validation.py +++ b/tools/import_validation/import_validation.py @@ -22,7 +22,7 @@ import json FLAGS = flags.FLAGS -flags.DEFINE_string('config_file', 'validation_config.json', +flags.DEFINE_string('validation_config_file', 'validation_config.json', 'Path to the validation config file.') flags.DEFINE_string('differ_output_location', '.', 'Path to the differ output data folder.') @@ -61,7 +61,7 @@ class ImportValidation: Class to perform validations for import automation. Usage: - $ python import_validation.py --config_file= \ + $ python import_validation.py --validation_config_file= \ --differ_output_location= --stats_summary_location= Each import can provide configuration (JSON) to select which validation @@ -129,7 +129,7 @@ def run_validations(self): def main(_): validation = ImportValidation( - FLAGS.config_file, + FLAGS.validation_config_file, os.path.join(FLAGS.differ_output_location, POINT_ANALAYSIS_FILE), os.path.join(FLAGS.stats_summary_location, STATS_SUMMARY_FILE), os.paht.join(FLAGS.validation_output_location, VALIDATION_OUTPUT_FILE)) diff --git a/tools/import_validation/import_validation_test.py b/tools/import_validation/import_validation_test.py index a33ca72d66..cab586fa13 100644 --- a/tools/import_validation/import_validation_test.py +++ b/tools/import_validation/import_validation_test.py @@ -17,7 +17,7 @@ import unittest from pandas.testing import assert_frame_equal -from import_validation import ImportValidation +from tools.import_validation import import_validation module_dir = os.path.dirname(__file__) @@ -32,8 +32,11 @@ def test_validation(self): result_file = os.path.join(module_dir, 'test', 'test_output.csv') config_file = os.path.join(module_dir, 'test', 'test_config.json') differ_output = os.path.join(module_dir, 'test', 'differ_output.csv') + validation_output = os.path.join(module_dir, 'validation_output.csv') - validation = ImportValidation(config_file, differ_output, '') + validation = import_validation.ImportValidation(config_file, + differ_output, '', + validation_output) validation.run_validations() expected = pd.read_csv(result_file)