From 746a67116ad4da635242d752e042ef336c3808b8 Mon Sep 17 00:00:00 2001 From: Keith Battocchi Date: Thu, 4 Mar 2021 16:07:31 -0500 Subject: [PATCH] Enable memory profiling --- azure-pipelines.yml | 137 ++----------------- econml/tests/test_ate_inference.py | 1 + econml/tests/test_bootstrap.py | 1 + econml/tests/test_dowhy.py | 1 + econml/tests/test_drlearner.py | 1 + econml/tests/test_linear_model.py | 1 + econml/tests/test_montecarlo.py | 1 + econml/tests/test_orf.py | 1 + econml/tests/test_ortho_learner.py | 1 + econml/tests/test_rscorer.py | 1 + econml/tests/test_statsmodels.py | 1 + econml/tests/test_two_stage_least_squares.py | 1 + pyproject.toml | 5 +- 13 files changed, 28 insertions(+), 125 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 56afea596..a262a5a7f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -48,115 +48,10 @@ jobs: name: output displayName: 'Determine type of code change' -- template: azure-pipelines-steps.yml - parameters: - versions: ['3.6'] - images: ['ubuntu-18.04'] - package: '-e .[all]' - job: - job: 'Docs' - displayName: 'Build documentation' - dependsOn: 'EvalChanges' - condition: eq(dependencies.EvalChanges.outputs['output.buildDocs'], 'True') - steps: - - script: 'sudo apt-get -yq install graphviz' - displayName: 'Install graphviz' - - - script: 'pip install sklearn-contrib-lightning' - displayName: 'Install lightning' - - - script: 'pip install git+https://github.com/slundberg/shap.git@d1d2700acc0259f211934373826d5ff71ad514de' - displayName: 'Install specific version of shap' - - - script: 'pip install "sphinx < 3.2" sphinx_rtd_theme' - displayName: 'Install sphinx' - - - script: 'python setup.py build_sphinx -W' - displayName: 'Build documentation' - - - publish: 'build/sphinx/html' - artifact: 'Documentation' - displayName: 'Publish documentation as artifact' - - - script: 'python setup.py build_sphinx -b doctest' - displayName: 'Run doctests' - - template: azure-pipelines-steps.yml parameters: versions: ['3.8'] images: ['ubuntu-18.04'] - package: '-e .[tf]' - job: - job: 'Notebooks' - dependsOn: 'EvalChanges' - condition: eq(dependencies.EvalChanges.outputs['output.buildNbs'], 'True') - steps: - # Work around https://github.com/pypa/pip/issues/9542 - - script: 'pip install -U numpy' - displayName: 'Upgrade numpy' - - - script: 'pip install pytest-runner && python setup.py pytest' - displayName: 'Unit tests' - env: - PYTEST_ADDOPTS: '-m "notebook"' - - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/test-results.xml' - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Notebooks' - condition: succeededOrFailed() - -# - job: 'AutoML' -# dependsOn: 'EvalChanges' -# condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') -# variables: -# python.version: '3.6' -# pool: -# vmImage: 'ubuntu-18.04' -# steps: -# - template: azure-pipelines-steps.yml -# parameters: -# body: -# - task: AzureCLI@2 -# displayName: 'AutoML tests' -# inputs: -# azureSubscription: 'automl' -# scriptLocation: 'inlineScript' -# scriptType: 'pscore' -# powerShellIgnoreLASTEXITCODE: '' # string for now due to https://github.com/microsoft/azure-pipelines-tasks/issues/12266 -# inlineScript: | -# $env:SUBSCRIPTION_ID = az account show --query id -o tsv -# python setup.py pytest -# env: -# WORKSPACE_NAME: 'testWorkspace' -# RESOURCE_GROUP: 'testingAutoMLEconML' -# PYTEST_ADDOPTS: '-m "automl" -n 0' -# COVERAGE_PROCESS_START: 'setup.cfg' - -# - task: PublishTestResults@2 -# displayName: 'Publish Test Results **/test-results.xml' -# inputs: -# testResultsFiles: '**/test-results.xml' -# testRunTitle: 'AutoML' -# condition: succeededOrFailed() -# package: '.[automl]' - -- template: azure-pipelines-steps.yml - parameters: - versions: ['3.8'] - images: ['macOS-10.15'] - job: - job: 'Linting' - dependsOn: 'EvalChanges' - condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') - steps: - - script: 'pip install pycodestyle && pycodestyle econml' - failOnStderr: true - displayName: Linting - -- template: azure-pipelines-steps.yml - parameters: package: '-e .[tf,plt]' job: job: Tests_no_dml @@ -164,10 +59,10 @@ jobs: condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') displayName: 'Run tests (non-DML)' steps: - - script: 'pip install pytest-runner && python setup.py pytest' + - script: 'pip install pytest-runner filprofiler && fil-profile run setup.py pytest' displayName: 'Unit tests' env: - PYTEST_ADDOPTS: '-m "not (notebook or automl or dml)"' + PYTEST_ADDOPTS: '-m "not (notebook or automl or dml or half)"' COVERAGE_PROCESS_START: 'setup.cfg' - task: PublishTestResults@2 displayName: 'Publish Test Results **/test-results.xml' @@ -175,26 +70,25 @@ jobs: testResultsFiles: '**/test-results.xml' testRunTitle: 'Python $(python.version), image $(imageName)' condition: succeededOrFailed() - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Code Coverage Results' - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + - publish: 'fil-result' + artifact: 'Profiling' + displayName: 'Publish profiling results as artifact' - template: azure-pipelines-steps.yml parameters: + versions: ['3.8'] + images: ['ubuntu-18.04'] package: '-e .[tf,plt]' job: - job: Tests_dml + job: Tests_no_dml_half dependsOn: 'EvalChanges' condition: eq(dependencies.EvalChanges.outputs['output.testCode'], 'True') - displayName: 'Run tests (DML)' + displayName: 'Run tests (non-DML)' steps: - - script: 'pip install pytest-runner && python setup.py pytest' + - script: 'pip install pytest-runner filprofiler && fil-profile run setup.py pytest' displayName: 'Unit tests' env: - PYTEST_ADDOPTS: '-m "dml and (not (notebook or automl))"' + PYTEST_ADDOPTS: '-m "half and (not (notebook or automl or dml))"' COVERAGE_PROCESS_START: 'setup.cfg' - task: PublishTestResults@2 displayName: 'Publish Test Results **/test-results.xml' @@ -202,9 +96,6 @@ jobs: testResultsFiles: '**/test-results.xml' testRunTitle: 'Python $(python.version), image $(imageName)' condition: succeededOrFailed() - - - task: PublishCodeCoverageResults@1 - displayName: 'Publish Code Coverage Results' - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + - publish: 'fil-result' + artifact: 'Profiling' + displayName: 'Publish profiling results as artifact' diff --git a/econml/tests/test_ate_inference.py b/econml/tests/test_ate_inference.py index e6b9edaeb..c526270ec 100644 --- a/econml/tests/test_ate_inference.py +++ b/econml/tests/test_ate_inference.py @@ -9,6 +9,7 @@ from econml.inference import BootstrapInference +@pytest.mark.half class TestATEInference(unittest.TestCase): @classmethod diff --git a/econml/tests/test_bootstrap.py b/econml/tests/test_bootstrap.py index c410ebc6e..cb9d1514c 100644 --- a/econml/tests/test_bootstrap.py +++ b/econml/tests/test_bootstrap.py @@ -13,6 +13,7 @@ import joblib +@pytest.mark.half class TestBootstrap(unittest.TestCase): def test_with_sklearn(self): diff --git a/econml/tests/test_dowhy.py b/econml/tests/test_dowhy.py index f007e4b0f..cb721e894 100644 --- a/econml/tests/test_dowhy.py +++ b/econml/tests/test_dowhy.py @@ -11,6 +11,7 @@ from sklearn.linear_model import LinearRegression, LogisticRegression, Lasso +@pytest.mark.half class TestDowhy(unittest.TestCase): def _get_data(self): diff --git a/econml/tests/test_drlearner.py b/econml/tests/test_drlearner.py index 132c3b9d5..1fffd6875 100644 --- a/econml/tests/test_drlearner.py +++ b/econml/tests/test_drlearner.py @@ -24,6 +24,7 @@ import econml.tests.utilities # bugfix for assertWarns +@pytest.mark.half class TestDRLearner(unittest.TestCase): @classmethod diff --git a/econml/tests/test_linear_model.py b/econml/tests/test_linear_model.py index fda93d5c9..55e85e753 100644 --- a/econml/tests/test_linear_model.py +++ b/econml/tests/test_linear_model.py @@ -16,6 +16,7 @@ from sklearn.base import clone +@pytest.mark.half class TestLassoExtensions(unittest.TestCase): """Test WeightedLasso.""" diff --git a/econml/tests/test_montecarlo.py b/econml/tests/test_montecarlo.py index 323f4bd46..487271fea 100644 --- a/econml/tests/test_montecarlo.py +++ b/econml/tests/test_montecarlo.py @@ -12,6 +12,7 @@ import numpy as np +@pytest.mark.half class TestMonteCarlo(unittest.TestCase): def test_montecarlo(self): diff --git a/econml/tests/test_orf.py b/econml/tests/test_orf.py index df9b4987b..b149cd63a 100644 --- a/econml/tests/test_orf.py +++ b/econml/tests/test_orf.py @@ -14,6 +14,7 @@ from econml.sklearn_extensions.linear_model import WeightedLassoCVWrapper +@pytest.mark.half class TestOrthoForest(unittest.TestCase): @classmethod diff --git a/econml/tests/test_ortho_learner.py b/econml/tests/test_ortho_learner.py index cda7c8f9c..960039de7 100644 --- a/econml/tests/test_ortho_learner.py +++ b/econml/tests/test_ortho_learner.py @@ -12,6 +12,7 @@ import pytest +@pytest.mark.half class TestOrthoLearner(unittest.TestCase): def test_crossfit(self): diff --git a/econml/tests/test_rscorer.py b/econml/tests/test_rscorer.py index 93f4779a9..66813a720 100644 --- a/econml/tests/test_rscorer.py +++ b/econml/tests/test_rscorer.py @@ -19,6 +19,7 @@ def _fit_model(name, model, Y, T, X): return name, model.fit(Y, T, X=X) +@pytest.mark.half class TestRScorer(unittest.TestCase): def _get_data(self): diff --git a/econml/tests/test_statsmodels.py b/econml/tests/test_statsmodels.py index fd8c6bb6e..5827bbe53 100644 --- a/econml/tests/test_statsmodels.py +++ b/econml/tests/test_statsmodels.py @@ -21,6 +21,7 @@ from sklearn.preprocessing import PolynomialFeatures +@pytest.mark.half class StatsModelsOLS: """ Helper class to wrap a StatsModels OLS model to conform to the sklearn API. diff --git a/econml/tests/test_two_stage_least_squares.py b/econml/tests/test_two_stage_least_squares.py index dab526c48..bc228f662 100644 --- a/econml/tests/test_two_stage_least_squares.py +++ b/econml/tests/test_two_stage_least_squares.py @@ -14,6 +14,7 @@ from sklearn.preprocessing import PolynomialFeatures +@pytest.mark.half class Test2SLS(unittest.TestCase): def test_hermite_shape(self): diff --git a/pyproject.toml b/pyproject.toml index 1d29bce41..10d38ab5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,10 +9,11 @@ requires = [ build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -addopts = "--junitxml=junit/test-results.xml -n auto --strict-markers --cov-config=setup.cfg --cov=econml --cov-report=xml" +addopts = "--junitxml=junit/test-results.xml --strict-markers" markers = [ "slow", "notebook", "automl", - "dml" + "dml", + "half" ] \ No newline at end of file