From 508d9f3c28125840ff8f0a2a8df0c05d7a2438b6 Mon Sep 17 00:00:00 2001 From: Mark Roth Date: Wed, 27 Mar 2024 19:19:54 -0400 Subject: [PATCH] Fix import (#14) * Bump to 0.28.1 * Fix namespace package in dist * Upgrade jinja2 (security patch) * Remove .gitlab-ci.yml * Fix test failures due to static test methods --- .gitignore | 1 + .gitlab-ci.yml | 54 ---------------------- pyproject.toml | 6 +-- tests/test_configuration.py | 3 +- tests/test_runner_backend.py | 24 ++++------ tests/test_runner_null.py | 3 +- tests/test_storage_backend.py | 84 ++++++++++++----------------------- twosigma/memento/__about__.py | 2 +- 8 files changed, 43 insertions(+), 134 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitignore b/.gitignore index d927ef5..bff6583 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ memento/twosigma/memento/generated/ .settings .classpath .project +venv/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 1972bf8..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,54 +0,0 @@ -variables: - DOCKER_IMAGE_TAG: "$CI_PIPELINE_ID.$CI_COMMIT_SHORT_SHA" - QA_GCP_PROJECT_NAME: "ts-alphastudio-qa" - QA_DATAGEN_CLUSTER_NAME: "datagen" - QA_CLUSTER_REGION: "us-east4-a" - -include: - - /memento_server_client/.gitlab-ci.yml - -image: gcr.io/ts-quantsource/as-gitlab-runner:20210911-005939 - -stages: - - build_client - - build_container - - build - - test - - deploy - -build-and-push-docker-containers: - stage: build_container - needs: - - job: "memento_server_client:bundle" - artifacts: true - script: - - PUSH=true memento_server/build.sh $DOCKER_IMAGE_TAG - -build-39: - stage: build - script: - - CONDA_ENV="py39" EXPECTED_PYTHON_VERSION="Python 3.9" ./build.sh - needs: - - build-and-push-docker-containers - except: - refs: - - master - -build-and-push-39: - stage: deploy - script: - - PUSH=true RUN_LONG_TESTS=true CONDA_ENV="py39" EXPECTED_PYTHON_VERSION="Python 3.9" ./build.sh - needs: - - build-and-push-docker-containers - only: - refs: - - master - -memento_server:tests: - stage: test - needs: [] - before_script: - - useradd -m memento - - cd memento_server/ - script: - - su memento -c './gradlew check --info' diff --git a/pyproject.toml b/pyproject.toml index 32707ab..c20a716 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "graphviz>=0.13", "pandas>=1.0.5", "numpy>=1.15.4", - "jinja2>=2.10", + "jinja2>=3.1.3", "python-dateutil>=2.8", "pyyaml>=3.13", "tqdm>=4.31.1" @@ -40,7 +40,7 @@ Issues = "https://github.com/twosigma/memento/issues" Source = "https://github.com/twosigma/memento" [tool.hatch.build.targets.wheel] -packages = ["twosigma/memento"] +packages = ["twosigma"] [tool.hatch.version] path = "twosigma/memento/__about__.py" @@ -75,4 +75,4 @@ exclude_lines = [ markers = [ "slow: marks tests as slow", "needs_canonical_version: marks tests as needing to run with a specific version of python, for hash stability" -] \ No newline at end of file +] diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 8b35ede..6a6cf0d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -91,8 +91,7 @@ def test_environment_set_file_yaml(self): if d: shutil.rmtree(d) - @staticmethod - def get_sample_config(): + def get_sample_config(self): return { "name": "config1", "description": "description1", diff --git a/tests/test_runner_backend.py b/tests/test_runner_backend.py index 31f54b6..a7ec4e8 100644 --- a/tests/test_runner_backend.py +++ b/tests/test_runner_backend.py @@ -54,8 +54,7 @@ def setup_method(self): def teardown_method(self): pass - @staticmethod - def test_memoize(): + def test_memoize(self): # This also tests serializing function references assert { "a": 1, @@ -68,8 +67,7 @@ def test_memoize(): 1, c=[{"three": 3}], b=2, e=runner_fn_test_1, f=[{"a": runner_fn_test_1}] ) - @staticmethod - def test_call_stack_invocations_tracked(): + def test_call_stack_invocations_tracked(self): # runner_fn_test_apply_and_double --> # runner_fn_test_apply_and_double --> # runner_fn_test_add @@ -102,8 +100,7 @@ def test_call_stack_invocations_tracked(): == invocations[1].fn_reference.function_name ) - @staticmethod - def test_correlation_id(): + def test_correlation_id(self): """ Test that a correlation id is generated and properly propagated @@ -119,8 +116,7 @@ def test_correlation_id(): corr_id_2 = memento_2.correlation_id assert corr_id_1 == corr_id_2 - @staticmethod - def test_prevent_further_calls(): + def test_prevent_further_calls(self): """ Test that prevent_further_calls in recursive context prevents additional Memento calls. @@ -135,8 +131,7 @@ def test_prevent_further_calls(): # the error, that means the RuntimeError happened on the wrong call (too early). assert "Caught further calls error" in str(e) - @staticmethod - def test_refuse_execution_of_non_dependencies(): + def test_refuse_execution_of_non_dependencies(self): """ Test that Memento refuses to execute a down-stream function that is not declared or detected as a dependency in the parent chain. @@ -148,8 +143,7 @@ def test_refuse_execution_of_non_dependencies(): # But allowed if explicit version is set: assert 1 == fn_with_explicit_version_calls_undeclared_dependency() - @staticmethod - def test_key_override_result(): + def test_key_override_result(self): """ Test that, if a function returns `KeyOverrideResult`, the result is stored at the overridden key instead of the standard location. @@ -159,8 +153,7 @@ def test_key_override_result(): mem = fn_returns_key_override_result.memento() assert "custom_key" == mem.content_key.key - @staticmethod - def test_memento_dependencies(): + def test_memento_dependencies(self): """ Test that, given a -> b -> c, we can extract the full call stack from the memento. In this test, we make a -> b a batch call so that we override any speculative local @@ -184,8 +177,7 @@ def test_memento_dependencies(): == i_b.invocations[0].fn_reference.qualified_name ) - @staticmethod - def test_memento_graph(): + def test_memento_graph(self): """ Test that, given a -> b -> c, we can extract the full call stack from the memento. In this test, we make a -> b a batch call so that we override any speculative local diff --git a/tests/test_runner_null.py b/tests/test_runner_null.py index 3bcec05..57ca97a 100644 --- a/tests/test_runner_null.py +++ b/tests/test_runner_null.py @@ -72,7 +72,6 @@ def teardown_method(self): shutil.rmtree(self.base_path) m.Environment.set(self.original_env) - @staticmethod - def test_null_runner(): + def test_null_runner(self): with pytest.raises(RuntimeError): fn1() diff --git a/tests/test_storage_backend.py b/tests/test_storage_backend.py index db6de31..2d694a8 100644 --- a/tests/test_storage_backend.py +++ b/tests/test_storage_backend.py @@ -436,72 +436,63 @@ def test_memoize(self): fn1_reference.fn_reference, fn1_reference.arg_hash ) - @staticmethod - def test_memoize_exception(): + def test_memoize_exception(self): assert fn_raise_value_error.memento() is None with pytest.raises(ValueError): fn_raise_value_error() with pytest.raises(ValueError): fn_raise_value_error() - @staticmethod - def test_memoize_null(): + def test_memoize_null(self): assert fn_return_none_1.memento() is None first = fn_return_none_1() assert fn_return_none_1.memento() is not None second = fn_return_none_1() assert first == second - @staticmethod - def test_memoize_str(): + def test_memoize_str(self): assert fn_return_str.memento() is None first = fn_return_str() assert fn_return_str.memento() is not None second = fn_return_str() assert first == second - @staticmethod - def test_memoize_int(): + def test_memoize_int(self): assert fn_return_int.memento() is None first = fn_return_int() assert fn_return_int.memento() is not None second = fn_return_int() assert first == second - @staticmethod - def test_memoize_float(): + def test_memoize_float(self): assert fn_return_float.memento() is None first = fn_return_float() assert fn_return_float.memento() is not None second = fn_return_float() assert first == second - @staticmethod - def test_memoize_bool(): + def test_memoize_bool(self): assert fn_return_bool.memento() is None first = fn_return_bool() assert fn_return_bool.memento() is not None second = fn_return_bool() assert first == second - @staticmethod - def test_memoize_date(): + def test_memoize_date(self): assert fn_return_date.memento() is None first = fn_return_date() assert fn_return_date.memento() is not None second = fn_return_date() assert first == second - @staticmethod - def test_memoize_time(): + def test_memoize_time(self): assert fn_return_time.memento() is None first = fn_return_time() assert fn_return_time.memento() is not None second = fn_return_time() assert first == second - @staticmethod - def test_memoize_dictionary(): + def test_memoize_dictionary(self): assert fn_return_dictionary.memento() is None first = fn_return_dictionary() assert fn_return_dictionary.memento() is not None @@ -510,8 +501,7 @@ def test_memoize_dictionary(): assert first[key] == second[key] assert first["g"].equals(second["g"]) - @staticmethod - def test_memoize_list(): + def test_memoize_list(self): assert fn_return_list.memento() is None first = fn_return_list() assert fn_return_list.memento() is not None @@ -520,120 +510,105 @@ def test_memoize_list(): assert first[x] == second[x] assert first[6].equals(second[6]) - @staticmethod - def test_memoize_2d_array(): + def test_memoize_2d_array(self): assert fn_return_2d_array.memento() is None first = fn_return_2d_array() assert fn_return_2d_array.memento() is not None second = fn_return_2d_array() assert first == second - @staticmethod - def test_numpy_array_bool(): + def test_numpy_array_bool(self): assert fn_return_numpy_array_bool.memento() is None first = fn_return_numpy_array_bool() assert fn_return_numpy_array_bool.memento() is not None second = fn_return_numpy_array_bool() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_int8(): + def test_numpy_array_int8(self): assert fn_return_numpy_array_int8.memento() is None first = fn_return_numpy_array_int8() assert fn_return_numpy_array_int8.memento() is not None second = fn_return_numpy_array_int8() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_int16(): + def test_numpy_array_int16(self): assert fn_return_numpy_array_int16.memento() is None first = fn_return_numpy_array_int16() assert fn_return_numpy_array_int16.memento() is not None second = fn_return_numpy_array_int16() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_int32(): + def test_numpy_array_int32(self): assert fn_return_numpy_array_int32.memento() is None first = fn_return_numpy_array_int32() assert fn_return_numpy_array_int32.memento() is not None second = fn_return_numpy_array_int32() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_int64(): + def test_numpy_array_int64(self): assert fn_return_numpy_array_int64.memento() is None first = fn_return_numpy_array_int64() assert fn_return_numpy_array_int64.memento() is not None second = fn_return_numpy_array_int64() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_float32(): + def test_numpy_array_float32(self): assert fn_return_numpy_array_float32.memento() is None first = fn_return_numpy_array_float32() assert fn_return_numpy_array_float32.memento() is not None second = fn_return_numpy_array_float32() assert np.array_equal(first, second) - @staticmethod - def test_numpy_array_float64(): + def test_numpy_array_float64(self): assert fn_return_numpy_array_float64.memento() is None first = fn_return_numpy_array_float64() assert fn_return_numpy_array_float64.memento() is not None second = fn_return_numpy_array_float64() assert np.array_equal(first, second) - @staticmethod - def test_memoize_index(): + def test_memoize_index(self): assert fn_return_index.memento() is None first = fn_return_index() assert fn_return_index.memento() is not None second = fn_return_index() assert first.equals(second) - @staticmethod - def test_memoize_series(): + def test_memoize_series(self): assert fn_return_series.memento() is None first = fn_return_series() assert fn_return_series.memento() is not None second = fn_return_series() assert_series_equal(first, second) - @staticmethod - def test_memoize_series_with_index(): + def test_memoize_series_with_index(self): assert fn_return_series_with_index.memento() is None first = fn_return_series_with_index() assert fn_return_series_with_index.memento() is not None second = fn_return_series_with_index() assert_series_equal(first, second) - @staticmethod - def test_memoize_series_with_multiindex(): + def test_memoize_series_with_multiindex(self): assert fn_return_series_with_multiindex.memento() is None first = fn_return_series_with_multiindex() assert fn_return_series_with_multiindex.memento() is not None second = fn_return_series_with_multiindex() assert_series_equal(first, second) - @staticmethod - def test_memoize_dataframe(): + def test_memoize_dataframe(self): assert fn_return_dataframe.memento() is None first = fn_return_dataframe() assert fn_return_dataframe.memento() is not None second = fn_return_dataframe() assert first.equals(second) - @staticmethod - def test_memoize_dataframe_with_index(): + def test_memoize_dataframe_with_index(self): assert fn_return_dataframe_with_index.memento() is None first = fn_return_dataframe_with_index() assert fn_return_dataframe_with_index.memento() is not None second = fn_return_dataframe_with_index() assert first.equals(second) - @staticmethod - def test_memoize_partition(): + def test_memoize_partition(self): assert fn_return_partition.memento() is None first = fn_return_partition() assert fn_return_partition.memento() is not None @@ -795,8 +770,7 @@ def test_readonly(self): self.backend.forget_call(fn_ref_1.fn_reference_with_arg_hash()) assert self.backend.is_memoized(fn_ref_1.fn_reference, fn_ref_1.arg_hash) - @staticmethod - def test_metadata_rw(): + def test_metadata_rw(self): fn_add_one(1) memento = fn_add_one.memento(1) @@ -824,8 +798,7 @@ def test_metadata_rw(): is None ) - @staticmethod - def test_metadata_rw_store_with_data(): + def test_metadata_rw_store_with_data(self): fn_add_one(1) memento = fn_add_one.memento(1) @@ -853,8 +826,7 @@ def test_metadata_rw_store_with_data(): is None ) - @staticmethod - def test_content_addressable_storage(): + def test_content_addressable_storage(self): r1 = fn1(100) r2 = fn2(100) assert r1 == r2 diff --git a/twosigma/memento/__about__.py b/twosigma/memento/__about__.py index 381bc76..e8fc291 100644 --- a/twosigma/memento/__about__.py +++ b/twosigma/memento/__about__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.28" +__version__ = "0.28.1"