From e575fce3396e20990b985d485cc75eb3a801db3a Mon Sep 17 00:00:00 2001 From: David Hall Date: Fri, 4 Oct 2024 11:33:39 -0700 Subject: [PATCH] make deepdiff optional --- src/levanter/store/cache.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/levanter/store/cache.py b/src/levanter/store/cache.py index eae9f8402..836f4db17 100644 --- a/src/levanter/store/cache.py +++ b/src/levanter/store/cache.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, TypeVar, Union -import deepdiff +# import deepdiff import fsspec.core import pyarrow as pa import ray @@ -508,7 +508,7 @@ class CacheMetadata: options: CacheOptions = CacheOptions.default() preprocessor_metadata: Optional[dict[str, Any]] = None - def compare_to(self, other: "CacheMetadata") -> deepdiff.DeepDiff: + def compare_to(self, other: "CacheMetadata"): """ Compare this metadata to another set of metadata. This is used to check if the cache being loaded was created with the expected configuration. @@ -519,7 +519,15 @@ def compare_to(self, other: "CacheMetadata") -> deepdiff.DeepDiff: sorta_self = dataclasses.replace(self, preprocessor_metadata=None) else: sorta_self = self - return deepdiff.DeepDiff(sorta_self, other) + try: + import deepdiff + + return deepdiff.DeepDiff(sorta_self, other) + except Exception as e: + if "deepdiff" in str(e): + if sorta_self != other: + return {"message": "Metadata mismatch"} + return {} @staticmethod def empty():