From 838c59b65ee751486e3bec0b51d42b81235faab7 Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Tue, 22 Oct 2024 14:46:15 -0400 Subject: [PATCH 1/5] Install plugins on Index and IndexGRPC --- pinecone/data/index.py | 35 ++++++++++++++++++++++++++++++----- pinecone/grpc/base.py | 18 ++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/pinecone/data/index.py b/pinecone/data/index.py index cc1bae6c..e542b962 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -1,11 +1,11 @@ from tqdm.autonotebook import tqdm +import logging from typing import Union, List, Optional, Dict, Any from pinecone.config import ConfigBuilder from pinecone.core.openapi.shared import API_VERSION -from pinecone.core.openapi.data.models import SparseValues from pinecone.core.openapi.data import ApiClient from pinecone.core.openapi.data.models import ( FetchResponse, @@ -22,12 +22,22 @@ UpdateRequest, DescribeIndexStatsRequest, ListResponse, + SparseValues, ) -from .features.bulk_import import ImportFeatureMixin from pinecone.core.openapi.data.api.data_plane_api import DataPlaneApi -from ..utils import setup_openapi_client, parse_non_empty_args +from ..utils import ( + setup_openapi_client, + parse_non_empty_args, + build_plugin_setup_client, + validate_and_convert_errors, +) +from .features.bulk_import import ImportFeatureMixin from .vector_factory import VectorFactory +from pinecone_plugin_interface import load_and_install as install_plugins + +logger = logging.getLogger(__name__) + __all__ = [ "Index", "FetchResponse", @@ -47,8 +57,6 @@ "SparseValues", ] -from ..utils.error_handling import validate_and_convert_errors - _OPENAPI_ENDPOINT_PARAMS = ( "_return_http_data_only", "_preload_content", @@ -103,6 +111,23 @@ def __init__( api_version=API_VERSION, ) + self._load_plugins() + + def _load_plugins(self): + """@private""" + try: + # I don't expect this to ever throw, but wrapping this in a + # try block just in case to make sure a bad plugin doesn't + # halt client initialization. + openapi_client_builder = build_plugin_setup_client( + config=self.config, + openapi_config=self.openapi_config, + pool_threads=self.pool_threads, + ) + install_plugins(self, openapi_client_builder) + except Exception as e: + logger.error(f"Error loading plugins in Index: {e}") + def __enter__(self): return self diff --git a/pinecone/grpc/base.py b/pinecone/grpc/base.py index 17580d7e..187afd58 100644 --- a/pinecone/grpc/base.py +++ b/pinecone/grpc/base.py @@ -1,6 +1,7 @@ from abc import ABC, abstractmethod from typing import Optional +import logging import grpc from grpc._channel import Channel @@ -10,6 +11,10 @@ from .config import GRPCClientConfig from .grpc_runner import GrpcRunner +from pinecone_plugin_interface import load_and_install as install_plugins + +_logger = logging.getLogger(__name__) + class GRPCIndexBase(ABC): """ @@ -40,6 +45,19 @@ def __init__( self._channel = channel or self._gen_channel() self.stub = self.stub_class(self._channel) + self._load_plugins() + + def _load_plugins(self): + """@private""" + try: + + def stub_openapi_client_builder(**kwargs): + pass + + install_plugins(self, stub_openapi_client_builder) + except Exception as e: + _logger.error(f"Error loading plugins in GRPCIndex: {e}") + @property @abstractmethod def stub_class(self): From d93f5972071b83cc5012dc99e12cbde1dc4e0afc Mon Sep 17 00:00:00 2001 From: Jen Hamon Date: Tue, 22 Oct 2024 15:08:13 -0400 Subject: [PATCH 2/5] Fix import issue --- pinecone/grpc/base.py | 2 +- pinecone/utils/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pinecone/grpc/base.py b/pinecone/grpc/base.py index 187afd58..cc4ca2c6 100644 --- a/pinecone/grpc/base.py +++ b/pinecone/grpc/base.py @@ -51,7 +51,7 @@ def _load_plugins(self): """@private""" try: - def stub_openapi_client_builder(**kwargs): + def stub_openapi_client_builder(*args, **kwargs): pass install_plugins(self, stub_openapi_client_builder) diff --git a/pinecone/utils/__init__.py b/pinecone/utils/__init__.py index 999daabf..34e1d5aa 100644 --- a/pinecone/utils/__init__.py +++ b/pinecone/utils/__init__.py @@ -9,6 +9,7 @@ from .parse_args import parse_non_empty_args from .docslinks import docslinks from .repr_overrides import install_json_repr_override +from .error_handling import validate_and_convert_errors __all__ = [ "check_kwargs", @@ -23,4 +24,5 @@ "parse_non_empty_args", "docslinks", "install_json_repr_override", + "validate_and_convert_errors", ] From 65d519772424fcff9ead434e52364f8021f6d0a3 Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Fri, 25 Oct 2024 00:48:18 -0400 Subject: [PATCH 3/5] access self._config rather than self.config in Index --- pinecone/data/index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinecone/data/index.py b/pinecone/data/index.py index e542b962..1a011b81 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -120,7 +120,7 @@ def _load_plugins(self): # try block just in case to make sure a bad plugin doesn't # halt client initialization. openapi_client_builder = build_plugin_setup_client( - config=self.config, + config=self._config, openapi_config=self.openapi_config, pool_threads=self.pool_threads, ) From a0fb2879a48084158215f311c92faa4b074030f7 Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Fri, 25 Oct 2024 01:49:17 -0400 Subject: [PATCH 4/5] thread openapi_config and pool_threads through as well --- pinecone/data/index.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pinecone/data/index.py b/pinecone/data/index.py index 1a011b81..07845aac 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -100,14 +100,15 @@ def __init__( self._config = ConfigBuilder.build( api_key=api_key, host=host, additional_headers=additional_headers, **kwargs ) - openapi_config = ConfigBuilder.build_openapi_config(self._config, openapi_config) + self._openapi_config = ConfigBuilder.build_openapi_config(self._config, openapi_config) + self._pool_threads = pool_threads self._vector_api = setup_openapi_client( api_client_klass=ApiClient, api_klass=DataPlaneApi, config=self._config, - openapi_config=openapi_config, - pool_threads=pool_threads, + openapi_config=self._openapi_config, + pool_threads=self._pool_threads, api_version=API_VERSION, ) @@ -121,8 +122,8 @@ def _load_plugins(self): # halt client initialization. openapi_client_builder = build_plugin_setup_client( config=self._config, - openapi_config=self.openapi_config, - pool_threads=self.pool_threads, + openapi_config=self._openapi_config, + pool_threads=self._pool_threads, ) install_plugins(self, openapi_client_builder) except Exception as e: From 90df95544179b9d10170b3edf954410bf49a7ac6 Mon Sep 17 00:00:00 2001 From: Austin DeNoble Date: Fri, 25 Oct 2024 10:40:03 -0400 Subject: [PATCH 5/5] rebase + swap to using Index.config instead of Index._config since the python-plugin-interface expects that field on target --- pinecone/data/index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pinecone/data/index.py b/pinecone/data/index.py index 07845aac..3f63d046 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -97,16 +97,16 @@ def __init__( **kwargs, ) - self._config = ConfigBuilder.build( + self.config = ConfigBuilder.build( api_key=api_key, host=host, additional_headers=additional_headers, **kwargs ) - self._openapi_config = ConfigBuilder.build_openapi_config(self._config, openapi_config) + self._openapi_config = ConfigBuilder.build_openapi_config(self.config, openapi_config) self._pool_threads = pool_threads self._vector_api = setup_openapi_client( api_client_klass=ApiClient, api_klass=DataPlaneApi, - config=self._config, + config=self.config, openapi_config=self._openapi_config, pool_threads=self._pool_threads, api_version=API_VERSION, @@ -121,7 +121,7 @@ def _load_plugins(self): # try block just in case to make sure a bad plugin doesn't # halt client initialization. openapi_client_builder = build_plugin_setup_client( - config=self._config, + config=self.config, openapi_config=self._openapi_config, pool_threads=self._pool_threads, )