From d9f365bcc0f6128d3c903e0e0b273d880c35f768 Mon Sep 17 00:00:00 2001 From: Jennifer Hamon Date: Mon, 18 Nov 2024 11:12:03 -0500 Subject: [PATCH] [Feat] Helpful error when incorrectly instantiating Index client (#418) ## Problem Sometimes users incorrectly instantiate the Index client like this: ```python import pinecone # Initialize the Index with the host index = pinecone.Index(index_name, host=index_host) ``` Then they will later get an authentication exception when using it, since the `Index` class does not have the configuration values it needs when attempting to perform vector operations. ```python ForbiddenException: (403) Reason: Forbidden HTTP response headers: HTTPHeaderDict({'Date': 'Wed, 13 Nov 2024 02:06:45 GMT', 'Content-Type': 'text/plain', 'Content-Length': '9', 'Connection': 'keep-alive', 'x-pinecone-auth-rejected-reason': 'Wrong API key', 'www-authenticate': 'Wrong API key', 'server': 'envoy'}) HTTP response body: Forbidden ``` ## Solution - Rename the `Index` implementation to `_Index` so that people will not accidentally interact with it. - Add a new stub implementation for `Index` that throws an informative message. Bonus: - Move the Index client docstrings off the implementation class and into a related abstract base class. This wasn't strictly necessary, but I was feeling a bit overwhelmed by the size of the `index.py` file. I copied this approach from the grpc module. pdoc seems to still find and render the docs okay when doing this. ## Usage The error message incorporates the args/kwargs the user was attempting to pass. One positional arg ```python >>> import pinecone >>> i = pinecone.Index('my-index') Traceback (most recent call last): File "", line 1, in File "/Users/jhamon/workspace/pinecone-python-client/pinecone/data/index.py", line 113, in __init__ raise IndexClientInstantiationError(args, kwargs) pinecone.data.index.IndexClientInstantiationError: You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. INCORRECT USAGE: ``` import pinecone pc = pinecone.Pinecone(api_key='your-api-key') index = pinecone.Index('my-index') ``` CORRECT USAGE: ``` from pinecone import Pinecone pc = Pinecone(api_key='your-api-key') index = pc.Index('my-index') ``` ``` Multiple positional args ```python >>> i = pinecone.Index('my-index', 'https://my-index.blahblah.com') Traceback (most recent call last): File "", line 1, in File "/Users/jhamon/workspace/pinecone-python-client/pinecone/data/index.py", line 113, in __init__ raise IndexClientInstantiationError(args, kwargs) pinecone.data.index.IndexClientInstantiationError: You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. INCORRECT USAGE: ``` import pinecone pc = pinecone.Pinecone(api_key='your-api-key') index = pinecone.Index('my-index', 'https://my-index.blahblah.com') ``` CORRECT USAGE: ``` from pinecone import Pinecone pc = Pinecone(api_key='your-api-key') index = pc.Index('my-index', 'https://my-index.blahblah.com') ``` ``` One keyword arg: ```python >>> i = pinecone.Index(host='https://my-index.blahblah.com') Traceback (most recent call last): File "", line 1, in File "/Users/jhamon/workspace/pinecone-python-client/pinecone/data/index.py", line 113, in __init__ raise IndexClientInstantiationError(args, kwargs) pinecone.data.index.IndexClientInstantiationError: You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. INCORRECT USAGE: ``` import pinecone pc = pinecone.Pinecone(api_key='your-api-key') index = pinecone.Index(host='https://my-index.blahblah.com') ``` CORRECT USAGE: ``` from pinecone import Pinecone pc = Pinecone(api_key='your-api-key') index = pc.Index(host='https://my-index.blahblah.com') ``` ``` Multiple kwargs ```python >>> i = pinecone.Index(name='my-index', host='https://my-index.blahblah.com', pool_threads=20) Traceback (most recent call last): File "", line 1, in File "/Users/jhamon/workspace/pinecone-python-client/pinecone/data/index.py", line 113, in __init__ raise IndexClientInstantiationError(args, kwargs) pinecone.data.index.IndexClientInstantiationError: You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. INCORRECT USAGE: ``` import pinecone pc = pinecone.Pinecone(api_key='your-api-key') index = pinecone.Index(name='my-index', host='https://my-index.blahblah.com', pool_threads=20) ``` CORRECT USAGE: ``` from pinecone import Pinecone pc = Pinecone(api_key='your-api-key') index = pc.Index(name='my-index', host='https://my-index.blahblah.com', pool_threads=20) ``` ``` Mixed args/kwargs ```python >>> i = pinecone.Index('my-index', host='https://my-index.blahblah.com', pool_threads=20) Traceback (most recent call last): File "", line 1, in File "/Users/jhamon/workspace/pinecone-python-client/pinecone/data/index.py", line 113, in __init__ raise IndexClientInstantiationError(args, kwargs) pinecone.data.index.IndexClientInstantiationError: You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. INCORRECT USAGE: ``` import pinecone pc = pinecone.Pinecone(api_key='your-api-key') index = pinecone.Index('my-index', host='https://my-index.blahblah.com', pool_threads=20) ``` CORRECT USAGE: ``` from pinecone import Pinecone pc = Pinecone(api_key='your-api-key') index = pc.Index('my-index', host='https://my-index.blahblah.com', pool_threads=20) ``` ## Type of Change - [x] New feature (non-breaking change which adds functionality) - [x] Breaking change This change is a UX feature that could be considered breaking if someone was importing `Index` directly and going out of their way to set it up correctly. This was never documented usage, but somebody skilled at reading code could have figured out how to do this. --- pinecone/control/pinecone.py | 4 +- pinecone/data/index.py | 329 ++------------ pinecone/data/interfaces.py | 409 ++++++++++++++++++ .../serverless/test_index_instantiation_ux.py | 13 + tests/unit/test_index.py | 6 +- 5 files changed, 463 insertions(+), 298 deletions(-) create mode 100644 pinecone/data/interfaces.py create mode 100644 tests/integration/control/serverless/test_index_instantiation_ux.py diff --git a/pinecone/control/pinecone.py b/pinecone/control/pinecone.py index 6b51dd17..2216986f 100644 --- a/pinecone/control/pinecone.py +++ b/pinecone/control/pinecone.py @@ -27,7 +27,7 @@ from pinecone.models import ServerlessSpec, PodSpec, IndexModel, IndexList, CollectionList from .langchain_import_warnings import _build_langchain_attribute_error_message -from pinecone.data import Index +from pinecone.data import _Index from pinecone_plugin_interface import load_and_install as install_plugins @@ -788,7 +788,7 @@ def Index(self, name: str = "", host: str = "", **kwargs): # Otherwise, get host url from describe_index using the index name index_host = self.index_host_store.get_host(self.index_api, self.config, name) - return Index( + return _Index( host=index_host, api_key=api_key, pool_threads=pt, diff --git a/pinecone/data/index.py b/pinecone/data/index.py index d75b0da9..97063cd0 100644 --- a/pinecone/data/index.py +++ b/pinecone/data/index.py @@ -26,6 +26,7 @@ ListResponse, SparseValues, ) +from .interfaces import IndexInterface from .features.bulk_import import ImportFeatureMixin from ..utils import ( setup_openapi_client, @@ -45,6 +46,7 @@ __all__ = [ "Index", + "_Index", "FetchResponse", "QueryRequest", "QueryResponse", @@ -79,7 +81,39 @@ def parse_query_response(response: QueryResponse): return response -class Index(ImportFeatureMixin): +class IndexClientInstantiationError(Exception): + def __init__(self, index_args, index_kwargs): + formatted_args = ", ".join(map(repr, index_args)) + formatted_kwargs = ", ".join(f"{key}={repr(value)}" for key, value in index_kwargs.items()) + combined_args = ", ".join([a for a in [formatted_args, formatted_kwargs] if a.strip()]) + + self.message = f"""You are attempting to access the Index client directly from the pinecone module. The Index client must be instantiated through the parent Pinecone client instance so that it can inherit shared configurations such as API key. + + INCORRECT USAGE: + ``` + import pinecone + + pc = pinecone.Pinecone(api_key='your-api-key') + index = pinecone.Index({combined_args}) + ``` + + CORRECT USAGE: + ``` + from pinecone import Pinecone + + pc = Pinecone(api_key='your-api-key') + index = pc.Index({combined_args}) + ``` + """ + super().__init__(self.message) + + +class Index: + def __init__(self, *args, **kwargs): + raise IndexClientInstantiationError(args, kwargs) + + +class _Index(IndexInterface, ImportFeatureMixin): """ A client for interacting with a Pinecone index via REST API. For improved performance, use the Pinecone GRPC index client. @@ -153,58 +187,6 @@ def upsert( show_progress: bool = True, **kwargs, ) -> UpsertResponse: - """ - The upsert operation writes vectors into a namespace. - If a new value is upserted for an existing vector id, it will overwrite the previous value. - - To upsert in parallel follow: https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel - - A vector can be represented by a 1) Vector object, a 2) tuple or 3) a dictionary - - If a tuple is used, it must be of the form `(id, values, metadata)` or `(id, values)`. - where id is a string, vector is a list of floats, metadata is a dict, - and sparse_values is a dict of the form `{'indices': List[int], 'values': List[float]}`. - - Examples: - >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}, {'indices': [1, 2], 'values': [0.2, 0.4]}) - >>> ('id1', [1.0, 2.0, 3.0], None, {'indices': [1, 2], 'values': [0.2, 0.4]}) - >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0]) - - If a Vector object is used, a Vector object must be of the form - `Vector(id, values, metadata, sparse_values)`, where metadata and sparse_values are optional - arguments. - - Examples: - >>> Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}) - >>> Vector(id='id2', values=[1.0, 2.0, 3.0]) - >>> Vector(id='id3', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4])) - - **Note:** the dimension of each vector must match the dimension of the index. - - If a dictionary is used, it must be in the form `{'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]}, 'metadata': dict}` - - Examples: - >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])]) - >>> - >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}}, - >>> {'id': 'id2', 'values': [1.0, 2.0, 3.0], 'sparse_values': {'indices': [1, 8], 'values': [0.2, 0.4]}]) - >>> index.upsert([Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}), - >>> Vector(id='id2', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))]) - - API reference: https://docs.pinecone.io/reference/upsert - - Args: - vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert. - namespace (str): The namespace to write to. If not specified, the default namespace is used. [optional] - batch_size (int): The number of vectors to upsert in each batch. - If not specified, all vectors will be upserted in a single batch. [optional] - show_progress (bool): Whether to show a progress bar using tqdm. - Applied only if batch_size is provided. Default is True. - Keyword Args: - Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpsertRequest for more details. - - Returns: UpsertResponse, includes the number of vectors upserted. - """ _check_type = kwargs.pop("_check_type", True) if kwargs.get("async_req", False) and batch_size is not None: @@ -263,14 +245,6 @@ def _iter_dataframe(df, batch_size): def upsert_from_dataframe( self, df, namespace: Optional[str] = None, batch_size: int = 500, show_progress: bool = True ) -> UpsertResponse: - """Upserts a dataframe into the index. - - Args: - df: A pandas dataframe with the following columns: id, values, sparse_values, and metadata. - namespace: The namespace to upsert into. - batch_size: The number of rows to upsert in a single batch. - show_progress: Whether to show a progress bar. - """ try: import pandas as pd except ImportError: @@ -303,41 +277,6 @@ def delete( filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, **kwargs, ) -> Dict[str, Any]: - """ - The Delete operation deletes vectors from the index, from a single namespace. - No error raised if the vector id does not exist. - Note: for any delete call, if namespace is not specified, the default namespace is used. - - Delete can occur in the following mutual exclusive ways: - 1. Delete by ids from a single namespace - 2. Delete all vectors from a single namespace by setting delete_all to True - 3. Delete all vectors from a single namespace by specifying a metadata filter - (note that for this option delete all must be set to False) - - API reference: https://docs.pinecone.io/reference/delete_post - - Examples: - >>> index.delete(ids=['id1', 'id2'], namespace='my_namespace') - >>> index.delete(delete_all=True, namespace='my_namespace') - >>> index.delete(filter={'key': 'value'}, namespace='my_namespace') - - Args: - ids (List[str]): Vector ids to delete [optional] - delete_all (bool): This indicates that all vectors in the index namespace should be deleted.. [optional] - Default is False. - namespace (str): The namespace to delete vectors from [optional] - If not specified, the default namespace is used. - filter (Dict[str, Union[str, float, int, bool, List, dict]]): - If specified, the metadata filter here will be used to select the vectors to delete. - This is mutually exclusive with specifying ids to delete in the ids param or using delete_all=True. - See https://www.pinecone.io/docs/metadata-filtering/.. [optional] - - Keyword Args: - Supports OpenAPI client keyword arguments. See pinecone.core.client.models.DeleteRequest for more details. - - - Returns: An empty dictionary if the delete operation was successful. - """ _check_type = kwargs.pop("_check_type", False) args_dict = parse_non_empty_args( [("ids", ids), ("delete_all", delete_all), ("namespace", namespace), ("filter", filter)] @@ -358,26 +297,6 @@ def delete( @validate_and_convert_errors def fetch(self, ids: List[str], namespace: Optional[str] = None, **kwargs) -> FetchResponse: - """ - The fetch operation looks up and returns vectors, by ID, from a single namespace. - The returned vectors include the vector data and/or metadata. - - API reference: https://docs.pinecone.io/reference/fetch - - Examples: - >>> index.fetch(ids=['id1', 'id2'], namespace='my_namespace') - >>> index.fetch(ids=['id1', 'id2']) - - Args: - ids (List[str]): The vector IDs to fetch. - namespace (str): The namespace to fetch vectors from. - If not specified, the default namespace is used. [optional] - Keyword Args: - Supports OpenAPI client keyword arguments. See pinecone.core.client.models.FetchResponse for more details. - - - Returns: FetchResponse object which contains the list of Vector objects, and namespace name. - """ args_dict = parse_non_empty_args([("namespace", namespace)]) return self._vector_api.fetch_vectors(ids=ids, **args_dict, **kwargs) @@ -397,47 +316,6 @@ def query( ] = None, **kwargs, ) -> Union[QueryResponse, ApplyResult]: - """ - The Query operation searches a namespace, using a query vector. - It retrieves the ids of the most similar items in a namespace, along with their similarity scores. - - API reference: https://docs.pinecone.io/reference/query - - Examples: - >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace') - >>> index.query(id='id1', top_k=10, namespace='my_namespace') - >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace', filter={'key': 'value'}) - >>> index.query(id='id1', top_k=10, namespace='my_namespace', include_metadata=True, include_values=True) - >>> index.query(vector=[1, 2, 3], sparse_vector={'indices': [1, 2], 'values': [0.2, 0.4]}, - >>> top_k=10, namespace='my_namespace') - >>> index.query(vector=[1, 2, 3], sparse_vector=SparseValues([1, 2], [0.2, 0.4]), - >>> top_k=10, namespace='my_namespace') - - Args: - vector (List[float]): The query vector. This should be the same length as the dimension of the index - being queried. Each `query()` request can contain only one of the parameters - `id` or `vector`.. [optional] - id (str): The unique ID of the vector to be used as a query vector. - Each `query()` request can contain only one of the parameters - `vector` or `id`. [optional] - top_k (int): The number of results to return for each query. Must be an integer greater than 1. - namespace (str): The namespace to fetch vectors from. - If not specified, the default namespace is used. [optional] - filter (Dict[str, Union[str, float, int, bool, List, dict]): - The filter to apply. You can use vector metadata to limit your search. - See https://www.pinecone.io/docs/metadata-filtering/.. [optional] - include_values (bool): Indicates whether vector values are included in the response. - If omitted the server will use the default value of False [optional] - include_metadata (bool): Indicates whether metadata is included in the response as well as the ids. - If omitted the server will use the default value of False [optional] - sparse_vector: (Union[SparseValues, Dict[str, Union[List[float], List[int]]]]): sparse values of the query vector. - Expected to be either a SparseValues object or a dict of the form: - {'indices': List[int], 'values': List[float]}, where the lists each have the same length. - - Returns: QueryResponse object which contains the list of the closest vectors as ScoredVector objects, - and namespace name. - """ - response = self._query( *args, top_k=top_k, @@ -495,7 +373,7 @@ def _query( ("sparse_vector", sparse_vector), ] ) - + response = self._vector_api.query_vectors( QueryRequest( **args_dict, @@ -520,48 +398,6 @@ def query_namespaces( ] = None, **kwargs, ) -> QueryNamespacesResults: - """The query_namespaces() method is used to make a query to multiple namespaces in parallel and combine the results into one result set. - - Since several asynchronous calls are made on your behalf when calling this method, you will need to tune the pool_threads and connection_pool_maxsize parameter of the Index constructor to suite your workload. - - Examples: - - ```python - from pinecone import Pinecone - - pc = Pinecone(api_key="your-api-key") - index = pc.Index( - host="index-name", - pool_threads=32, - connection_pool_maxsize=32 - ) - - query_vec = [0.1, 0.2, 0.3] # An embedding that matches the index dimension - combined_results = index.query_namespaces( - vector=query_vec, - namespaces=['ns1', 'ns2', 'ns3', 'ns4'], - top_k=10, - filter={'genre': {"$eq": "drama"}}, - include_values=True, - include_metadata=True - ) - for vec in combined_results.matches: - print(vec.id, vec.score) - print(combined_results.usage) - ``` - - Args: - vector (List[float]): The query vector, must be the same length as the dimension of the index being queried. - namespaces (List[str]): The list of namespaces to query. - top_k (Optional[int], optional): The number of results you would like to request from each namespace. Defaults to 10. - filter (Optional[Dict[str, Union[str, float, int, bool, List, dict]]], optional): Pass an optional filter to filter results based on metadata. Defaults to None. - include_values (Optional[bool], optional): Boolean field indicating whether vector values should be included with results. Defaults to None. - include_metadata (Optional[bool], optional): Boolean field indicating whether vector metadata should be included with results. Defaults to None. - sparse_vector (Optional[ Union[SparseValues, Dict[str, Union[List[float], List[int]]]] ], optional): If you are working with a dotproduct index, you can pass a sparse vector as part of your hybrid search. Defaults to None. - - Returns: - QueryNamespacesResults: A QueryNamespacesResults object containing the combined results from all namespaces, as well as the combined usage cost in read units. - """ if namespaces is None or len(namespaces) == 0: raise ValueError("At least one namespace must be specified") if len(vector) == 0: @@ -609,37 +445,6 @@ def update( ] = None, **kwargs, ) -> Dict[str, Any]: - """ - The Update operation updates vector in a namespace. - If a value is included, it will overwrite the previous value. - If a set_metadata is included, - the values of the fields specified in it will be added or overwrite the previous value. - - API reference: https://docs.pinecone.io/reference/update - - Examples: - >>> index.update(id='id1', values=[1, 2, 3], namespace='my_namespace') - >>> index.update(id='id1', set_metadata={'key': 'value'}, namespace='my_namespace') - >>> index.update(id='id1', values=[1, 2, 3], sparse_values={'indices': [1, 2], 'values': [0.2, 0.4]}, - >>> namespace='my_namespace') - >>> index.update(id='id1', values=[1, 2, 3], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]), - >>> namespace='my_namespace') - - Args: - id (str): Vector's unique id. - values (List[float]): vector values to set. [optional] - set_metadata (Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]]]): - metadata to set for vector. [optional] - namespace (str): Namespace name where to update the vector.. [optional] - sparse_values: (Dict[str, Union[List[float], List[int]]]): sparse values to update for the vector. - Expected to be either a SparseValues object or a dict of the form: - {'indices': List[int], 'values': List[float]} where the lists each have the same length. - - Keyword Args: - Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpdateRequest for more details. - - Returns: An empty dictionary if the update was successful. - """ _check_type = kwargs.pop("_check_type", False) sparse_values = self._parse_sparse_values_arg(sparse_values) args_dict = parse_non_empty_args( @@ -664,23 +469,6 @@ def update( def describe_index_stats( self, filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, **kwargs ) -> DescribeIndexStatsResponse: - """ - The DescribeIndexStats operation returns statistics about the index's contents. - For example: The vector count per namespace and the number of dimensions. - - API reference: https://docs.pinecone.io/reference/describe_index_stats_post - - Examples: - >>> index.describe_index_stats() - >>> index.describe_index_stats(filter={'key': 'value'}) - - Args: - filter (Dict[str, Union[str, float, int, bool, List, dict]]): - If this parameter is present, the operation only returns statistics for vectors that satisfy the filter. - See https://www.pinecone.io/docs/metadata-filtering/.. [optional] - - Returns: DescribeIndexStatsResponse object which contains stats about the index. - """ _check_type = kwargs.pop("_check_type", False) args_dict = parse_non_empty_args([("filter", filter)]) @@ -702,31 +490,6 @@ def list_paginated( namespace: Optional[str] = None, **kwargs, ) -> ListResponse: - """ - The list_paginated operation finds vectors based on an id prefix within a single namespace. - It returns matching ids in a paginated form, with a pagination token to fetch the next page of results. - This id list can then be passed to fetch or delete operations, depending on your use case. - - Consider using the `list` method to avoid having to handle pagination tokens manually. - - Examples: - >>> results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace') - >>> [v.id for v in results.vectors] - ['99', '990', '991', '992', '993'] - >>> results.pagination.next - eyJza2lwX3Bhc3QiOiI5OTMiLCJwcmVmaXgiOiI5OSJ9 - >>> next_results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace', pagination_token=results.pagination.next) - - Args: - prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will - be used with the effect of listing all ids in a namespace [optional] - limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional] - pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned - in the response if additional results are available. [optional] - namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional] - - Returns: ListResponse object which contains the list of ids, the namespace name, pagination information, and usage showing the number of read_units consumed. - """ args_dict = parse_non_empty_args( [ ("prefix", prefix), @@ -739,26 +502,6 @@ def list_paginated( @validate_and_convert_errors def list(self, **kwargs): - """ - The list operation accepts all of the same arguments as list_paginated, and returns a generator that yields - a list of the matching vector ids in each page of results. It automatically handles pagination tokens on your - behalf. - - Examples: - >>> for ids in index.list(prefix='99', limit=5, namespace='my_namespace'): - >>> print(ids) - ['99', '990', '991', '992', '993'] - ['994', '995', '996', '997', '998'] - ['999'] - - Args: - prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will - be used with the effect of listing all ids in a namespace [optional] - limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional] - pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned - in the response if additional results are available. [optional] - namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional] - """ done = False while not done: results = self.list_paginated(**kwargs) diff --git a/pinecone/data/interfaces.py b/pinecone/data/interfaces.py new file mode 100644 index 00000000..9589099c --- /dev/null +++ b/pinecone/data/interfaces.py @@ -0,0 +1,409 @@ +from abc import ABC, abstractmethod +from typing import Union, List, Optional, Dict, Any + +from pinecone.core.openapi.db_data.models import ( + FetchResponse, + QueryResponse, + IndexDescription as DescribeIndexStatsResponse, + UpsertResponse, + Vector, + ListResponse, + SparseValues, +) +from .query_results_aggregator import QueryNamespacesResults +from multiprocessing.pool import ApplyResult + + +class IndexInterface(ABC): + @abstractmethod + def upsert( + self, + vectors: Union[List[Vector], List[tuple], List[dict]], + namespace: Optional[str] = None, + batch_size: Optional[int] = None, + show_progress: bool = True, + **kwargs, + ) -> UpsertResponse: + """ + The upsert operation writes vectors into a namespace. + If a new value is upserted for an existing vector id, it will overwrite the previous value. + + To upsert in parallel follow: https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel + + A vector can be represented by a 1) Vector object, a 2) tuple or 3) a dictionary + + If a tuple is used, it must be of the form `(id, values, metadata)` or `(id, values)`. + where id is a string, vector is a list of floats, metadata is a dict, + and sparse_values is a dict of the form `{'indices': List[int], 'values': List[float]}`. + + Examples: + >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}, {'indices': [1, 2], 'values': [0.2, 0.4]}) + >>> ('id1', [1.0, 2.0, 3.0], None, {'indices': [1, 2], 'values': [0.2, 0.4]}) + >>> ('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0]) + + If a Vector object is used, a Vector object must be of the form + `Vector(id, values, metadata, sparse_values)`, where metadata and sparse_values are optional + arguments. + + Examples: + >>> Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}) + >>> Vector(id='id2', values=[1.0, 2.0, 3.0]) + >>> Vector(id='id3', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4])) + + **Note:** the dimension of each vector must match the dimension of the index. + + If a dictionary is used, it must be in the form `{'id': str, 'values': List[float], 'sparse_values': {'indices': List[int], 'values': List[float]}, 'metadata': dict}` + + Examples: + >>> index.upsert([('id1', [1.0, 2.0, 3.0], {'key': 'value'}), ('id2', [1.0, 2.0, 3.0])]) + >>> + >>> index.upsert([{'id': 'id1', 'values': [1.0, 2.0, 3.0], 'metadata': {'key': 'value'}}, + >>> {'id': 'id2', 'values': [1.0, 2.0, 3.0], 'sparse_values': {'indices': [1, 8], 'values': [0.2, 0.4]}]) + >>> index.upsert([Vector(id='id1', values=[1.0, 2.0, 3.0], metadata={'key': 'value'}), + >>> Vector(id='id2', values=[1.0, 2.0, 3.0], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]))]) + + API reference: https://docs.pinecone.io/reference/upsert + + Args: + vectors (Union[List[Vector], List[Tuple]]): A list of vectors to upsert. + namespace (str): The namespace to write to. If not specified, the default namespace is used. [optional] + batch_size (int): The number of vectors to upsert in each batch. + If not specified, all vectors will be upserted in a single batch. [optional] + show_progress (bool): Whether to show a progress bar using tqdm. + Applied only if batch_size is provided. Default is True. + Keyword Args: + Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpsertRequest for more details. + + Returns: UpsertResponse, includes the number of vectors upserted. + """ + pass + + @abstractmethod + def upsert_from_dataframe( + self, df, namespace: Optional[str] = None, batch_size: int = 500, show_progress: bool = True + ): + """Upserts a dataframe into the index. + + Args: + df: A pandas dataframe with the following columns: id, values, sparse_values, and metadata. + namespace: The namespace to upsert into. + batch_size: The number of rows to upsert in a single batch. + show_progress: Whether to show a progress bar. + """ + pass + + @abstractmethod + def delete( + self, + ids: Optional[List[str]] = None, + delete_all: Optional[bool] = None, + namespace: Optional[str] = None, + filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, + **kwargs, + ) -> Dict[str, Any]: + """ + The Delete operation deletes vectors from the index, from a single namespace. + No error raised if the vector id does not exist. + Note: for any delete call, if namespace is not specified, the default namespace is used. + + Delete can occur in the following mutual exclusive ways: + 1. Delete by ids from a single namespace + 2. Delete all vectors from a single namespace by setting delete_all to True + 3. Delete all vectors from a single namespace by specifying a metadata filter + (note that for this option delete all must be set to False) + + API reference: https://docs.pinecone.io/reference/delete_post + + Examples: + >>> index.delete(ids=['id1', 'id2'], namespace='my_namespace') + >>> index.delete(delete_all=True, namespace='my_namespace') + >>> index.delete(filter={'key': 'value'}, namespace='my_namespace') + + Args: + ids (List[str]): Vector ids to delete [optional] + delete_all (bool): This indicates that all vectors in the index namespace should be deleted.. [optional] + Default is False. + namespace (str): The namespace to delete vectors from [optional] + If not specified, the default namespace is used. + filter (Dict[str, Union[str, float, int, bool, List, dict]]): + If specified, the metadata filter here will be used to select the vectors to delete. + This is mutually exclusive with specifying ids to delete in the ids param or using delete_all=True. + See https://www.pinecone.io/docs/metadata-filtering/.. [optional] + + Keyword Args: + Supports OpenAPI client keyword arguments. See pinecone.core.client.models.DeleteRequest for more details. + + + Returns: An empty dictionary if the delete operation was successful. + """ + pass + + @abstractmethod + def fetch(self, ids: List[str], namespace: Optional[str] = None, **kwargs) -> FetchResponse: + """ + The fetch operation looks up and returns vectors, by ID, from a single namespace. + The returned vectors include the vector data and/or metadata. + + API reference: https://docs.pinecone.io/reference/fetch + + Examples: + >>> index.fetch(ids=['id1', 'id2'], namespace='my_namespace') + >>> index.fetch(ids=['id1', 'id2']) + + Args: + ids (List[str]): The vector IDs to fetch. + namespace (str): The namespace to fetch vectors from. + If not specified, the default namespace is used. [optional] + Keyword Args: + Supports OpenAPI client keyword arguments. See pinecone.core.client.models.FetchResponse for more details. + + + Returns: FetchResponse object which contains the list of Vector objects, and namespace name. + """ + pass + + @abstractmethod + def query( + self, + *args, + top_k: int, + vector: Optional[List[float]] = None, + id: Optional[str] = None, + namespace: Optional[str] = None, + filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, + include_values: Optional[bool] = None, + include_metadata: Optional[bool] = None, + sparse_vector: Optional[ + Union[SparseValues, Dict[str, Union[List[float], List[int]]]] + ] = None, + **kwargs, + ) -> Union[QueryResponse, ApplyResult]: + """ + The Query operation searches a namespace, using a query vector. + It retrieves the ids of the most similar items in a namespace, along with their similarity scores. + + API reference: https://docs.pinecone.io/reference/query + + Examples: + >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace') + >>> index.query(id='id1', top_k=10, namespace='my_namespace') + >>> index.query(vector=[1, 2, 3], top_k=10, namespace='my_namespace', filter={'key': 'value'}) + >>> index.query(id='id1', top_k=10, namespace='my_namespace', include_metadata=True, include_values=True) + >>> index.query(vector=[1, 2, 3], sparse_vector={'indices': [1, 2], 'values': [0.2, 0.4]}, + >>> top_k=10, namespace='my_namespace') + >>> index.query(vector=[1, 2, 3], sparse_vector=SparseValues([1, 2], [0.2, 0.4]), + >>> top_k=10, namespace='my_namespace') + + Args: + vector (List[float]): The query vector. This should be the same length as the dimension of the index + being queried. Each `query()` request can contain only one of the parameters + `id` or `vector`.. [optional] + id (str): The unique ID of the vector to be used as a query vector. + Each `query()` request can contain only one of the parameters + `vector` or `id`. [optional] + top_k (int): The number of results to return for each query. Must be an integer greater than 1. + namespace (str): The namespace to fetch vectors from. + If not specified, the default namespace is used. [optional] + filter (Dict[str, Union[str, float, int, bool, List, dict]): + The filter to apply. You can use vector metadata to limit your search. + See https://www.pinecone.io/docs/metadata-filtering/.. [optional] + include_values (bool): Indicates whether vector values are included in the response. + If omitted the server will use the default value of False [optional] + include_metadata (bool): Indicates whether metadata is included in the response as well as the ids. + If omitted the server will use the default value of False [optional] + sparse_vector: (Union[SparseValues, Dict[str, Union[List[float], List[int]]]]): sparse values of the query vector. + Expected to be either a SparseValues object or a dict of the form: + {'indices': List[int], 'values': List[float]}, where the lists each have the same length. + + Returns: QueryResponse object which contains the list of the closest vectors as ScoredVector objects, + and namespace name. + """ + pass + + @abstractmethod + def query_namespaces( + self, + vector: List[float], + namespaces: List[str], + top_k: Optional[int] = None, + filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, + include_values: Optional[bool] = None, + include_metadata: Optional[bool] = None, + sparse_vector: Optional[ + Union[SparseValues, Dict[str, Union[List[float], List[int]]]] + ] = None, + **kwargs, + ) -> QueryNamespacesResults: + """The query_namespaces() method is used to make a query to multiple namespaces in parallel and combine the results into one result set. + + Since several asynchronous calls are made on your behalf when calling this method, you will need to tune the pool_threads and connection_pool_maxsize parameter of the Index constructor to suite your workload. + + Examples: + + ```python + from pinecone import Pinecone + + pc = Pinecone(api_key="your-api-key") + index = pc.Index( + host="index-name", + pool_threads=32, + connection_pool_maxsize=32 + ) + + query_vec = [0.1, 0.2, 0.3] # An embedding that matches the index dimension + combined_results = index.query_namespaces( + vector=query_vec, + namespaces=['ns1', 'ns2', 'ns3', 'ns4'], + top_k=10, + filter={'genre': {"$eq": "drama"}}, + include_values=True, + include_metadata=True + ) + for vec in combined_results.matches: + print(vec.id, vec.score) + print(combined_results.usage) + ``` + + Args: + vector (List[float]): The query vector, must be the same length as the dimension of the index being queried. + namespaces (List[str]): The list of namespaces to query. + top_k (Optional[int], optional): The number of results you would like to request from each namespace. Defaults to 10. + filter (Optional[Dict[str, Union[str, float, int, bool, List, dict]]], optional): Pass an optional filter to filter results based on metadata. Defaults to None. + include_values (Optional[bool], optional): Boolean field indicating whether vector values should be included with results. Defaults to None. + include_metadata (Optional[bool], optional): Boolean field indicating whether vector metadata should be included with results. Defaults to None. + sparse_vector (Optional[ Union[SparseValues, Dict[str, Union[List[float], List[int]]]] ], optional): If you are working with a dotproduct index, you can pass a sparse vector as part of your hybrid search. Defaults to None. + + Returns: + QueryNamespacesResults: A QueryNamespacesResults object containing the combined results from all namespaces, as well as the combined usage cost in read units. + """ + pass + + @abstractmethod + def update( + self, + id: str, + values: Optional[List[float]] = None, + set_metadata: Optional[ + Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]] + ] = None, + namespace: Optional[str] = None, + sparse_values: Optional[ + Union[SparseValues, Dict[str, Union[List[float], List[int]]]] + ] = None, + **kwargs, + ) -> Dict[str, Any]: + """ + The Update operation updates vector in a namespace. + If a value is included, it will overwrite the previous value. + If a set_metadata is included, + the values of the fields specified in it will be added or overwrite the previous value. + + API reference: https://docs.pinecone.io/reference/update + + Examples: + >>> index.update(id='id1', values=[1, 2, 3], namespace='my_namespace') + >>> index.update(id='id1', set_metadata={'key': 'value'}, namespace='my_namespace') + >>> index.update(id='id1', values=[1, 2, 3], sparse_values={'indices': [1, 2], 'values': [0.2, 0.4]}, + >>> namespace='my_namespace') + >>> index.update(id='id1', values=[1, 2, 3], sparse_values=SparseValues(indices=[1, 2], values=[0.2, 0.4]), + >>> namespace='my_namespace') + + Args: + id (str): Vector's unique id. + values (List[float]): vector values to set. [optional] + set_metadata (Dict[str, Union[str, float, int, bool, List[int], List[float], List[str]]]]): + metadata to set for vector. [optional] + namespace (str): Namespace name where to update the vector.. [optional] + sparse_values: (Dict[str, Union[List[float], List[int]]]): sparse values to update for the vector. + Expected to be either a SparseValues object or a dict of the form: + {'indices': List[int], 'values': List[float]} where the lists each have the same length. + + Keyword Args: + Supports OpenAPI client keyword arguments. See pinecone.core.client.models.UpdateRequest for more details. + + Returns: An empty dictionary if the update was successful. + """ + pass + + @abstractmethod + def describe_index_stats( + self, filter: Optional[Dict[str, Union[str, float, int, bool, List, dict]]] = None, **kwargs + ) -> DescribeIndexStatsResponse: + """ + The DescribeIndexStats operation returns statistics about the index's contents. + For example: The vector count per namespace and the number of dimensions. + + API reference: https://docs.pinecone.io/reference/describe_index_stats_post + + Examples: + >>> index.describe_index_stats() + >>> index.describe_index_stats(filter={'key': 'value'}) + + Args: + filter (Dict[str, Union[str, float, int, bool, List, dict]]): + If this parameter is present, the operation only returns statistics for vectors that satisfy the filter. + See https://www.pinecone.io/docs/metadata-filtering/.. [optional] + + Returns: DescribeIndexStatsResponse object which contains stats about the index. + """ + pass + + @abstractmethod + def list_paginated( + self, + prefix: Optional[str] = None, + limit: Optional[int] = None, + pagination_token: Optional[str] = None, + namespace: Optional[str] = None, + **kwargs, + ) -> ListResponse: + """ + The list_paginated operation finds vectors based on an id prefix within a single namespace. + It returns matching ids in a paginated form, with a pagination token to fetch the next page of results. + This id list can then be passed to fetch or delete operations, depending on your use case. + + Consider using the `list` method to avoid having to handle pagination tokens manually. + + Examples: + >>> results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace') + >>> [v.id for v in results.vectors] + ['99', '990', '991', '992', '993'] + >>> results.pagination.next + eyJza2lwX3Bhc3QiOiI5OTMiLCJwcmVmaXgiOiI5OSJ9 + >>> next_results = index.list_paginated(prefix='99', limit=5, namespace='my_namespace', pagination_token=results.pagination.next) + + Args: + prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will + be used with the effect of listing all ids in a namespace [optional] + limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional] + pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned + in the response if additional results are available. [optional] + namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional] + + Returns: ListResponse object which contains the list of ids, the namespace name, pagination information, and usage showing the number of read_units consumed. + """ + pass + + @abstractmethod + def list(self, **kwargs): + """ + The list operation accepts all of the same arguments as list_paginated, and returns a generator that yields + a list of the matching vector ids in each page of results. It automatically handles pagination tokens on your + behalf. + + Examples: + >>> for ids in index.list(prefix='99', limit=5, namespace='my_namespace'): + >>> print(ids) + ['99', '990', '991', '992', '993'] + ['994', '995', '996', '997', '998'] + ['999'] + + Args: + prefix (Optional[str]): The id prefix to match. If unspecified, an empty string prefix will + be used with the effect of listing all ids in a namespace [optional] + limit (Optional[int]): The maximum number of ids to return. If unspecified, the server will use a default value. [optional] + pagination_token (Optional[str]): A token needed to fetch the next page of results. This token is returned + in the response if additional results are available. [optional] + namespace (Optional[str]): The namespace to fetch vectors from. If not specified, the default namespace is used. [optional] + """ + pass diff --git a/tests/integration/control/serverless/test_index_instantiation_ux.py b/tests/integration/control/serverless/test_index_instantiation_ux.py new file mode 100644 index 00000000..0e56f925 --- /dev/null +++ b/tests/integration/control/serverless/test_index_instantiation_ux.py @@ -0,0 +1,13 @@ +import pinecone +import pytest + + +class TestIndexInstantiationUX: + def test_index_instantiation_ux(self): + with pytest.raises(Exception) as e: + pinecone.Index(name="my-index", host="host") + + assert ( + "You are attempting to access the Index client directly from the pinecone module." + in str(e.value) + ) diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index c9037ff9..d3c8ff7c 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -2,7 +2,7 @@ import pytest import pinecone -from pinecone import Index +from pinecone.data import _Index from pinecone import UpsertRequest, Vector from pinecone import ( DescribeIndexStatsRequest, @@ -31,7 +31,7 @@ def setup_method(self): self.svv2 = [0.1, 0.2, 0.3] self.sv2 = {"indices": self.svi2, "values": self.svv2} - self.index = Index(api_key="asdf", host="https://test.pinecone.io") + self.index = _Index(api_key="asdf", host="https://test.pinecone.io") # region: upsert tests @@ -129,7 +129,7 @@ def test_upsert_parallelUpsert_callUpsertParallel(self, mocker): [Vector(id="vec1", values=self.vals1, metadata=self.md1)], [Vector(id="vec2", values=self.vals2, metadata=self.md2)], ] - with Index(api_key="asdf", host="https://test.pinecone.io", pool_threads=30) as index: + with _Index(api_key="asdf", host="https://test.pinecone.io", pool_threads=30) as index: mocker.patch.object(index._vector_api, "upsert_vectors", autospec=True) # Send requests in parallel