Refactoring and updates for 2024-10 API spec (#401)

## Problem

Needed to consume the 2024-10 API spec.

## Solution

The spec updates involved many changes to operation ids. This should not
touch the user experience, but does mean this diff is somewhat larger
than usual due to the large number of tests for existing functionality
that needed to be adjusted for the new names:
- upsert > upsert_vector
- fetch > fetch_vectors
- query > query_vectors
- delete > delete_vectors
- list > list_vectors
- start_import > start_bulk_import
- describe_import > describe_bulk_import
- etc

### Other changes

#### Cleanup prerelease code no longer needed
- Deleted `pinecone/core_ea/` which was the home of some code generated
off of the 2024-10 spec when bulk import functionality was in a
pre-release state. We no longer need this. That's why this PR has 21k
lines removed.

#### Changes to generation process
- Placed a copy of classes such as `ApiClient`, `Endpoint`,
`ModelNormal`, etc which were previously generated into a new folder,
`pinecone/openapi_support`. Even though these used to be generated, they
contain no dynamic content. So taking them out of the generation process
should make it significantly easier to work on core improvements to the
UX and performance of the generated code, since the source of truth will
no longer be a mustache file (FML)
- Updated the codegen script `./codegen/build-oas.sh 2024-10 false` to
delete instead of de-dupe copies of shared classes like `ApiClient`, and
edit model+api files that continue to be generated to find the classes
they need in the new `openapi_support` folder.

#### mypy fixes
- I needed to make adjustments in some `openapi_support` classes to
satisfy the mypy type checker (`pinecone/core` has been ignored in the
past, and our goal should be to eventually have 100% of code
typechecked).
- Wrote some initial unit tests to characterize some of the behavior in
`ApiClient`. The tests were to help give me more confidence I wasn't
breaking something along the way while making adjustments for mypy.
- Added a dev dependency on a package with types for datetime

## Type of Change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [x] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Infrastructure change (CI configs, etc)
- [ ] Non-code change (docs, etc)
- [ ] None of the above: (explain here)

codegen/build-oas.sh

@@ -13,7 +13,7 @@ if [ "$is_early_access" = "true" ]; then
 	template_dir="codegen/python-oas-templates/templates5.2.0"
 else
 	destination="pinecone/core/openapi"
-	modules=("control" "data")
+	modules=("db_control" "db_data")
 	py_module_name="core"
 	template_dir="codegen/python-oas-templates/templates5.2.0"
 fi
@@ -83,7 +83,7 @@ generate_client() {
 	docker run --rm -v $(pwd):/workspace openapitools/openapi-generator-cli:v5.2.0 generate \
 		--input-spec "/workspace/$oas_file" \
 		--generator-name python \
-		--additional-properties=packageName=$package_name,pythonAttrNoneIfUnset=true,exceptionsPackageName=pinecone.core.openapi.shared.exceptions \
+		--additional-properties=packageName=$package_name,pythonAttrNoneIfUnset=true,exceptionsPackageName=pinecone.openapi_support.exceptions \
 		--output "/workspace/${build_dir}" \
 		--template-dir "/workspace/$template_dir"
 
@@ -97,12 +97,10 @@ generate_client() {
 	rm -rf "${destination}/${module_name}"
 	mkdir -p "${destination}"
 	cp -r "build/pinecone/$py_module_name/openapi/${module_name}" "${destination}/${module_name}"
+	echo "API_VERSION = '${version}'" >> "${destination}/${module_name}/__init__.py"
 }
 
-extract_shared_classes() {
-	target_directory="${destination}/shared"
-	mkdir -p "$target_directory"
-
+remove_shared_classes() {
 	# Define the list of shared source files
 	sharedFiles=(
 		"api_client"
@@ -114,11 +112,6 @@ extract_shared_classes() {
 
 	source_directory="${destination}/${modules[0]}"
 
-	# Loop through each file we want to share and copy it to the target directory
-	for file in "${sharedFiles[@]}"; do
-		cp "${source_directory}/${file}.py" "$target_directory"
-	done
-
 	# Cleanup shared files in each module
 	for module in "${modules[@]}"; do
 		source_directory="${destination}/${module}"
@@ -127,26 +120,18 @@ extract_shared_classes() {
 		done
 	done
 
-	# Remove the docstring headers that aren't really correct in the
-	# context of this new shared package structure
-	find "$target_directory" -name "*.py" -print0 | xargs -0 -I {} sh -c 'sed -i "" "/^\"\"\"/,/^\"\"\"/d" "{}"'
-
-	echo "All shared files have been copied to $target_directory."
-
 	# Adjust import paths in every file
 	find "${destination}" -name "*.py" | while IFS= read -r file; do
-		sed -i '' "s/from \.\.model_utils/from pinecone\.$py_module_name\.openapi\.shared\.model_utils/g" "$file"
+		sed -i '' "s/from \.\.model_utils/from pinecone\.openapi_support\.model_utils/g" "$file"
 
 		for module in "${modules[@]}"; do
-			sed -i '' "s/from pinecone\.$py_module_name\.openapi\.$module import rest/from pinecone\.$py_module_name\.openapi\.shared import rest/g" "$file"
+			sed -i '' "s/from pinecone\.$py_module_name\.openapi\.$module import rest/from pinecone\.openapi_support import rest/g" "$file"
 
 			for sharedFile in "${sharedFiles[@]}"; do
-				sed -i '' "s/from pinecone\.$py_module_name\.openapi\.$module\.$sharedFile/from pinecone\.$py_module_name\.openapi\.shared\.$sharedFile/g" "$file"
+				sed -i '' "s/from pinecone\.$py_module_name\.openapi\.$module\.$sharedFile/from pinecone\.openapi_support/g" "$file"
 			done
 		done
 	done
-
-	echo "API_VERSION = '${version}'" > "${target_directory}/__init__.py"
 }
 
 update_apis_repo
@@ -162,8 +147,9 @@ done
 
 # Even though we want to generate multiple packages, we
 # don't want to duplicate every exception and utility class.
-# So we do a bit of surgery to combine the shared files.
-extract_shared_classes
+# So we do a bit of surgery to find these shared files
+# elsewhere, in the pinecone.openapi_support package.
+remove_shared_classes
 
 # Format generated files
 poetry run ruff format "${destination}"

pinecone/config/config.py

@@ -3,7 +3,7 @@
 
 from pinecone.exceptions.exceptions import PineconeConfigurationError
 from pinecone.config.openapi import OpenApiConfigFactory
-from pinecone.core.openapi.shared.configuration import Configuration as OpenApiConfiguration
+from pinecone.openapi_support.configuration import Configuration as OpenApiConfiguration
 from pinecone.utils import normalize_host
 from pinecone.utils.constants import SOURCE_TAG
 

pinecone/config/openapi.py

@@ -7,7 +7,7 @@
 
 from urllib3.connection import HTTPConnection
 
-from pinecone.core.openapi.shared.configuration import Configuration as OpenApiConfiguration
+from pinecone.openapi_support.configuration import Configuration as OpenApiConfiguration
 
 TCP_KEEPINTVL = 60  # Sec
 TCP_KEEPIDLE = 300  # Sec

pinecone/control/index_host_store.py

@@ -1,9 +1,9 @@
 from typing import Dict
 from pinecone.config import Config
-from pinecone.core.openapi.control.api.manage_indexes_api import (
+from pinecone.core.openapi.db_control.api.manage_indexes_api import (
     ManageIndexesApi as IndexOperationsApi,
 )
-from pinecone.core.openapi.shared.exceptions import PineconeException
+from pinecone.openapi_support.exceptions import PineconeException
 from pinecone.utils import normalize_host
 
 

pinecone/control/pinecone.py

@@ -6,12 +6,12 @@
 
 from pinecone.config import PineconeConfig, Config, ConfigBuilder
 
-from pinecone.core.openapi.control.api.manage_indexes_api import ManageIndexesApi
-from pinecone.core.openapi.shared.api_client import ApiClient
+from pinecone.core.openapi.db_control.api.manage_indexes_api import ManageIndexesApi
+from pinecone.openapi_support.api_client import ApiClient
 
 
 from pinecone.utils import normalize_host, setup_openapi_client, build_plugin_setup_client
-from pinecone.core.openapi.control.models import (
+from pinecone.core.openapi.db_control.models import (
     CreateCollectionRequest,
     CreateIndexRequest,
     ConfigureIndexRequest,
@@ -23,7 +23,7 @@
     PodSpec as PodSpecModel,
     PodSpecMetadataConfig,
 )
-from pinecone.core.openapi.shared import API_VERSION
+from pinecone.core.openapi.db_control import API_VERSION
 from pinecone.models import ServerlessSpec, PodSpec, IndexModel, IndexList, CollectionList
 from .langchain_import_warnings import _build_langchain_attribute_error_message
 

pinecone/control/repr_overrides.py

@@ -1,6 +1,6 @@
 from pinecone.utils import install_json_repr_override
 from pinecone.models.index_model import IndexModel
-from pinecone.core.openapi.control.models import CollectionModel
+from pinecone.core.openapi.db_control.models import CollectionModel
 
 
 def install_repr_overrides():