V1.7.0 (qdrant#56)

* update grpc * update generated_schema * provide timeout to api * create shards api * shard keys public api * discovery api * add new fields * v1.7.0 * update proto (qdrant#57) * update undici * fix unit tests * renamed recommend_batch to recommendBatch for consistency and kept alias (qdrant#54) Co-authored-by: Mohamed Morad <[email protected]> Co-authored-by: Ivan Pleshkov <[email protected]> * fix ci * restore tests * add comments to new api * shard_key args docs * create collection docs * qdrant integration tests version up * thanks to Morad-m11 --------- Co-authored-by: Morad-m11 <[email protected]> Co-authored-by: Mohamed Morad <[email protected]>
harshalmittal4 · Dec 8, 2023 · 48db336 · 48db336
1 parent 2c8dd87
commit 48db336
Show file tree

Hide file tree

Showing 28 changed files with 2,792 additions and 367 deletions.
diff --git a/examples/node-js-basic/package.json b/examples/node-js-basic/package.json
@@ -15,6 +15,6 @@
     "author": "Qdrant Team",
     "license": "Apache-2.0",
     "dependencies": {
-        "@qdrant/qdrant-js": "^1.6.0"
+        "@qdrant/qdrant-js": "^1.7.0"
     }
 }
diff --git a/packages/js-client-grpc/CHANGELOG.md b/packages/js-client-grpc/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @qdrant/js-client-grpc
 
+## 1.7.0
+
+### Minor Changes
+
+-   Qdrant v1.7.0 API
+-   [#54](https://github.com/qdrant/qdrant-js/pull/54) Thanks [@Morad-m11](https://github.com/Morad-m11)! - recommendBatch mispell
+
 ## 1.6.0
 
 ### Minor Changes

diff --git a/packages/js-client-grpc/package.json b/packages/js-client-grpc/package.json
@@ -1,6 +1,6 @@
 {
     "name": "@qdrant/js-client-grpc",
-    "version": "1.6.0",
+    "version": "1.7.0",
     "engines": {
         "node": ">=18.0.0",
         "pnpm": ">=8"
@@ -48,12 +48,12 @@
         "codegen:grpc-typescript": "./scripts/generate-grpc-sources.sh"
     },
     "dependencies": {
-        "@bufbuild/connect": "^0.9.1",
-        "@bufbuild/connect-node": "^0.9.1",
+        "@bufbuild/connect": "^0.10.0",
+        "@bufbuild/connect-node": "^0.10.0",
         "@bufbuild/protobuf": "^1.2.1"
     },
     "devDependencies": {
-        "@bufbuild/protoc-gen-connect-es": "^0.9.1",
+        "@bufbuild/protoc-gen-connect-es": "^0.10.0",
         "@bufbuild/protoc-gen-es": "^1.2.1",
         "@protobuf-ts/protoc": "^2.9.0",
         "@sevinf/maybe": "^0.5.0",

diff --git a/packages/js-client-grpc/proto/collections.proto b/packages/js-client-grpc/proto/collections.proto
@@ -1,5 +1,6 @@
 syntax = "proto3";
 package qdrant;
+option csharp_namespace = "Qdrant.Client.Grpc";
 
 message VectorParams {
   uint64 size = 1; // Size of the vectors
@@ -37,6 +38,14 @@ message VectorsConfigDiff {
   }
 }
 
+message SparseVectorParams {
+  optional SparseIndexConfig index = 1; // Configuration of sparse index
+}
+
+message SparseVectorConfig {
+  map<string, SparseVectorParams> map = 1;
+}
+
 message GetCollectionInfoRequest {
   string collection_name = 1; // Name of the collection
 }
@@ -63,6 +72,7 @@ enum Distance {
   Cosine = 1;
   Euclid = 2;
   Dot = 3;
+  Manhattan = 4;
 }
 
 enum CollectionStatus {
@@ -130,6 +140,18 @@ message HnswConfigDiff {
   optional uint64 payload_m = 6;
 }
 
+message SparseIndexConfig {
+  /*
+    Prefer a full scan search upto (excluding) this number of vectors.
+    Note: this is number of vectors, not KiloBytes.
+   */
+  optional uint64 full_scan_threshold = 1;
+  /*
+  Store inverted index on disk. If set to false, the index will be stored in RAM.
+   */
+  optional bool on_disk = 2;
+}
+
 message WalConfigDiff {
   optional uint64 wal_capacity_mb = 1; // Size of a single WAL block file
   optional uint64 wal_segments_ahead = 2; // Number of segments to create in advance
@@ -233,6 +255,11 @@ message QuantizationConfigDiff {
   }
 }
 
+enum ShardingMethod {
+  Auto = 0; // Auto-sharding based on record ids
+  Custom = 1; // Shard by user-defined key
+}
+
 message CreateCollection {
   string collection_name = 1; // Name of the collection
   reserved 2; // Deprecated
@@ -248,6 +275,8 @@ message CreateCollection {
   optional uint32 write_consistency_factor = 12; // How many replicas should apply the operation for us to consider it successful, default = 1
   optional string init_from_collection = 13; // Specify name of the other collection to copy data from
   optional QuantizationConfig quantization_config = 14; // Quantization configuration of vector
+  optional ShardingMethod sharding_method = 15; // Sharding method
+  optional SparseVectorConfig sparse_vectors_config = 16; // Configuration for sparse vectors
 }
 
 message UpdateCollection {
@@ -258,6 +287,7 @@ message UpdateCollection {
   optional HnswConfigDiff hnsw_config = 5; // New HNSW parameters for the collection index
   optional VectorsConfigDiff vectors_config = 6; // New vector parameters
   optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector
+  optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters
 }
 
 message DeleteCollection {
@@ -279,6 +309,8 @@ message CollectionParams {
   optional uint32 replication_factor = 6; // Number of replicas of each shard that network tries to maintain
   optional uint32 write_consistency_factor = 7; // How many replicas should apply the operation for us to consider it successful
   optional uint32 read_fan_out_factor = 8; // Fan-out every read request to these many additional remote nodes (and return first available response)
+  optional ShardingMethod sharding_method = 9; // Sharding method
+  optional SparseVectorConfig sparse_vectors_config = 10; // Configuration for sparse vectors
 }
 
 message CollectionParamsDiff {
@@ -326,14 +358,14 @@ message PayloadSchemaInfo {
 message CollectionInfo {
   CollectionStatus status = 1; // operating condition of the collection
   OptimizerStatus optimizer_status = 2; // status of collection optimizers
-  uint64 vectors_count = 3; // number of vectors in the collection
+  optional uint64 vectors_count = 3; // Approximate number of vectors in the collection
   uint64 segments_count = 4; // Number of independent segments
   reserved 5; // Deprecated
   reserved 6; // Deprecated
   CollectionConfig config = 7; // Configuration
   map<string, PayloadSchemaInfo> payload_schema = 8; // Collection data types
-  uint64 points_count = 9; // number of points in the collection
-  optional uint64 indexed_vectors_count = 10; // number of indexed vectors in the collection.
+  optional uint64 points_count = 9; // Approximate number of points in the collection
+  optional uint64 indexed_vectors_count = 10; // Approximate number of indexed vectors in the collection.
 }
 
 message ChangeAliases {
@@ -390,18 +422,28 @@ enum ReplicaState {
   Partial = 2; // The shard is partially loaded and is currently receiving data from other shards
   Initializing = 3; // Collection is being created
   Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
+  PartialSnapshot = 5; // Snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
+}
+
+message ShardKey {
+  oneof key {
+    string keyword = 1; // String key
+    uint64 number = 2; // Number key
+  }
 }
 
 message LocalShardInfo {
   uint32 shard_id = 1; // Local shard id
   uint64 points_count = 2; // Number of points in the shard
   ReplicaState state = 3;  // Is replica active
+  optional ShardKey shard_key = 4; // User-defined shard key
 }
 
 message RemoteShardInfo {
   uint32 shard_id = 1; // Local shard id
   uint64 peer_id = 2; // Remote peer id
   ReplicaState state = 3; // Is replica active
+  optional ShardKey shard_key = 4; // User-defined shard key
 }
 
 message ShardTransferInfo {
@@ -423,24 +465,63 @@ message MoveShard {
   uint32 shard_id = 1; // Local shard id
   uint64 from_peer_id = 2;
   uint64 to_peer_id = 3;
+  optional ShardTransferMethod method = 4;
+}
+
+enum ShardTransferMethod {
+  StreamRecords = 0;
+  Snapshot = 1;
 }
 
 message Replica {
   uint32 shard_id = 1;
   uint64 peer_id = 2;
 }
 
+message CreateShardKey {
+    ShardKey shard_key = 1; // User-defined shard key
+    optional uint32 shards_number = 2; // Number of shards to create per shard key
+    optional uint32 replication_factor = 3; // Number of replicas of each shard to create
+    repeated uint64 placement = 4; // List of peer ids, allowed to create shards. If empty - all peers are allowed
+}
+
+message DeleteShardKey {
+    ShardKey shard_key = 1; // Shard key to delete
+}
+
 message UpdateCollectionClusterSetupRequest {
   string collection_name = 1; // Name of the collection
   oneof operation {
     MoveShard move_shard = 2;
     MoveShard replicate_shard = 3;
     MoveShard abort_transfer = 4;
     Replica drop_replica = 5;
+    CreateShardKey create_shard_key = 7;
+    DeleteShardKey delete_shard_key = 8;
   }
   optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
 }
 
 message UpdateCollectionClusterSetupResponse {
   bool result = 1;
 }
+
+message CreateShardKeyRequest {
+    string collection_name = 1; // Name of the collection
+    CreateShardKey request = 2; // Request to create shard key
+    optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
+}
+
+message DeleteShardKeyRequest {
+    string collection_name = 1; // Name of the collection
+    DeleteShardKey request = 2; // Request to delete shard key
+    optional uint64 timeout = 3; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
+}
+
+message CreateShardKeyResponse {
+    bool result = 1;
+}
+
+message DeleteShardKeyResponse {
+    bool result = 1;
+}
diff --git a/packages/js-client-grpc/proto/collections_service.proto b/packages/js-client-grpc/proto/collections_service.proto
@@ -3,6 +3,7 @@ syntax = "proto3";
 import "collections.proto";
 
 package qdrant;
+option csharp_namespace = "Qdrant.Client.Grpc";
 
 service Collections {
   /*
@@ -45,4 +46,12 @@ service Collections {
   Update cluster setup for a collection
   */
   rpc UpdateCollectionClusterSetup (UpdateCollectionClusterSetupRequest) returns (UpdateCollectionClusterSetupResponse) {}
+  /*
+  Create shard key
+  */
+  rpc CreateShardKey (CreateShardKeyRequest) returns (CreateShardKeyResponse) {}
+  /*
+  Delete shard key
+  */
+  rpc DeleteShardKey (DeleteShardKeyRequest) returns (DeleteShardKeyResponse) {}
 }
diff --git a/packages/js-client-grpc/proto/json_with_int.proto b/packages/js-client-grpc/proto/json_with_int.proto
@@ -3,6 +3,7 @@
 syntax = "proto3";
 
 package qdrant;
+option csharp_namespace = "Qdrant.Client.Grpc";
 
 // `Struct` represents a structured data value, consisting of fields
 // which map to dynamically typed values. In some languages, `Struct`