Update v1.8.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "qdrant-client"
-version = "1.6.0"
+version = "1.7.0"
 edition = "2021"
 authors = ["Qdrant Team <[email protected]>"]
 description = "Rust client for Qdrant Vector Search Engine"
@@ -18,12 +18,12 @@ anyhow = "1"
 
 serde = { version = "1", features = ["derive"], optional = true }
 serde_json = { version = "1", optional = true }
-reqwest = { version = "0.11.22", optional = true, default-features = false, features = ["stream", "rustls-tls"] }
-futures-util = { version = "0.3.29", optional = true }
+reqwest = { version = "0.11.24", optional = true, default-features = false, features = ["stream", "rustls-tls"] }
+futures-util = { version = "0.3.30", optional = true }
 
 [dev-dependencies]
 tonic-build = { version = "0.9.2", features = ["prost"] }
-tokio = { version = "1.34.0", features = ["rt-multi-thread"] }
+tokio = { version = "1.36.0", features = ["rt-multi-thread"] }
 
 [features]
 default = ["download_snapshots", "serde"]

proto/collections.proto

@@ -50,6 +50,19 @@ message GetCollectionInfoRequest {
   string collection_name = 1; // Name of the collection
 }
 
+message CollectionExistsRequest {
+  string collection_name = 1;
+}
+
+message CollectionExists {
+  bool exists = 1;
+}
+
+message CollectionExistsResponse {
+  CollectionExists result = 1;
+  double time = 2; // Time spent to process
+}
+
 message ListCollectionsRequest {
 }
 
@@ -90,6 +103,7 @@ enum PayloadSchemaType {
   Geo = 4;
   Text = 5;
   Bool = 6;
+  Datetime = 7;
 }
 
 enum QuantizationType {
@@ -113,7 +127,7 @@ message OptimizerStatus {
 message HnswConfigDiff {
   /*
   Number of edges per node in the index graph. Larger the value - more accurate the search, more space required.
-   */
+  */
   optional uint64 m = 1;
   /*
   Number of neighbours to consider during the index building. Larger the value - more accurate the search, more time required to build the index.
@@ -127,16 +141,19 @@ message HnswConfigDiff {
   */
   optional uint64 full_scan_threshold = 3;
   /*
-  Number of parallel threads used for background index building. If 0 - auto selection.
-   */
+  Number of parallel threads used for background index building.
+  If 0 - automatically select from 8 to 16.
+  Best to keep between 8 and 16 to prevent likelihood of building broken/inefficient HNSW graphs.
+  On small CPUs, less threads are used.
+  */
   optional uint64 max_indexing_threads = 4;
   /*
   Store HNSW index on disk. If set to false, the index will be stored in RAM.
-   */
+  */
   optional bool on_disk = 5;
   /*
-   Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used.
-   */
+  Number of additional payload-aware links per node in the index graph. If not set - regular M parameter will be used.
+  */
   optional uint64 payload_m = 6;
 }
 
@@ -160,11 +177,11 @@ message WalConfigDiff {
 message OptimizersConfigDiff {
   /*
   The minimal fraction of deleted vectors in a segment, required to perform segment optimization
-   */
+  */
   optional double deleted_threshold = 1;
   /*
   The minimal number of vectors in a segment, required to perform segment optimization
-   */
+  */
   optional uint64 vacuum_min_vector_number = 2;
   /*
   Target amount of segments the optimizer will try to keep.
@@ -181,7 +198,7 @@ message OptimizersConfigDiff {
   Do not create segments larger this size (in kilobytes).
   Large segments might require disproportionately long indexation times,
   therefore it makes sense to limit the size of segments.
-    
+
   If indexing speed is more important - make this parameter lower.
   If search speed is more important - make this parameter higher.
   Note: 1Kb = 1 vector of size 256
@@ -201,11 +218,11 @@ message OptimizersConfigDiff {
   optional uint64 memmap_threshold = 5;
   /*
   Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing
-  
+
   Default value is 20,000, based on <https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md>.
-  
+
   To disable vector indexing, set to `0`.
-  
+
   Note: 1kB = 1 vector of size 256.
   */
   optional uint64 indexing_threshold = 6;
@@ -214,7 +231,10 @@ message OptimizersConfigDiff {
   */
   optional uint64 flush_interval_sec = 7;
   /*
-  Max number of threads, which can be used for optimization. If 0 - `NUM_CPU - 1` will be used
+  Max number of threads (jobs) for running optimizations per shard.
+  Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
+  If null - have no limit and choose dynamically to saturate CPU.
+  If 0 - no optimization threads, optimizations will be disabled.
   */
   optional uint64 max_optimization_threads = 8;
 }
@@ -343,9 +363,15 @@ message TextIndexParams {
   optional uint64 max_token_len = 4; // Maximal token length
 }
 
+message IntegerIndexParams {
+  bool lookup = 1; // If true - support direct lookups.
+  bool range = 2; // If true - support ranges filters.
+}
+
 message PayloadIndexParams {
   oneof index_params {
     TextIndexParams text_index_params = 1; // Parameters for text index
+    IntegerIndexParams integer_index_params = 2; // Parameters for integer index
   }
 }
 
@@ -423,6 +449,8 @@ enum ReplicaState {
   Initializing = 3; // Collection is being created
   Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
   PartialSnapshot = 5; // Snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
+  Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
+  // TODO(1.9): deprecate PartialSnapshot state
 }
 
 message ShardKey {
@@ -468,9 +496,17 @@ message MoveShard {
   optional ShardTransferMethod method = 4;
 }
 
+message RestartTransfer {
+  uint32 shard_id = 1; // Local shard id
+  uint64 from_peer_id = 2;
+  uint64 to_peer_id = 3;
+  ShardTransferMethod method = 4;
+}
+
 enum ShardTransferMethod {
-  StreamRecords = 0;
-  Snapshot = 1;
+  StreamRecords = 0; // Stream shard records in batches
+  Snapshot = 1; // Snapshot the shard and recover it on the target peer
+  WalDelta = 2; // Resolve WAL delta between peers and transfer the difference
 }
 
 message Replica {
@@ -498,6 +534,7 @@ message UpdateCollectionClusterSetupRequest {
     Replica drop_replica = 5;
     CreateShardKey create_shard_key = 7;
     DeleteShardKey delete_shard_key = 8;
+    RestartTransfer restart_transfer = 9;
   }
   optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
 }

proto/collections_service.proto

@@ -8,23 +8,23 @@ option csharp_namespace = "Qdrant.Client.Grpc";
 service Collections {
   /*
   Get detailed information about specified existing collection
-   */
+  */
   rpc Get (GetCollectionInfoRequest) returns (GetCollectionInfoResponse) {}
   /*
   Get list name of all existing collections
-   */
+  */
   rpc List (ListCollectionsRequest) returns (ListCollectionsResponse) {}
   /*
   Create new collection with given parameters
-   */
+  */
   rpc Create (CreateCollection) returns (CollectionOperationResponse) {}
   /*
   Update parameters of the existing collection
-   */
+  */
   rpc Update (UpdateCollection) returns (CollectionOperationResponse) {}
   /*
   Drop collection and all associated data
-   */
+  */
   rpc Delete (DeleteCollection) returns (CollectionOperationResponse) {}
   /*
   Update Aliases of the existing collection
@@ -43,6 +43,10 @@ service Collections {
   */
   rpc CollectionClusterInfo (CollectionClusterInfoRequest) returns (CollectionClusterInfoResponse) {}
   /*
+  Check the existence of a collection
+  */
+  rpc CollectionExists (CollectionExistsRequest) returns (CollectionExistsResponse) {}
+  /*
   Update cluster setup for a collection
   */
   rpc UpdateCollectionClusterSetup (UpdateCollectionClusterSetupRequest) returns (UpdateCollectionClusterSetupResponse) {}

proto/points.proto

@@ -3,8 +3,9 @@ syntax = "proto3";
 package qdrant;
 option csharp_namespace = "Qdrant.Client.Grpc";
 
-import "json_with_int.proto";
 import "collections.proto";
+import "google/protobuf/timestamp.proto";
+import "json_with_int.proto";
 
 
 enum WriteOrderingType {
@@ -118,6 +119,7 @@ message SetPayloadPoints {
   optional PointsSelector points_selector = 5; // Affected points
   optional WriteOrdering ordering = 6; // Write ordering guarantees
   optional ShardKeySelector shard_key_selector = 7; // Option for custom sharding to specify used shard keys
+  optional string key = 8; // Option for indicate property of payload
 }
 
 message DeletePayloadPoints {
@@ -145,6 +147,7 @@ enum FieldType {
   FieldTypeGeo = 3;
   FieldTypeText = 4;
   FieldTypeBool = 5;
+  FieldTypeDatetime = 6;
 }
 
 message CreateFieldIndexCollection {
@@ -204,12 +207,12 @@ message WithVectorsSelector {
 message QuantizationSearchParams {
   /*
   If set to true, search will ignore quantized vector data
-   */
+  */
   optional bool ignore = 1;
 
   /*
   If true, use original vectors to re-score top-k results. If ignored, qdrant decides automatically does rescore enabled or not.
-   */
+  */
   optional bool rescore = 2;
 
   /*
@@ -220,15 +223,15 @@ message QuantizationSearchParams {
 
   For example, if `oversampling` is 2.4 and `limit` is 100, then 240 vectors will be pre-selected using quantized index,
   and then top-100 will be returned after re-scoring.
-   */
+  */
   optional double oversampling = 3;
 }
 
 message SearchParams {
   /*
   Params relevant to HNSW index. Size of the beam in a beam-search.
   Larger the value - more accurate the result, more time required for search.
-   */
+  */
   optional uint64 hnsw_ef = 1;
 
   /*
@@ -244,7 +247,7 @@ message SearchParams {
   If enabled, the engine will only perform search among indexed or small segments.
   Using this option prevents slow searches in case of delayed index, but does not
   guarantee that all uploaded vectors will be included in search results
-   */
+  */
   optional bool indexed_only = 4;
 }
 
@@ -299,6 +302,26 @@ message SearchPointGroups {
   optional SparseIndices sparse_indices = 16;
 }
 
+enum Direction {
+  Asc = 0;
+  Desc = 1;
+}
+
+message StartFrom {
+  oneof value {
+    double float = 1;
+    int64 integer = 2;
+    google.protobuf.Timestamp timestamp = 3;
+    string datetime = 4;
+  }
+}
+
+message OrderBy {
+  string key = 1; // Payload key to order by
+  optional Direction direction = 2; // Ascending or descending order
+  optional StartFrom start_from = 3; // Start from this value
+}
+
 message ScrollPoints {
   string collection_name = 1;
   Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
@@ -309,6 +332,7 @@ message ScrollPoints {
   optional WithVectorsSelector with_vectors = 7; // Options for specifying which vectors to include into response
   optional ReadConsistency read_consistency = 8; // Options for specifying read consistency guarantees
   optional ShardKeySelector shard_key_selector = 9; // Specify in which shards to look for the points, if not specified - look in all shards
+  optional OrderBy order_by = 10; // Order the records by a payload field
 }
 
 // How to use positive and negative vectors to find the results, default is `AverageVector`:
@@ -442,6 +466,7 @@ message PointsUpdateOperation {
       map<string, Value> payload = 1;
       optional PointsSelector points_selector = 2; // Affected points
       optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys
+      optional string key = 4; // Option for indicate property of payload
   }
   message DeletePayload {
       repeated string keys = 1;
@@ -623,6 +648,12 @@ message Filter {
   repeated Condition should = 1; // At least one of those conditions should match
   repeated Condition must = 2; // All conditions must match
   repeated Condition must_not = 3; // All conditions must NOT match
+  optional MinShould min_should = 4; // At least minimum amount of given conditions should match 
+}
+
+message MinShould {
+  repeated Condition conditions = 1; 
+  uint64 min_count = 2; 
 }
 
 message Condition {
@@ -661,6 +692,7 @@ message FieldCondition {
   GeoRadius geo_radius = 5; // Check if geo point is within a given radius
   ValuesCount values_count = 6; // Check number of values for a specific field
   GeoPolygon geo_polygon = 7; // Check if geo point is within a given polygon
+  DatetimeRange datetime_range = 8; // Check if datetime is within a given range
 }
 
 message Match {
@@ -691,6 +723,13 @@ message Range {
   optional double lte = 4;
 }
 
+message DatetimeRange {
+  optional google.protobuf.Timestamp lt = 1;
+  optional google.protobuf.Timestamp gt = 2;
+  optional google.protobuf.Timestamp gte = 3;
+  optional google.protobuf.Timestamp lte = 4;
+}
+
 message GeoBoundingBox {
   GeoPoint top_left = 1; // north-west corner
   GeoPoint bottom_right = 2; // south-east corner