upd to v1.10.0 (#45)

proto/collections.proto

@@ -5,6 +5,7 @@ enum Datatype {
   Default = 0;
   Float32 = 1;
   Uint8 = 2;
+  Float16 = 3;
 }
 
 message VectorParams {
@@ -14,6 +15,7 @@ message VectorParams {
   optional QuantizationConfig quantization_config = 4; // Configuration of vector quantization config. If omitted - the collection configuration will be used
   optional bool on_disk = 5; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM.
   optional Datatype datatype = 6; // Data type of the vectors
+  optional MultiVectorConfig multivector_config = 7; // Configuration for multi-vector search
 }
 
 message VectorParamsDiff {
@@ -44,14 +46,29 @@ message VectorsConfigDiff {
   }
 }
 
+enum Modifier {
+    None = 0;
+    Idf = 1; // Apply Inverse Document Frequency
+}
+
 message SparseVectorParams {
   optional SparseIndexConfig index = 1; // Configuration of sparse index
+  optional Modifier modifier = 2; // If set - apply modifier to the vector values
 }
 
 message SparseVectorConfig {
   map<string, SparseVectorParams> map = 1;
 }
 
+enum MultiVectorComparator {
+    MaxSim = 0;
+}
+
+message MultiVectorConfig {
+    MultiVectorComparator comparator = 1; // Comparator for multi-vector search
+}
+
+
 message GetCollectionInfoRequest {
   string collection_name = 1; // Name of the collection
 }
@@ -174,6 +191,10 @@ message SparseIndexConfig {
   Store inverted index on disk. If set to false, the index will be stored in RAM.
    */
   optional bool on_disk = 2;
+  /*
+  Datatype used to store weights in the index.
+  */
+  optional Datatype datatype = 3;
 }
 
 message WalConfigDiff {
@@ -457,6 +478,7 @@ enum ReplicaState {
   Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
   PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
   Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
+  Resharding = 7; // Points are being migrated to this shard as part of resharding
 }
 
 message ShardKey {
@@ -482,13 +504,14 @@ message RemoteShardInfo {
 
 message ShardTransferInfo {
   uint32 shard_id = 1; // Local shard id
+  optional uint32 to_shard_id = 5;
   uint64 from = 2;
   uint64 to = 3;
   bool sync = 4; // If `true` transfer is a synchronization of a replicas; If `false` transfer is a moving of a shard from one peer to another
 }
 
 message CollectionClusterInfoResponse {
-  uint64 peer_id = 1;  // ID of this peer 
+  uint64 peer_id = 1;  // ID of this peer
   uint64 shard_count = 2; // Total number of shards
   repeated LocalShardInfo local_shards = 3; // Local shards
   repeated RemoteShardInfo remote_shards = 4; // Remote shards
@@ -497,19 +520,30 @@ message CollectionClusterInfoResponse {
 
 message MoveShard {
   uint32 shard_id = 1; // Local shard id
+  optional uint32 to_shard_id = 5;
+  uint64 from_peer_id = 2;
+  uint64 to_peer_id = 3;
+  optional ShardTransferMethod method = 4;
+}
+
+message ReplicateShard {
+  uint32 shard_id = 1; // Local shard id
+  optional uint32 to_shard_id = 5;
   uint64 from_peer_id = 2;
   uint64 to_peer_id = 3;
   optional ShardTransferMethod method = 4;
 }
 
 message AbortShardTransfer {
   uint32 shard_id = 1; // Local shard id
+  optional uint32 to_shard_id = 4;
   uint64 from_peer_id = 2;
   uint64 to_peer_id = 3;
 }
 
 message RestartTransfer {
   uint32 shard_id = 1; // Local shard id
+  optional uint32 to_shard_id = 5;
   uint64 from_peer_id = 2;
   uint64 to_peer_id = 3;
   ShardTransferMethod method = 4;
@@ -519,6 +553,7 @@ enum ShardTransferMethod {
   StreamRecords = 0; // Stream shard records in batches
   Snapshot = 1; // Snapshot the shard and recover it on the target peer
   WalDelta = 2; // Resolve WAL delta between peers and transfer the difference
+  ReshardingStreamRecords = 3; // Stream shard records in batches for resharding
 }
 
 message Replica {
@@ -541,7 +576,7 @@ message UpdateCollectionClusterSetupRequest {
   string collection_name = 1; // Name of the collection
   oneof operation {
     MoveShard move_shard = 2;
-    MoveShard replicate_shard = 3;
+    ReplicateShard replicate_shard = 3;
     AbortShardTransfer abort_transfer = 4;
     Replica drop_replica = 5;
     CreateShardKey create_shard_key = 7;

proto/points.proto

@@ -45,14 +45,40 @@ message SparseIndices {
   repeated uint32 data = 1;
 }
 
+// Legacy vector format, which determines the vector type by the configuration of its fields.
 message Vector {
+  repeated float data = 1; // Vector data (flatten for multi vectors)
+  optional SparseIndices indices = 2; // Sparse indices for sparse vectors
+  optional uint32 vectors_count = 3; // Number of vectors per multi vector
+}
+
+message DenseVector {
   repeated float data = 1;
-  optional SparseIndices indices = 2;
+}
+
+message SparseVector {
+  repeated float values = 1;
+  repeated uint32 indices = 2;
+}
+
+message MultiDenseVector {
+  repeated DenseVector vectors = 1;
+}
+
+// Vector type to be used in queries. Ids will be substituted with their corresponding vectors from the collection.
+message VectorInput {
+  oneof variant {
+    PointId id = 1;
+    DenseVector dense = 2;
+    SparseVector sparse = 3;
+    MultiDenseVector multi_dense = 4;
+  }
 }
 
 // ---------------------------------------------
 // ----------------- ShardKeySelector ----------
 // ---------------------------------------------
+
 message ShardKeySelector {
   repeated ShardKey shard_keys = 1; // List of shard keys which should be used in the request
 }
@@ -334,7 +360,7 @@ message ScrollPoints {
   optional OrderBy order_by = 10; // Order the records by a payload field
 }
 
-// How to use positive and negative vectors to find the results, default is `AverageVector`:
+// How to use positive and negative vectors to find the results, default is `AverageVector`.
 enum RecommendStrategy {
   // Average positive and negative vectors and create a single query with the formula 
   // `query = avg_pos + avg_pos - avg_neg`. Then performs normal search.
@@ -449,13 +475,84 @@ message DiscoverBatchPoints {
 }
 
 message CountPoints {
-  string collection_name = 1; // name of the collection
+  string collection_name = 1; // Name of the collection
   Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
   optional bool exact = 3; // If `true` - return exact count, if `false` - return approximate count
   optional ReadConsistency read_consistency = 4; // Options for specifying read consistency guarantees
   optional ShardKeySelector shard_key_selector = 5; // Specify in which shards to look for the points, if not specified - look in all shards
 }
 
+message RecommendInput {
+  repeated VectorInput positive = 1; // Look for vectors closest to the vectors from these points
+  repeated VectorInput negative = 2; // Try to avoid vectors like the vector from these points
+  optional RecommendStrategy strategy = 3; // How to use the provided vectors to find the results
+}
+
+message ContextInputPair {
+  VectorInput positive = 1; // A positive vector
+  VectorInput negative = 2; // Repel from this vector
+}
+
+message DiscoverInput {
+  VectorInput target = 1; // Use this as the primary search objective
+  ContextInput context = 2; // Search space will be constrained by these pairs of vectors
+}
+
+message ContextInput {
+    repeated ContextInputPair pairs = 1; // Search space will be constrained by these pairs of vectors
+}
+
+enum Fusion {
+    RRF = 0; // Reciprocal Rank Fusion
+}
+
+message Query {
+  oneof variant {
+    VectorInput nearest = 1; // Find the nearest neighbors to this vector.
+    RecommendInput recommend = 2; // Use multiple positive and negative vectors to find the results.
+    DiscoverInput discover = 3; // Search for nearest points, but constrain the search space with context
+    ContextInput context = 4; // Return points that live in positive areas.
+    OrderBy order_by = 5; // Order the points by a payload field.
+    Fusion fusion = 6; // Fuse the results of multiple prefetches.
+  }
+}
+
+message PrefetchQuery {
+  repeated PrefetchQuery prefetch = 1; // Sub-requests to perform first. If present, the query will be performed on the results of the prefetches.
+  optional Query query = 2; // Query to perform. If missing, returns points ordered by their IDs.
+  optional string using = 3; // Define which vector to use for querying. If missing, the default vector is is used.
+  optional Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions.
+  optional SearchParams params = 5; // Search params for when there is no prefetch.
+  optional float score_threshold = 6; // Return points with scores better than this threshold.
+  optional uint64 limit = 7; // Max number of points. Default is 10
+  optional LookupLocation lookup_from = 8; // The location to use for IDs lookup, if not specified - use the current collection and the 'using' vector
+}
+
+message QueryPoints {
+  string collection_name = 1; // Name of the collection
+  repeated PrefetchQuery prefetch = 2; // Sub-requests to perform first. If present, the query will be performed on the results of the prefetches.
+  optional Query query = 3; // Query to perform. If missing, returns points ordered by their IDs.
+  optional string using = 4; // Define which vector to use for querying. If missing, the default vector is used.
+  optional Filter filter = 5; // Filter conditions - return only those points that satisfy the specified conditions.
+  optional SearchParams params = 6; // Search params for when there is no prefetch.
+  optional float score_threshold = 7; // Return points with scores better than this threshold.
+  optional uint64 limit = 8; // Max number of points. Default is 10.
+  optional uint64 offset = 9; // Offset of the result. Skip this many points. Default is 0.
+  optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into the response.
+  optional WithPayloadSelector with_payload = 11; // Options for specifying which payload to include or not.
+  optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees.
+  optional ShardKeySelector shard_key_selector = 13; // Specify in which shards to look for the points, if not specified - look in all shards.
+  optional LookupLocation lookup_from = 14; // The location to use for IDs lookup, if not specified - use the current collection and the 'using' vector
+  optional uint64 timeout = 15; // If set, overrides global timeout setting for this request. Unit is seconds.
+}
+
+message QueryBatchPoints {
+  string collection_name = 1;
+  repeated QueryPoints query_points = 2;
+  optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees
+  optional uint64 timeout = 4; // If set, overrides global timeout setting for this request. Unit is seconds.
+}
+
 message PointsUpdateOperation {
   message PointStructList {
     repeated PointStruct points = 1;
@@ -467,6 +564,12 @@ message PointsUpdateOperation {
       optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys
       optional string key = 4; // Option for indicate property of payload
   }
+  message OverwritePayload {
+      map<string, Value> payload = 1;
+      optional PointsSelector points_selector = 2; // Affected points
+      optional ShardKeySelector shard_key_selector = 3; // Option for custom sharding to specify used shard keys
+      optional string key = 4; // Option for indicate property of payload
+  }
   message DeletePayload {
       repeated string keys = 1;
       optional PointsSelector points_selector = 2; // Affected points
@@ -494,7 +597,7 @@ message PointsUpdateOperation {
     PointStructList upsert = 1;
     PointsSelector delete_deprecated = 2 [deprecated=true];
     SetPayload set_payload = 3;
-    SetPayload overwrite_payload = 4;
+    OverwritePayload overwrite_payload = 4;
     DeletePayload delete_payload = 5;
     PointsSelector clear_payload_deprecated = 6 [deprecated=true];
     UpdateVectors update_vectors = 7;
@@ -532,6 +635,13 @@ enum UpdateStatus {
   ClockRejected = 3; // Internal: update is rejected due to an outdated clock
 }
 
+message OrderValue {
+  oneof variant {
+    int64 int = 1;
+    double float = 2;
+  }
+}
+
 message ScoredPoint {
   PointId id = 1; // Point id
   map<string, Value> payload = 2; // Payload
@@ -540,6 +650,7 @@ message ScoredPoint {
   uint64 version = 5; // Last update operation applied to this point
   optional Vectors vectors = 6; // Vectors to search
   optional ShardKey shard_key = 7; // Shard key
+  optional OrderValue order_value = 8; // Order by value
 }
 
 message GroupId {
@@ -568,6 +679,16 @@ message SearchResponse {
   double time = 2; // Time spent to process
 }
 
+message QueryResponse {
+  repeated ScoredPoint result = 1;
+  double time = 2; // Time spent to process
+}
+
+message QueryBatchResponse {
+  repeated BatchResult result = 1;
+  double time = 2; // Time spent to process
+}
+
 message BatchResult {
   repeated ScoredPoint result = 1;
 }
@@ -603,6 +724,7 @@ message RetrievedPoint {
   reserved 3; // deprecated "vector" field
   optional Vectors vectors = 4;
   optional ShardKey shard_key = 5; // Shard key
+  optional OrderValue order_value = 6; // Order-by value
 }
 
 message GetResponse {

proto/points_service.proto

@@ -108,4 +108,12 @@ service Points {
   Perform multiple update operations in one request
   */
   rpc UpdateBatch (UpdateBatchPoints) returns (UpdateBatchResponse) {}
+  /*
+  Universally query points. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries.
+  */
+  rpc Query (QueryPoints) returns (QueryResponse) {}
+  /*
+  Universally query points in a batch fashion. This endpoint covers all capabilities of search, recommend, discover, filters. But also enables hybrid and multi-stage queries.
+  */
+  rpc QueryBatch (QueryBatchPoints) returns (QueryBatchResponse) {}
 }