Skip to content

Commit

Permalink
Update protocol definitions for Qdrant 1.13 (#208)
Browse files Browse the repository at this point in the history 
* Update gRPC protocol buffer definitions

* Generate gRPC types, update builders and conversions

* Add max optimization threads builder

* Add bool index params builder

* Add has vector condition builder

* Add dense, sparse and multi dense vector builders and conversions

* Remove has vector condition builder, follow pattern of other conditions
  • Loading branch information
@timvisee
timvisee authored Jan 16, 2025
1 parent 9110a8b commit f2c3883
Show file tree
Hide file tree
Showing 17 changed files with 941 additions and 74 deletions.
45 changes: 39 additions & 6 deletions proto/collections.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,17 @@ enum CompressionRatio {
x64 = 4;
}

message MaxOptimizationThreads {
enum Setting {
Auto = 0;
}

oneof variant {
uint64 value = 1;
Setting setting = 2;
}
}

message OptimizerStatus {
bool ok = 1;
string error = 2;
Expand Down Expand Up @@ -237,7 +248,7 @@ message OptimizersConfigDiff {
optional uint64 max_segment_size = 4;
/*
Maximum size (in kilobytes) of vectors to store in-memory per segment.
Segments larger than this threshold will be stored as read-only memmaped file.
Segments larger than this threshold will be stored as read-only memmapped file.
Memmap storage is disabled by default, to enable it, set this threshold to a reasonable value.
Expand All @@ -260,13 +271,17 @@ message OptimizersConfigDiff {
Interval between forced flushes.
*/
optional uint64 flush_interval_sec = 7;

// Deprecated in favor of `max_optimization_threads`
optional uint64 deprecated_max_optimization_threads = 8;

/*
Max number of threads (jobs) for running optimizations per shard.
Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
If null - have no limit and choose dynamically to saturate CPU.
If "auto" - have no limit and choose dynamically to saturate CPU.
If 0 - no optimization threads, optimizations will be disabled.
*/
optional uint64 max_optimization_threads = 8;
optional MaxOptimizationThreads max_optimization_threads = 9;
}

message ScalarQuantization {
Expand Down Expand Up @@ -320,6 +335,13 @@ message StrictModeConfig {
optional uint32 search_max_hnsw_ef = 6;
optional bool search_allow_exact = 7;
optional float search_max_oversampling = 8;
optional uint64 upsert_max_batchsize = 9;
optional uint64 max_collection_vector_size_bytes = 10;
optional uint32 read_rate_limit = 11; // Max number of read operations per minute per replica
optional uint32 write_rate_limit = 12; // Max number of write operations per minute per replica
optional uint64 max_collection_payload_size_bytes = 13;
optional uint64 filter_max_conditions = 14;
optional uint64 condition_max_size = 15;
}

message CreateCollection {
Expand Down Expand Up @@ -351,6 +373,7 @@ message UpdateCollection {
optional VectorsConfigDiff vectors_config = 6; // New vector parameters
optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector
optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters
optional StrictModeConfig strict_mode_config = 9; // New strict mode configuration
}

message DeleteCollection {
Expand Down Expand Up @@ -430,6 +453,7 @@ message TextIndexParams {
}

message BoolIndexParams {
optional bool on_disk = 1; // If true - store index on disk.
}

message DatetimeIndexParams {
Expand Down Expand Up @@ -530,7 +554,8 @@ enum ReplicaState {
Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
Resharding = 7; // Points are being migrated to this shard as part of resharding
Resharding = 7; // Points are being migrated to this shard as part of scale-up resharding
ReshardingScaleDown = 8; // Points are being migrated to this shard as part of scale-down resharding
}

message ShardKey {
Expand Down Expand Up @@ -566,6 +591,15 @@ message ReshardingInfo {
uint32 shard_id = 1;
uint64 peer_id = 2;
optional ShardKey shard_key = 3;
ReshardingDirection direction = 4;
}

/*
Resharding direction, scale up or down in number of shards
*/
enum ReshardingDirection {
Up = 0; // Scale up, add a new shard
Down = 1; // Scale down, remove a shard
}

message CollectionClusterInfoResponse {
Expand All @@ -574,8 +608,7 @@ message CollectionClusterInfoResponse {
repeated LocalShardInfo local_shards = 3; // Local shards
repeated RemoteShardInfo remote_shards = 4; // Remote shards
repeated ShardTransferInfo shard_transfers = 5; // Shard transfers
// TODO(resharding): enable on release:
// repeated ReshardingInfo resharding_operations = 6; // Resharding operations
repeated ReshardingInfo resharding_operations = 6; // Resharding operations
}

message MoveShard {
Expand Down
90 changes: 85 additions & 5 deletions proto/points.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,48 @@ message SparseIndices {
repeated uint32 data = 1;
}

message Document {
string text = 1; // Text of the document
string model = 3; // Model name
map<string, Value> options = 4; // Model options
}

message Image {
Value image = 1; // Image data, either base64 encoded or URL
string model = 2; // Model name
map<string, Value> options = 3; // Model options
}

message InferenceObject {
Value object = 1; // Object to infer
string model = 2; // Model name
map<string, Value> options = 3; // Model options
}

// Legacy vector format, which determines the vector type by the configuration of its fields.
message Vector {
repeated float data = 1; // Vector data (flatten for multi vectors)
optional SparseIndices indices = 2; // Sparse indices for sparse vectors
optional uint32 vectors_count = 3; // Number of vectors per multi vector
repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
oneof vector {
DenseVector dense = 101; // Dense vector
SparseVector sparse = 102; // Sparse vector
MultiDenseVector multi_dense = 103; // Multi dense vector
Document document = 104;
Image image = 105;
InferenceObject object = 106;
}
}

message VectorOutput {
repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
oneof vector {
DenseVector dense = 101; // Dense vector
SparseVector sparse = 102; // Sparse vector
MultiDenseVector multi_dense = 103; // Multi dense vector
}
}

message DenseVector {
Expand All @@ -73,6 +110,9 @@ message VectorInput {
DenseVector dense = 2;
SparseVector sparse = 3;
MultiDenseVector multi_dense = 4;
Document document = 5;
Image image = 6;
InferenceObject object = 7;
}
}

Expand Down Expand Up @@ -214,13 +254,24 @@ message NamedVectors {
map<string, Vector> vectors = 1;
}

message NamedVectorsOutput {
map<string, VectorOutput> vectors = 1;
}

message Vectors {
oneof vectors_options {
Vector vector = 1;
NamedVectors vectors = 2;
}
}

message VectorsOutput {
oneof vectors_options {
VectorOutput vector = 1;
NamedVectorsOutput vectors = 2;
}
}

message VectorsSelector {
repeated string names = 1; // List of vectors to include into result
}
Expand Down Expand Up @@ -737,7 +788,7 @@ message ScoredPoint {
float score = 3; // Similarity score
reserved 4; // deprecated "vector" field
uint64 version = 5; // Last update operation applied to this point
optional Vectors vectors = 6; // Vectors to search
optional VectorsOutput vectors = 6; // Vectors to search
optional ShardKey shard_key = 7; // Shard key
optional OrderValue order_value = 8; // Order by value
}
Expand Down Expand Up @@ -766,21 +817,25 @@ message GroupsResult {
message SearchResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message QueryResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message QueryBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message QueryGroupsResponse {
GroupsResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message BatchResult {
Expand All @@ -790,16 +845,19 @@ message BatchResult {
message SearchBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message SearchGroupsResponse {
GroupsResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message CountResponse {
CountResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message ScrollResponse {
Expand All @@ -816,7 +874,7 @@ message RetrievedPoint {
PointId id = 1;
map<string, Value> payload = 2;
reserved 3; // deprecated "vector" field
optional Vectors vectors = 4;
optional VectorsOutput vectors = 4;
optional ShardKey shard_key = 5; // Shard key
optional OrderValue order_value = 6; // Order-by value
}
Expand All @@ -829,26 +887,31 @@ message GetResponse {
message RecommendResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message RecommendBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message DiscoverResponse {
repeated ScoredPoint result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message DiscoverBatchResponse {
repeated BatchResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message RecommendGroupsResponse {
GroupsResult result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message UpdateBatchResponse {
Expand All @@ -864,11 +927,13 @@ message FacetResponse {
message SearchMatrixPairsResponse {
SearchMatrixPairs result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

message SearchMatrixOffsetsResponse {
SearchMatrixOffsets result = 1;
double time = 2; // Time spent to process
optional HardwareUsage usage = 3;
}

// ---------------------------------------------
Expand All @@ -895,6 +960,7 @@ message Condition {
Filter filter = 4;
IsNullCondition is_null = 5;
NestedCondition nested = 6;
HasVectorCondition has_vector = 7;
}
}

Expand All @@ -910,6 +976,10 @@ message HasIdCondition {
repeated PointId has_id = 1;
}

message HasVectorCondition {
string has_vector = 1;
}

message NestedCondition {
string key = 1; // Path to nested object
Filter filter = 2; // Filter condition
Expand Down Expand Up @@ -1021,3 +1091,13 @@ message GeoPoint {
double lon = 1;
double lat = 2;
}

// ---------------------------------------------
// ------------ Hardware measurements ----------
// ---------------------------------------------

message HardwareUsage {
uint64 cpu = 1;
uint64 io_read = 2;
uint64 io_write = 3;
}
Loading

0 comments on commit f2c3883

Please sign in to comment.