enhance: pick prs from main for knowhere v2.3.13

cmake/libs/libfaiss.cmake

@@ -55,7 +55,7 @@ endif()
 
 # ToDo: Add distances_vsx.cc for powerpc64 SIMD acceleration
 if(__PPC64)
-  set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc)
+  set(UTILS_SRC src/simd/hook.cc src/simd/distances_ref.cc src/simd/distances_powerpc.cc)
   add_library(knowhere_utils STATIC ${UTILS_SRC})
   target_link_libraries(knowhere_utils PUBLIC glog::glog)
 endif()

cmake/libs/libraft.cmake

@@ -17,7 +17,7 @@ add_definitions(-DKNOWHERE_WITH_RAFT)
 add_definitions(-DRAFT_EXPLICIT_INSTANTIATE_ONLY)
 set(RAFT_VERSION "${RAPIDS_VERSION}")
 set(RAFT_FORK "milvus-io")
-set(RAFT_PINNED_TAG "branch-24.04")
+set(RAFT_PINNED_TAG "branch-24.10")
 
 rapids_find_package(CUDAToolkit REQUIRED BUILD_EXPORT_SET knowhere-exports
                     INSTALL_EXPORT_SET knowhere-exports)

cmake/libs/librapids.cmake

@@ -13,7 +13,7 @@
 # License for the specific language governing permissions and limitations under
 # the License.
 
-set(RAPIDS_VERSION 24.04)
+set(RAPIDS_VERSION 24.10)
 
 if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/RAPIDS.cmake)
   file(

include/knowhere/comp/index_param.h

@@ -60,6 +60,8 @@ constexpr const char* INDEX_FAISS_HNSW_PRQ = "FAISS_HNSW_PRQ";
 
 constexpr const char* INDEX_SPARSE_INVERTED_INDEX = "SPARSE_INVERTED_INDEX";
 constexpr const char* INDEX_SPARSE_WAND = "SPARSE_WAND";
+constexpr const char* INDEX_SPARSE_INVERTED_INDEX_CC = "SPARSE_INVERTED_INDEX_CC";
+constexpr const char* INDEX_SPARSE_WAND_CC = "SPARSE_WAND_CC";
 }  // namespace IndexEnum
 
 namespace ClusterEnum {

include/knowhere/config.h

@@ -58,6 +58,32 @@ typedef nlohmann::json Json;
 #define CFG_MATERIALIZED_VIEW_SEARCH_INFO_TYPE std::optional<knowhere::MaterializedViewSearchInfo>
 #endif
 
+template <typename T>
+struct Range {
+    T left;
+    T right;
+    bool include_left;
+    bool include_right;
+
+    Range(T left, T right, bool includeLeft, bool includeRight)
+        : left(left), right(right), include_left(includeLeft), include_right(includeRight) {
+    }
+
+    bool
+    within(T val) {
+        bool left_range_check = left < val || (include_left && left <= val);
+        bool right_range_check = val < right || (include_right && val <= right);
+        return left_range_check && right_range_check;
+    }
+
+    std::string
+    to_string() {
+        std::string left_mark = include_left ? "[" : "(";
+        std::string right_mark = include_right ? "]" : ")";
+        return left_mark + std::to_string(left) + ", " + std::to_string(right) + right_mark;
+    }
+};
+
 template <typename T>
 struct Entry {};
 
@@ -114,7 +140,7 @@ struct Entry<CFG_FLOAT> {
     CFG_FLOAT* val;
     std::optional<CFG_FLOAT::value_type> default_val;
     uint32_t type;
-    std::optional<std::pair<CFG_FLOAT::value_type, CFG_FLOAT::value_type>> range;
+    std::optional<Range<CFG_FLOAT::value_type>> range;
     std::optional<std::string> desc;
     bool allow_empty_without_default = false;
 };
@@ -139,7 +165,7 @@ struct Entry<CFG_INT> {
     CFG_INT* val;
     std::optional<CFG_INT::value_type> default_val;
     uint32_t type;
-    std::optional<std::pair<CFG_INT::value_type, CFG_INT::value_type>> range;
+    std::optional<Range<CFG_INT::value_type>> range;
     std::optional<std::string> desc;
     bool allow_empty_without_default = false;
 };
@@ -164,7 +190,7 @@ struct Entry<CFG_INT64> {
     CFG_INT64* val;
     std::optional<CFG_INT64::value_type> default_val;
     uint32_t type;
-    std::optional<std::pair<CFG_INT64::value_type, CFG_INT64::value_type>> range;
+    std::optional<Range<CFG_INT64::value_type>> range;
     std::optional<std::string> desc;
     bool allow_empty_without_default = false;
 };
@@ -228,8 +254,8 @@ class EntryAccess {
     }
 
     EntryAccess&
-    set_range(typename T::value_type a, typename T::value_type b) {
-        entry->range = std::make_pair(a, b);
+    set_range(typename T::value_type a, typename T::value_type b, bool include_left = true, bool include_right = true) {
+        entry->range = Range<typename T::value_type>(a, b, include_left, include_right);
         return *this;
     }
 
@@ -360,13 +386,11 @@ class Config {
                     }
                     CFG_INT::value_type v = json[it.first];
                     auto range_val = ptr->range.value();
-                    if (range_val.first <= v && v <= range_val.second) {
+                    if (range_val.within(v)) {
                         *ptr->val = v;
                     } else {
                         std::string msg = "Out of range in json: param '" + it.first + "' (" +
-                                          to_string(json[it.first]) + ") should be in range [" +
-                                          std::to_string(range_val.first) + ", " + std::to_string(range_val.second) +
-                                          "]";
+                                          to_string(json[it.first]) + ") should be in range " + range_val.to_string();
                         show_err_msg(msg);
                         return Status::out_of_range_in_json;
                     }
@@ -408,13 +432,11 @@ class Config {
                     }
                     CFG_INT64::value_type v = json[it.first];
                     auto range_val = ptr->range.value();
-                    if (range_val.first <= v && v <= range_val.second) {
+                    if (range_val.within(v)) {
                         *ptr->val = v;
                     } else {
                         std::string msg = "Out of range in json: param '" + it.first + "' (" +
-                                          to_string(json[it.first]) + ") should be in range [" +
-                                          std::to_string(range_val.first) + ", " + std::to_string(range_val.second) +
-                                          "]";
+                                          to_string(json[it.first]) + ") should be in range " + range_val.to_string();
                         show_err_msg(msg);
                         return Status::out_of_range_in_json;
                     }
@@ -456,13 +478,11 @@ class Config {
                     }
                     CFG_FLOAT::value_type v = json[it.first];
                     auto range_val = ptr->range.value();
-                    if (range_val.first <= v && v <= range_val.second) {
+                    if (range_val.within(v)) {
                         *ptr->val = v;
                     } else {
                         std::string msg = "Out of range in json: param '" + it.first + "' (" +
-                                          to_string(json[it.first]) + ") should be in range [" +
-                                          std::to_string(range_val.first) + ", " + std::to_string(range_val.second) +
-                                          "]";
+                                          to_string(json[it.first]) + ") should be in range " + range_val.to_string();
                         show_err_msg(msg);
                         return Status::out_of_range_in_json;
                     }

include/knowhere/expected.h

@@ -46,6 +46,9 @@ enum class Status {
     invalid_cluster_error = 24,
     cluster_inner_error = 25,
     timeout = 26,
+    internal_error = 27,
+    invalid_serialized_index_type = 28,
+    sparse_inner_error = 29,
 };
 
 inline std::string
@@ -95,6 +98,12 @@ Status2String(knowhere::Status status) {
             return "invalid cluster type";
         case knowhere::Status::cluster_inner_error:
             return "cluster inner error";
+        case knowhere::Status::internal_error:
+            return "internal error (something that must not have happened at all)";
+        case knowhere::Status::invalid_serialized_index_type:
+            return "the serialized index type is not recognized";
+        case knowhere::Status::sparse_inner_error:
+            return "sparse index inner error";
         default:
             return "unexpected status";
     }

include/knowhere/sparse_utils.h

@@ -274,4 +274,82 @@ class MaxMinHeap {
     std::vector<SparseIdVal<T>> pool_;
 };  // class MaxMinHeap
 
+// A std::vector like container but uses fixed size free memory(typically from
+// mmap) as backing store and can only be appended at the end.
+//
+// Must be initialized with a valid pointer to memory when used. The memory must be
+// valid during the lifetime of this object. After initialization, GrowableVectorView will
+// have space for mmap_element_count_ elements, none of which are initialized.
+//
+// Currently only used in sparse InvertedIndex. Move to other places if needed.
+template <typename T>
+class GrowableVectorView {
+ public:
+    using value_type = T;
+    using size_type = size_t;
+
+    GrowableVectorView() = default;
+
+    void
+    initialize(void* data, size_type byte_size) {
+        if (byte_size % sizeof(T) != 0) {
+            throw std::invalid_argument("GrowableVectorView byte_size must be a multiple of element size");
+        }
+        mmap_data_ = data;
+        mmap_byte_size_ = byte_size;
+        mmap_element_count_ = 0;
+    }
+
+    size_type
+    capacity() const {
+        return mmap_byte_size_ / sizeof(T);
+    }
+
+    size_type
+    size() const {
+        return mmap_element_count_;
+    }
+
+    template <typename... Args>
+    T&
+    emplace_back(Args&&... args) {
+        if (size() == capacity()) {
+            throw std::out_of_range("emplace_back on a full GrowableVectorView");
+        }
+        auto* elem = reinterpret_cast<T*>(mmap_data_) + mmap_element_count_++;
+        return *new (elem) T(std::forward<Args>(args)...);
+    }
+
+    T&
+    operator[](size_type i) {
+        return reinterpret_cast<T*>(mmap_data_)[i];
+    }
+
+    const T&
+    operator[](size_type i) const {
+        return reinterpret_cast<const T*>(mmap_data_)[i];
+    }
+
+    T&
+    at(size_type i) {
+        if (i >= mmap_element_count_) {
+            throw std::out_of_range("GrowableVectorView index out of range");
+        }
+        return reinterpret_cast<T*>(mmap_data_)[i];
+    }
+
+    const T&
+    at(size_type i) const {
+        if (i >= mmap_element_count_) {
+            throw std::out_of_range("GrowableVectorView index out of range");
+        }
+        return reinterpret_cast<const T*>(mmap_data_)[i];
+    }
+
+ private:
+    void* mmap_data_ = nullptr;
+    size_type mmap_byte_size_ = 0;
+    size_type mmap_element_count_ = 0;
+};
+
 }  // namespace knowhere::sparse

src/common/comp/brute_force.cc

@@ -111,7 +111,12 @@ BruteForce::Search(const DataSetPtr base_dataset, const DataSetPtr query_dataset
     int topk = cfg.k.value();
     auto labels = std::make_unique<int64_t[]>(nq * topk);
     auto distances = std::make_unique<float[]>(nq * topk);
-
+    std::unique_ptr<float[]> norms = nullptr;
+    if (is_cosine) {
+        ThreadPool::ScopedSearchOmpSetter setter(1);
+        norms = std::make_unique<float[]>(nb);
+        faiss::fvec_norms_L2(norms.get(), (float*)xb, dim, nb);
+    }
     auto pool = ThreadPool::GetGlobalSearchThreadPool();
     std::vector<folly::Future<Status>> futs;
     futs.reserve(nq);
@@ -136,7 +141,8 @@ BruteForce::Search(const DataSetPtr base_dataset, const DataSetPtr query_dataset
                     faiss::float_minheap_array_t buf{(size_t)1, (size_t)topk, cur_labels, cur_distances};
                     if (is_cosine) {
                         auto copied_query = CopyAndNormalizeVecs(cur_query, 1, dim);
-                        faiss::knn_cosine(copied_query.get(), (const float*)xb, nullptr, dim, 1, nb, &buf, id_selector);
+                        faiss::knn_cosine(copied_query.get(), (const float*)xb, norms.get(), dim, 1, nb, &buf,
+                                          id_selector);
                     } else {
                         faiss::knn_inner_product(cur_query, (const float*)xb, dim, 1, nb, &buf, id_selector);
                     }
@@ -239,6 +245,13 @@ BruteForce::SearchWithBuf(const DataSetPtr base_dataset, const DataSetPtr query_
     auto labels = ids;
     auto distances = dis;
 
+    std::unique_ptr<float[]> norms = nullptr;
+    if (is_cosine) {
+        ThreadPool::ScopedSearchOmpSetter setter(1);
+        norms = std::make_unique<float[]>(nb);
+        faiss::fvec_norms_L2(norms.get(), (float*)xb, dim, nb);
+    }
+
     auto pool = ThreadPool::GetGlobalSearchThreadPool();
     std::vector<folly::Future<Status>> futs;
     futs.reserve(nq);
@@ -263,7 +276,8 @@ BruteForce::SearchWithBuf(const DataSetPtr base_dataset, const DataSetPtr query_
                     faiss::float_minheap_array_t buf{(size_t)1, (size_t)topk, cur_labels, cur_distances};
                     if (is_cosine) {
                         auto copied_query = CopyAndNormalizeVecs(cur_query, 1, dim);
-                        faiss::knn_cosine(copied_query.get(), (const float*)xb, nullptr, dim, 1, nb, &buf, id_selector);
+                        faiss::knn_cosine(copied_query.get(), (const float*)xb, norms.get(), dim, 1, nb, &buf,
+                                          id_selector);
                     } else {
                         faiss::knn_inner_product(cur_query, (const float*)xb, dim, 1, nb, &buf, id_selector);
                     }
@@ -393,6 +407,13 @@ BruteForce::RangeSearch(const DataSetPtr base_dataset, const DataSetPtr query_da
     std::vector<std::vector<int64_t>> result_id_array(nq);
     std::vector<std::vector<float>> result_dist_array(nq);
 
+    std::unique_ptr<float[]> norms = nullptr;
+    if (is_cosine) {
+        ThreadPool::ScopedSearchOmpSetter setter(1);
+        norms = std::make_unique<float[]>(nb);
+        faiss::fvec_norms_L2(norms.get(), (float*)xb, dim, nb);
+    }
+
     std::vector<folly::Future<Status>> futs;
     futs.reserve(nq);
     for (int i = 0; i < nq; ++i) {
@@ -437,8 +458,8 @@ BruteForce::RangeSearch(const DataSetPtr base_dataset, const DataSetPtr query_da
                     auto cur_query = (const float*)xq + dim * index;
                     if (is_cosine) {
                         auto copied_query = CopyAndNormalizeVecs(cur_query, 1, dim);
-                        faiss::range_search_cosine(copied_query.get(), (const float*)xb, nullptr, dim, 1, nb, radius,
-                                                   &res, id_selector);
+                        faiss::range_search_cosine(copied_query.get(), (const float*)xb, norms.get(), dim, 1, nb,
+                                                   radius, &res, id_selector);
                     } else {
                         faiss::range_search_inner_product(cur_query, (const float*)xb, dim, 1, nb, radius, &res,
                                                           id_selector);
@@ -660,6 +681,13 @@ BruteForce::AnnIterator(const DataSetPtr base_dataset, const DataSetPtr query_da
     faiss::MetricType faiss_metric_type = result.value();
     bool is_cosine = IsMetricType(metric_str, metric::COSINE);
 
+    std::unique_ptr<float[]> norms = nullptr;
+    if (is_cosine) {
+        ThreadPool::ScopedSearchOmpSetter setter(1);
+        norms = std::make_unique<float[]>(nb);
+        faiss::fvec_norms_L2(norms.get(), (float*)xb, dim, nb);
+    }
+
     auto pool = ThreadPool::GetGlobalSearchThreadPool();
     auto vec = std::vector<IndexNode::IteratorPtr>(nq, nullptr);
     std::vector<folly::Future<Status>> futs;
@@ -685,7 +713,7 @@ BruteForce::AnnIterator(const DataSetPtr base_dataset, const DataSetPtr query_da
                     auto cur_query = (const float*)xq + dim * index;
                     if (is_cosine) {
                         auto copied_query = CopyAndNormalizeVecs(cur_query, 1, dim);
-                        faiss::all_cosine(copied_query.get(), (const float*)xb, nullptr, dim, 1, nb, distances_ids,
+                        faiss::all_cosine(copied_query.get(), (const float*)xb, norms.get(), dim, 1, nb, distances_ids,
                                           id_selector);
                     } else {
                         faiss::all_inner_product(cur_query, (const float*)xb, dim, 1, nb, distances_ids, id_selector);