From 6339e5fb4106bc338738e1156c69b9b5c23cc98b Mon Sep 17 00:00:00 2001 From: "xianliang.li" Date: Thu, 7 Nov 2024 11:46:23 +0800 Subject: [PATCH] use faiss hnsw implementation to replace hnswlib Signed-off-by: xianliang.li --- include/knowhere/comp/index_param.h | 11 +-- include/knowhere/index/index_table.h | 42 +++------ src/index/hnsw/faiss_hnsw.cc | 26 +++-- src/index/hnsw/hnsw.cc | 9 +- src/index/hnsw/hnsw.h | 9 +- src/index/hnsw/impl/IndexBruteForceWrapper.cc | 11 +-- src/index/index_static.cc | 5 +- tests/ut/test_faiss_hnsw.cc | 18 ++-- tests/ut/test_feder.cc | 47 +++++----- tests/ut/test_get_vector.cc | 2 + tests/ut/test_index_check.cc | 94 ++++++++----------- tests/ut/test_iterator.cc | 60 ++++++------ tests/ut/test_search.cc | 21 ++--- 13 files changed, 152 insertions(+), 203 deletions(-) diff --git a/include/knowhere/comp/index_param.h b/include/knowhere/comp/index_param.h index 1b5c847b0..8525c5ad7 100644 --- a/include/knowhere/comp/index_param.h +++ b/include/knowhere/comp/index_param.h @@ -49,14 +49,11 @@ constexpr const char* INDEX_GPU_IVFPQ = "GPU_IVF_PQ"; constexpr const char* INDEX_GPU_CAGRA = "GPU_CAGRA"; constexpr const char* INDEX_HNSW = "HNSW"; -constexpr const char* INDEX_HNSW_SQ8 = "HNSW_SQ8"; -constexpr const char* INDEX_HNSW_SQ8_REFINE = "HNSW_SQ8_REFINE"; -constexpr const char* INDEX_DISKANN = "DISKANN"; +constexpr const char* INDEX_HNSW_SQ = "HNSW_SQ"; +constexpr const char* INDEX_HNSW_PQ = "HNSW_PQ"; +constexpr const char* INDEX_HNSW_PRQ = "HNSW_PRQ"; -constexpr const char* INDEX_FAISS_HNSW_FLAT = "FAISS_HNSW_FLAT"; -constexpr const char* INDEX_FAISS_HNSW_SQ = "FAISS_HNSW_SQ"; -constexpr const char* INDEX_FAISS_HNSW_PQ = "FAISS_HNSW_PQ"; -constexpr const char* INDEX_FAISS_HNSW_PRQ = "FAISS_HNSW_PRQ"; +constexpr const char* INDEX_DISKANN = "DISKANN"; constexpr const char* INDEX_SPARSE_INVERTED_INDEX = "SPARSE_INVERTED_INDEX"; constexpr const char* INDEX_SPARSE_WAND = "SPARSE_WAND"; diff --git a/include/knowhere/index/index_table.h b/include/knowhere/index/index_table.h index 0004d1318..2fc8ada95 100644 --- a/include/knowhere/index/index_table.h +++ b/include/knowhere/index/index_table.h @@ -62,30 +62,17 @@ static std::set> legal_knowhere_index = { {IndexEnum::INDEX_HNSW, VecType::VECTOR_FLOAT16}, {IndexEnum::INDEX_HNSW, VecType::VECTOR_BFLOAT16}, - {IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_BFLOAT16}, + {IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_FLOAT}, + {IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_FLOAT16}, + {IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_BFLOAT16}, - {IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_BFLOAT16}, + {IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_FLOAT}, + {IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_FLOAT16}, + {IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_BFLOAT16}, - // faiss hnsw - {IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_BFLOAT16}, - - {IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_BFLOAT16}, - - {IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_BFLOAT16}, - - {IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT}, - {IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT16}, - {IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_BFLOAT16}, + {IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_FLOAT}, + {IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_FLOAT16}, + {IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_BFLOAT16}, // diskann {IndexEnum::INDEX_DISKANN, VecType::VECTOR_FLOAT}, @@ -112,14 +99,9 @@ static std::set legal_support_mmap_knowhere_index = { // hnsw IndexEnum::INDEX_HNSW, - IndexEnum::INDEX_HNSW_SQ8, - IndexEnum::INDEX_HNSW_SQ8_REFINE, - - // faiss hnsw - IndexEnum::INDEX_FAISS_HNSW_FLAT, - IndexEnum::INDEX_FAISS_HNSW_SQ, - IndexEnum::INDEX_FAISS_HNSW_PQ, - IndexEnum::INDEX_FAISS_HNSW_PRQ, + IndexEnum::INDEX_HNSW_SQ, + IndexEnum::INDEX_HNSW_PQ, + IndexEnum::INDEX_HNSW_PRQ, // sparse index IndexEnum::INDEX_SPARSE_INVERTED_INDEX, diff --git a/src/index/hnsw/faiss_hnsw.cc b/src/index/hnsw/faiss_hnsw.cc index a3c900a80..0f43c83e0 100644 --- a/src/index/hnsw/faiss_hnsw.cc +++ b/src/index/hnsw/faiss_hnsw.cc @@ -1279,7 +1279,7 @@ class BaseFaissRegularIndexHNSWFlatNode : public BaseFaissRegularIndexHNSWNode { std::string Type() const override { - return knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT; + return knowhere::IndexEnum::INDEX_HNSW; } protected: @@ -1789,7 +1789,7 @@ class BaseFaissRegularIndexHNSWSQNode : public BaseFaissRegularIndexHNSWNode { std::string Type() const override { - return knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ; + return knowhere::IndexEnum::INDEX_HNSW_SQ; } protected: @@ -1904,7 +1904,7 @@ class BaseFaissRegularIndexHNSWPQNode : public BaseFaissRegularIndexHNSWNode { std::string Type() const override { - return knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ; + return knowhere::IndexEnum::INDEX_HNSW_PQ; } protected: @@ -2103,7 +2103,7 @@ class BaseFaissRegularIndexHNSWPRQNode : public BaseFaissRegularIndexHNSWNode { std::string Type() const override { - return knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ; + return knowhere::IndexEnum::INDEX_HNSW_PRQ; } protected: @@ -2288,15 +2288,21 @@ class BaseFaissRegularIndexHNSWPRQNodeTemplate : public BaseFaissRegularIndexHNS } }; -// -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(FAISS_HNSW_FLAT, +// MV is only for compatibility +#ifdef KNOWHERE_WITH_CARDINAL +KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_DEPRECATED, BaseFaissRegularIndexHNSWFlatNodeTemplateWithSearchFallback, + knowhere::feature::MMAP | knowhere::feature::MV) +#else +KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW, BaseFaissRegularIndexHNSWFlatNodeTemplateWithSearchFallback, + knowhere::feature::MMAP | knowhere::feature::MV) +#endif + +KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_SQ, BaseFaissRegularIndexHNSWSQNodeTemplate, knowhere::feature::MMAP) -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(FAISS_HNSW_SQ, BaseFaissRegularIndexHNSWSQNodeTemplate, - knowhere::feature::MMAP) -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(FAISS_HNSW_PQ, BaseFaissRegularIndexHNSWPQNodeTemplate, +KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_PQ, BaseFaissRegularIndexHNSWPQNodeTemplate, knowhere::feature::MMAP) -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(FAISS_HNSW_PRQ, BaseFaissRegularIndexHNSWPRQNodeTemplate, +KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_PRQ, BaseFaissRegularIndexHNSWPRQNodeTemplate, knowhere::feature::MMAP) } // namespace knowhere diff --git a/src/index/hnsw/hnsw.cc b/src/index/hnsw/hnsw.cc index b465b662c..15f3213b6 100644 --- a/src/index/hnsw/hnsw.cc +++ b/src/index/hnsw/hnsw.cc @@ -36,14 +36,7 @@ template class HnswIndexNode; template class HnswIndexNode; template class HnswIndexNode; -#ifdef KNOWHERE_WITH_CARDINAL -KNOWHERE_SIMPLE_REGISTER_DENSE_ALL_GLOBAL(HNSW_DEPRECATED, HnswIndexNode, +KNOWHERE_SIMPLE_REGISTER_DENSE_ALL_GLOBAL(HNSWLIB_DEPRECATED, HnswIndexNode, knowhere::feature::MMAP | knowhere::feature::MV) -#else -KNOWHERE_SIMPLE_REGISTER_DENSE_ALL_GLOBAL(HNSW, HnswIndexNode, knowhere::feature::MMAP | knowhere::feature::MV) -#endif -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_SQ8, HnswIndexNode, knowhere::feature::MMAP, QuantType::SQ8) -KNOWHERE_SIMPLE_REGISTER_DENSE_FLOAT_ALL_GLOBAL(HNSW_SQ8_REFINE, HnswIndexNode, knowhere::feature::MMAP, - QuantType::SQ8Refine) } // namespace knowhere diff --git a/src/index/hnsw/hnsw.h b/src/index/hnsw/hnsw.h index 0452eb2e8..687561fcf 100644 --- a/src/index/hnsw/hnsw.h +++ b/src/index/hnsw/hnsw.h @@ -565,14 +565,7 @@ class HnswIndexNode : public IndexNode { std::string Type() const override { - if constexpr (quant_type == QuantType::SQ8) { - return knowhere::IndexEnum::INDEX_HNSW_SQ8; - } else if constexpr (quant_type == QuantType::SQ8Refine) { - return knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE; - - } else { - return knowhere::IndexEnum::INDEX_HNSW; - } + return knowhere::IndexEnum::INDEX_HNSW; } ~HnswIndexNode() override { diff --git a/src/index/hnsw/impl/IndexBruteForceWrapper.cc b/src/index/hnsw/impl/IndexBruteForceWrapper.cc index ba51b823c..748bc8263 100644 --- a/src/index/hnsw/impl/IndexBruteForceWrapper.cc +++ b/src/index/hnsw/impl/IndexBruteForceWrapper.cc @@ -69,17 +69,10 @@ IndexBruteForceWrapper::search(faiss::idx_t n, const float* __restrict x, faiss: // set up a filter faiss::IDSelector* sel = (params == nullptr) ? nullptr : params->sel; - // sel is assumed to be non-null - if (sel == nullptr) { - throw; - } - // try knowhere-specific filter const knowhere::BitsetViewIDSelector* __restrict bw_idselector = dynamic_cast(sel); - BitsetViewIDSelectorWrapper bw_idselector_w(bw_idselector->bitset_view); - if (is_similarity_metric(index->metric_type)) { using C = faiss::CMin; @@ -88,6 +81,8 @@ IndexBruteForceWrapper::search(faiss::idx_t n, const float* __restrict x, faiss: faiss::cppcontrib::knowhere::brute_force_search_impl( index->ntotal, *dis, sel_all, k, local_distances, local_ids); } else { + BitsetViewIDSelectorWrapper bw_idselector_w(bw_idselector->bitset_view); + faiss::cppcontrib::knowhere::brute_force_search_impl( index->ntotal, *dis, bw_idselector_w, k, local_distances, local_ids); @@ -100,6 +95,8 @@ IndexBruteForceWrapper::search(faiss::idx_t n, const float* __restrict x, faiss: faiss::cppcontrib::knowhere::brute_force_search_impl( index->ntotal, *dis, sel_all, k, local_distances, local_ids); } else { + BitsetViewIDSelectorWrapper bw_idselector_w(bw_idselector->bitset_view); + faiss::cppcontrib::knowhere::brute_force_search_impl( index->ntotal, *dis, bw_idselector_w, k, local_distances, local_ids); diff --git a/src/index/index_static.cc b/src/index/index_static.cc index 4746f291c..59d614952 100644 --- a/src/index/index_static.cc +++ b/src/index/index_static.cc @@ -116,9 +116,8 @@ IndexStaticFaced::HasRawData(const IndexType& indexType, const IndexVe } static std::set has_raw_data_index_set = { - IndexEnum::INDEX_FAISS_BIN_IDMAP, IndexEnum::INDEX_FAISS_BIN_IVFFLAT, IndexEnum::INDEX_FAISS_IVFFLAT, - IndexEnum::INDEX_FAISS_IVFFLAT_CC, IndexEnum::INDEX_HNSW_SQ8_REFINE, IndexEnum::INDEX_SPARSE_INVERTED_INDEX, - IndexEnum::INDEX_SPARSE_WAND}; + IndexEnum::INDEX_FAISS_BIN_IDMAP, IndexEnum::INDEX_FAISS_BIN_IVFFLAT, IndexEnum::INDEX_FAISS_IVFFLAT, + IndexEnum::INDEX_FAISS_IVFFLAT_CC, IndexEnum::INDEX_SPARSE_INVERTED_INDEX, IndexEnum::INDEX_SPARSE_WAND}; static std::set has_raw_data_index_alias_set = {"IVFBIN", "BINFLAT", "IVFFLAT", "IVFFLATCC"}; diff --git a/tests/ut/test_faiss_hnsw.cc b/tests/ut/test_faiss_hnsw.cc index 20b285dc5..a767f0b29 100644 --- a/tests/ut/test_faiss_hnsw.cc +++ b/tests/ut/test_faiss_hnsw.cc @@ -464,7 +464,7 @@ TEST_CASE("Search for FAISS HNSW Indices", "Benchmark and validation") { // is parallelized on its own SECTION("FLAT") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -539,7 +539,7 @@ TEST_CASE("Search for FAISS HNSW Indices", "Benchmark and validation") { } SECTION("SQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_SQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -714,7 +714,7 @@ TEST_CASE("Search for FAISS HNSW Indices", "Benchmark and validation") { } SECTION("PQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_PQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -881,7 +881,7 @@ TEST_CASE("Search for FAISS HNSW Indices", "Benchmark and validation") { } SECTION("PRQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_PRQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -1150,7 +1150,7 @@ TEST_CASE("RangeSearch for FAISS HNSW Indices", "Benchmark and validation for Ra // is parallelized on its own SECTION("FLAT") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW; // const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; @@ -1246,7 +1246,7 @@ TEST_CASE("RangeSearch for FAISS HNSW Indices", "Benchmark and validation for Ra } SECTION("SQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_SQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -1435,7 +1435,7 @@ TEST_CASE("RangeSearch for FAISS HNSW Indices", "Benchmark and validation for Ra } SECTION("PQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_PQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -1619,7 +1619,7 @@ TEST_CASE("RangeSearch for FAISS HNSW Indices", "Benchmark and validation for Ra } SECTION("PRQ") { - const std::string& index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ; + const std::string& index_type = knowhere::IndexEnum::INDEX_HNSW_PRQ; const std::string& golden_index_type = knowhere::IndexEnum::INDEX_FAISS_IDMAP; for (size_t distance_type = 0; distance_type < DISTANCE_TYPES.size(); distance_type++) { @@ -1834,7 +1834,7 @@ TEST_CASE("hnswlib to FAISS HNSW for HNSW_FLAT", "Check search fallback") { // const std::string hnswlib_index_type = knowhere::IndexEnum::INDEX_HNSW; - const std::string faiss_index_type = knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT; + const std::string faiss_index_type = knowhere::IndexEnum::INDEX_HNSW; // const auto dim = DIM; diff --git a/tests/ut/test_feder.cc b/tests/ut/test_feder.cc index e172d1f9a..336fa7b49 100644 --- a/tests/ut/test_feder.cc +++ b/tests/ut/test_feder.cc @@ -175,29 +175,30 @@ TEST_CASE("Test Feder", "[feder]") { const knowhere::Json conf = base_gen(); auto gt = knowhere::BruteForce::Search(train_ds, query_ds, conf, nullptr); - SECTION("Test HNSW Feder") { - auto name = knowhere::IndexEnum::INDEX_HNSW; - auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); - REQUIRE(idx.Type() == name); - - auto json = hnsw_gen(); - auto res = idx.Build(train_ds, json); - REQUIRE(res == knowhere::Status::success); - - auto res1 = idx.GetIndexMeta(json); - REQUIRE(res1.has_value()); - CheckHnswMeta(res1.value(), nb, json); - - auto res2 = idx.Search(query_ds, json, nullptr); - REQUIRE(res2.has_value()); - CheckHnswVisitInfo(res2.value(), nb); - - json[knowhere::meta::RADIUS] = 160000; - json[knowhere::meta::RANGE_FILTER] = 0; - auto res3 = idx.RangeSearch(query_ds, json, nullptr); - REQUIRE(res3.has_value()); - CheckHnswVisitInfo(res3.value(), nb); - } + // Feder is deprecated, and faiss hnsw does not implement it + // SECTION("Test HNSW Feder") { + // auto name = knowhere::IndexEnum::INDEX_HNSW; + // auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); + // REQUIRE(idx.Type() == name); + // + // auto json = hnsw_gen(); + // auto res = idx.Build(train_ds, json); + // REQUIRE(res == knowhere::Status::success); + // + // auto res1 = idx.GetIndexMeta(json); + // REQUIRE(res1.has_value()); + // CheckHnswMeta(res1.value(), nb, json); + // + // auto res2 = idx.Search(query_ds, json, nullptr); + // REQUIRE(res2.has_value()); + // CheckHnswVisitInfo(res2.value(), nb); + // + // json[knowhere::meta::RADIUS] = 160000; + // json[knowhere::meta::RANGE_FILTER] = 0; + // auto res3 = idx.RangeSearch(query_ds, json, nullptr); + // REQUIRE(res3.has_value()); + // CheckHnswVisitInfo(res3.value(), nb); + // } SECTION("Test IVF_FLAT Feder") { auto name = knowhere::IndexEnum::INDEX_FAISS_IVFFLAT; diff --git a/tests/ut/test_get_vector.cc b/tests/ut/test_get_vector.cc index 73223d539..61a055528 100644 --- a/tests/ut/test_get_vector.cc +++ b/tests/ut/test_get_vector.cc @@ -60,7 +60,9 @@ TEST_CASE("Test Binary Get Vector By Ids", "[Binary GetVectorByIds]") { auto [name, gen] = GENERATE_REF(table>({ make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, bin_flat_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, bin_ivfflat_gen), +#ifdef KNOWHERE_WITH_CARDINAL make_tuple(knowhere::IndexEnum::INDEX_HNSW, bin_hnsw_gen), +#endif })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump(); diff --git a/tests/ut/test_index_check.cc b/tests/ut/test_index_check.cc index a1c79e4cb..6b0b9b2a3 100644 --- a/tests/ut/test_index_check.cc +++ b/tests/ut/test_index_check.cc @@ -87,30 +87,17 @@ TEST_CASE("Test index and data type check", "[IndexCheckTest]") { CHECK_FALSE(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW, VecType::VECTOR_SPARSE_FLOAT)); #endif - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8, VecType::VECTOR_BFLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_FLOAT)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_FLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ, VecType::VECTOR_BFLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_SQ8_REFINE, VecType::VECTOR_BFLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_FLOAT)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_FLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PQ, VecType::VECTOR_BFLOAT16)); - // faiss hnsw - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_FLAT, VecType::VECTOR_BFLOAT16)); - - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_SQ, VecType::VECTOR_BFLOAT16)); - - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PQ, VecType::VECTOR_BFLOAT16)); - - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_FLOAT16)); - CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PRQ, VecType::VECTOR_BFLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_FLOAT)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_FLOAT16)); + CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_HNSW_PRQ, VecType::VECTOR_BFLOAT16)); // diskann CHECK(KnowhereCheck::IndexTypeAndDataTypeCheck(IndexEnum::INDEX_DISKANN, VecType::VECTOR_FLOAT)); @@ -139,14 +126,9 @@ TEST_CASE("Test support mmap index", "[IndexCheckTest]") { // hnsw CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW)); - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW_SQ8)); - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW_SQ8_REFINE)); - - // faiss hnsw - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_FAISS_HNSW_FLAT)); - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_FAISS_HNSW_SQ)); - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PQ)); - CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_FAISS_HNSW_PRQ)); + CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW_SQ)); + CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW_PQ)); + CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_HNSW_PRQ)); // sparse index CHECK(KnowhereCheck::SupportMmapIndexTypeCheck(IndexEnum::INDEX_SPARSE_INVERTED_INDEX)); @@ -182,14 +164,11 @@ TEST_CASE("Test index has raw data", "[IndexHasRawData]") { CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW, ver, {})); #endif - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ8, ver, {})); - CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ8_REFINE, ver, {})); - // faiss HNSW - CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_FLAT, ver, {})); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, {})); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_PQ, ver, {})); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_PRQ, ver, {})); + CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW, ver, {})); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, {})); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_PQ, ver, {})); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_PRQ, ver, {})); // diskann #ifdef KNOWHERE_WITH_DISKANN @@ -237,24 +216,24 @@ TEST_CASE("Test index has raw data", "[IndexHasRawData]") { CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_SCANN, ver, json)); knowhere::Json faiss_hnsw_cfg = {{"refine", true}, {"refine_type", "bf16"}}; - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); faiss_hnsw_cfg["refine_type"] = "fp16"; - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); faiss_hnsw_cfg["refine_type"] = "sq8"; - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); faiss_hnsw_cfg["refine_type"] = "sq6"; - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); - CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_FAISS_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); + CHECK_FALSE(knowhere::IndexStaticFaced::HasRawData(IndexEnum::INDEX_HNSW_SQ, ver, faiss_hnsw_cfg)); } } @@ -276,13 +255,14 @@ TEST_CASE("Test index feature check", "[IndexFeatureCheck]") { REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_RAFT_IVFPQ, knowhere::feature::MMAP)); REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_RAFT_CAGRA, knowhere::feature::MMAP)); #endif - +#ifdef KNOWHERE_WITH_DISKANN #ifdef KNOWHERE_WITH_CARDINAL REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_DISKANN, knowhere::feature::MMAP)); REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_DISKANN, knowhere::feature::DISK)); #else REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_DISKANN, knowhere::feature::MMAP)); REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_DISKANN, knowhere::feature::DISK)); +#endif #endif REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_IDMAP, knowhere::feature::DISK)); @@ -352,10 +332,10 @@ TEST_CASE("Test index feature check", "[IndexFeatureCheck]") { REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::FLOAT32)); REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::FP16)); REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::BF16)); - REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::BINARY)); // HNSW Index #ifdef KNOWHERE_WITH_CARDINAL + REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::BINARY)); REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::SPARSE_FLOAT32)); #else REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::SPARSE_FLOAT32)); @@ -475,12 +455,12 @@ TEST_CASE("Test index feature check", "[IndexFeatureCheck]") { REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_SCANN, knowhere::feature::MV)); REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_BIN_IDMAP, knowhere::feature::MV)); REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_BIN_IVFFLAT, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_SQ8, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_SQ8_REFINE, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_HNSW_FLAT, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_HNSW_SQ, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_HNSW_PQ, knowhere::feature::MV)); - REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_FAISS_HNSW_PRQ, knowhere::feature::MV)); + REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_SQ, knowhere::feature::MV)); + REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_SQ, knowhere::feature::MV)); + REQUIRE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW, knowhere::feature::MV)); + REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_SQ, knowhere::feature::MV)); + REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_PQ, knowhere::feature::MV)); + REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_HNSW_PRQ, knowhere::feature::MV)); REQUIRE_FALSE( IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_SPARSE_INVERTED_INDEX, knowhere::feature::MV)); REQUIRE_FALSE(IndexFactory::Instance().FeatureCheck(IndexEnum::INDEX_SPARSE_WAND, knowhere::feature::MV)); diff --git a/tests/ut/test_iterator.cc b/tests/ut/test_iterator.cc index 77e1697ab..9e89cd130 100644 --- a/tests/ut/test_iterator.cc +++ b/tests/ut/test_iterator.cc @@ -198,20 +198,20 @@ TEST_CASE("Test Iterator Mem Index With Float Vector", "[float metrics]") { using std::make_tuple; auto [name, gen] = GENERATE_REF(table>({ make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, ivfflatcc_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC, ivf_sq_cc_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_fp16_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_sq8_gen) - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ, hnsw_prq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_flat_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_fp16_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_gen), + // make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_flat_gen), + // make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_sq8_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PRQ, hnsw_prq_gen), })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump(); @@ -287,20 +287,19 @@ TEST_CASE("Test Iterator Mem Index With Float Vector", "[float metrics]") { using std::make_tuple; auto [name, gen] = GENERATE_REF(table>({ make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, ivfflatcc_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC, ivf_sq_cc_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_fp16_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_sq8_gen) - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ, hnsw_prq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_flat_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_fp16_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_gen), + // make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_flat_gen), + // make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_sq8_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PRQ, hnsw_prq_gen), })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump(); @@ -336,20 +335,19 @@ TEST_CASE("Test Iterator Mem Index With Float Vector", "[float metrics]") { using std::make_tuple; auto [name, gen] = GENERATE_REF(table>({ make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFFLAT_CC, ivfflatcc_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ8, ivf_base_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC, ivf_sq_cc_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_FLAT, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_gen), - make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_SQ, hnsw_sq_refine_fp16_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_flat_gen), - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PQ, hnsw_pq_refine_sq8_gen) - // make_tuple(knowhere::IndexEnum::INDEX_FAISS_HNSW_PRQ, hnsw_prq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_flat_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_sq_refine_fp16_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_flat_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PQ, hnsw_pq_refine_sq8_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_PRQ, hnsw_prq_gen), })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump(); @@ -496,6 +494,7 @@ TEST_CASE("Test Iterator Mem Index With Binary Metrics", "[binary metrics]") { {knowhere::meta::TOPK, topk}, }; +#ifdef KNOWHERE_WITH_CARDINAL auto gt = knowhere::BruteForce::Search(train_ds, query_ds, conf, nullptr); SECTION("Test Search using iterator") { using std::make_tuple; @@ -521,6 +520,7 @@ TEST_CASE("Test Iterator Mem Index With Binary Metrics", "[binary metrics]") { float recall = GetKNNRelativeRecall(*search_results.value(), *iterator_results, dist_less_better); REQUIRE(recall > kKnnRecallThreshold); } +#endif } TEST_CASE("Test Iterator BruteForce With Float Vector", "[float metrics]") { diff --git a/tests/ut/test_search.cc b/tests/ut/test_search.cc index 7463c5d82..ddb938752 100644 --- a/tests/ut/test_search.cc +++ b/tests/ut/test_search.cc @@ -165,7 +165,7 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen2), make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), })); knowhere::BinarySet bs; // build process @@ -232,8 +232,8 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { if (metric == knowhere::metric::COSINE) { if (name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 && name != knowhere::IndexEnum::INDEX_FAISS_IVFPQ && - name != knowhere::IndexEnum::INDEX_HNSW_SQ8 && name != knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE && - name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC && !scann_without_raw_data) { + name != knowhere::IndexEnum::INDEX_HNSW_SQ && name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC && + !scann_without_raw_data) { REQUIRE(CheckDistanceInScope(*results.value(), topk, -1.00001, 1.00001)); } } @@ -256,8 +256,7 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen2), make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), })); auto idx_expected = knowhere::IndexFactory::Instance().Create(name, version); if (name == knowhere::IndexEnum::INDEX_FAISS_SCANN) { @@ -293,8 +292,8 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { if (metric == knowhere::metric::COSINE) { if (name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ8 && name != knowhere::IndexEnum::INDEX_FAISS_IVFPQ && - name != knowhere::IndexEnum::INDEX_HNSW_SQ8 && name != knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE && - name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC && !scann_without_raw_data) { + name != knowhere::IndexEnum::INDEX_HNSW_SQ && name != knowhere::IndexEnum::INDEX_FAISS_IVFSQ_CC && + !scann_without_raw_data) { REQUIRE(CheckDistanceInScope(*results.value(), -1.00001, 1.00001)); } } @@ -429,8 +428,7 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { using std::make_tuple; auto [name, gen, threshold] = GENERATE_REF(table, float>({ make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen, hnswlib::kHnswSearchKnnBFFilterThreshold), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen, hnswlib::kHnswSearchKnnBFFilterThreshold), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen, hnswlib::kHnswSearchKnnBFFilterThreshold), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen, hnswlib::kHnswSearchKnnBFFilterThreshold), })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump(); @@ -471,8 +469,7 @@ TEST_CASE("Test Mem Index With Float Vector", "[float metrics]") { make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_SCANN, scann_gen2), make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8, hnsw_gen), - make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ8_REFINE, hnsw_gen), + make_tuple(knowhere::IndexEnum::INDEX_HNSW_SQ, hnsw_gen), })); auto idx_expected = knowhere::IndexFactory::Instance().Create(name, version); @@ -588,7 +585,9 @@ TEST_CASE("Test Mem Index With Binary Vector", "[float metrics]") { auto [name, gen] = GENERATE_REF(table>({ make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IDMAP, flat_gen), make_tuple(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT, ivfflat_gen), +#ifdef KNOWHERE_WITH_CARDINAL make_tuple(knowhere::IndexEnum::INDEX_HNSW, hnsw_gen), +#endif })); auto idx = knowhere::IndexFactory::Instance().Create(name, version).value(); auto cfg_json = gen().dump();