From 92f0b9f365de88ba302e573c1d7407fe1d293df5 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Tue, 3 Feb 2026 16:19:24 -0800 Subject: [PATCH 01/12] Start adding cpp runtime support for ivf index --- bindings/cpp/CMakeLists.txt | 21 + bindings/cpp/include/svs/runtime/ivf_index.h | 235 +++++++ bindings/cpp/src/ivf_index.cpp | 276 ++++++++ bindings/cpp/src/ivf_index_impl.h | 635 +++++++++++++++++++ bindings/cpp/tests/CMakeLists.txt | 5 + bindings/cpp/tests/runtime_test.cpp | 465 ++++++++++++++ 6 files changed, 1637 insertions(+) create mode 100644 bindings/cpp/include/svs/runtime/ivf_index.h create mode 100644 bindings/cpp/src/ivf_index.cpp create mode 100644 bindings/cpp/src/ivf_index_impl.h diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 122e5c8f7..ad0766134 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -16,6 +16,9 @@ cmake_minimum_required(VERSION 3.21) project(svs_runtime VERSION 0.1.0 LANGUAGES CXX) set(TARGET_NAME svs_runtime) +# IVF requires MKL, so it's optional +option(SVS_RUNTIME_ENABLE_IVF "Enable compilation of SVS runtime with IVF support (requires MKL)" OFF) + set(SVS_RUNTIME_HEADERS include/svs/runtime/version.h include/svs/runtime/api_defs.h @@ -38,6 +41,15 @@ set(SVS_RUNTIME_SOURCES src/flat_index.cpp ) +# Add IVF files if enabled +if (SVS_RUNTIME_ENABLE_IVF) + message(STATUS "SVS runtime will be built with IVF support (requires MKL)") + list(APPEND SVS_RUNTIME_HEADERS include/svs/runtime/ivf_index.h) + list(APPEND SVS_RUNTIME_SOURCES src/ivf_index_impl.h src/ivf_index.cpp) +else() + message(STATUS "SVS runtime will be built without IVF support") +endif() + option(SVS_RUNTIME_ENABLE_LVQ_LEANVEC "Enable compilation of SVS runtime with LVQ and LeanVec support" ON) if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC) message(STATUS "SVS runtime will be built with LVQ support") @@ -130,6 +142,10 @@ if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC) else() # Include the SVS library directly if needed. if (NOT TARGET svs::svs) + # Pass IVF flag to parent build if IVF is enabled + if (SVS_RUNTIME_ENABLE_IVF) + set(SVS_EXPERIMENTAL_ENABLE_IVF ON CACHE BOOL "" FORCE) + endif() add_subdirectory("../.." "${CMAKE_CURRENT_BINARY_DIR}/svs") endif() target_link_libraries(${TARGET_NAME} PRIVATE @@ -139,6 +155,11 @@ else() ) endif() +# IVF requires Intel(R) MKL support +if (SVS_RUNTIME_ENABLE_IVF) + target_compile_definitions(${TARGET_NAME} PRIVATE SVS_HAVE_MKL=1) +endif() + # installing include(GNUInstallDirs) diff --git a/bindings/cpp/include/svs/runtime/ivf_index.h b/bindings/cpp/include/svs/runtime/ivf_index.h new file mode 100644 index 000000000..b780571a3 --- /dev/null +++ b/bindings/cpp/include/svs/runtime/ivf_index.h @@ -0,0 +1,235 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include + +#include +#include +#include + +namespace svs { +namespace runtime { +namespace v0 { + +// Abstract interface for IVF (Inverted File) indices. +struct SVS_RUNTIME_API IVFIndex { + virtual ~IVFIndex(); + + /// @brief Parameters for building an IVF index. + struct BuildParams { + /// The number of centroids/clusters to create + size_t num_centroids = Unspecify(); + /// Minibatch size for k-means clustering + size_t minibatch_size = Unspecify(); + /// Number of iterations for k-means clustering + size_t num_iterations = Unspecify(); + /// Whether to use hierarchical clustering + OptionalBool is_hierarchical = Unspecify(); + /// Fraction of data to use for training (0.0 to 1.0) + float training_fraction = Unspecify(); + /// Number of level-1 clusters for hierarchical clustering + size_t hierarchical_level1_clusters = Unspecify(); + /// Random seed for clustering + size_t seed = Unspecify(); + }; + + /// @brief Parameters for IVF search operations. + struct SearchParams { + /// The number of nearest clusters to be explored during search + size_t n_probes = Unspecify(); + /// Level of reordering/reranking done when using compressed datasets (multiplier) + float k_reorder = Unspecify(); + }; + + /// @brief Perform k-NN search on the index. + /// + /// @param n Number of query vectors. + /// @param x Pointer to query vectors (row-major, n x dimensions). + /// @param k Number of nearest neighbors to find. + /// @param distances Output array for distances (n x k). + /// @param labels Output array for neighbor IDs (n x k). + /// @param params Optional search parameters (uses defaults if nullptr). + /// @return Status indicating success or error. + virtual Status search( + size_t n, + const float* x, + size_t k, + float* distances, + size_t* labels, + const SearchParams* params = nullptr + ) const noexcept = 0; +}; + +/// @brief Abstract interface for static IVF indices (read-only after construction). +struct SVS_RUNTIME_API StaticIVFIndex : public IVFIndex { + /// @brief Utility function to check storage kind support. + static Status check_storage_kind(StorageKind storage_kind) noexcept; + + /// @brief Build a static IVF index from data. + /// + /// @param index Output pointer to the created index. + /// @param dim Dimensionality of vectors. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param n Number of vectors in the dataset. + /// @param data Pointer to vector data (row-major, n x dim). + /// @param params Build parameters for clustering. + /// @param default_search_params Default search parameters. + /// @param num_threads Number of threads for building and searching. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status build( + StaticIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; + + /// @brief Destroy a static IVF index. + static Status destroy(StaticIVFIndex* index) noexcept; + + /// @brief Save the index to a stream. + virtual Status save(std::ostream& out) const noexcept = 0; + + /// @brief Load a static IVF index from a stream. + /// + /// @param index Output pointer to the loaded index. + /// @param in Input stream containing the serialized index. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param num_threads Number of threads for searching. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status load( + StaticIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; +}; + +/// @brief Abstract interface for dynamic IVF indices (supports add/delete). +struct SVS_RUNTIME_API DynamicIVFIndex : public IVFIndex { + /// @brief Utility function to check storage kind support. + static Status check_storage_kind(StorageKind storage_kind) noexcept; + + /// @brief Build a dynamic IVF index. + /// + /// @param index Output pointer to the created index. + /// @param dim Dimensionality of vectors. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param n Number of initial vectors (can be 0 for empty index). + /// @param data Pointer to initial vector data (can be nullptr if n=0). + /// @param labels Pointer to labels for initial vectors (can be nullptr if n=0). + /// @param params Build parameters for clustering. + /// @param default_search_params Default search parameters. + /// @param num_threads Number of threads for operations. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; + + /// @brief Destroy a dynamic IVF index. + static Status destroy(DynamicIVFIndex* index) noexcept; + + /// @brief Add vectors to the index. + /// + /// @param n Number of vectors to add. + /// @param labels Pointer to labels for the new vectors. + /// @param x Pointer to vector data (row-major, n x dimensions). + /// @param reuse_empty Whether to reuse empty slots from deleted vectors. + /// @return Status indicating success or error. + virtual Status + add(size_t n, const size_t* labels, const float* x, bool reuse_empty = false + ) noexcept = 0; + + /// @brief Remove vectors from the index by ID. + /// + /// @param n Number of vectors to remove. + /// @param labels Pointer to labels of vectors to remove. + /// @return Status indicating success or error. + virtual Status remove(size_t n, const size_t* labels) noexcept = 0; + + /// @brief Remove vectors matching a selector. + /// + /// @param num_removed Output: number of vectors actually removed. + /// @param selector Filter to determine which vectors to remove. + /// @return Status indicating success or error. + virtual Status + remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; + + /// @brief Check if an ID exists in the index. + /// + /// @param exists Output: true if the ID exists. + /// @param id The ID to check. + /// @return Status indicating success or error. + virtual Status has_id(bool* exists, size_t id) const noexcept = 0; + + /// @brief Consolidate the index (clean up deleted entries). + virtual Status consolidate() noexcept = 0; + + /// @brief Compact the index (reclaim memory from deleted entries). + /// + /// @param batchsize Number of entries to process per batch. + /// @return Status indicating success or error. + virtual Status compact(size_t batchsize = 1'000'000) noexcept = 0; + + /// @brief Save the index to a stream. + virtual Status save(std::ostream& out) const noexcept = 0; + + /// @brief Load a dynamic IVF index from a stream. + /// + /// @param index Output pointer to the loaded index. + /// @param in Input stream containing the serialized index. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param num_threads Number of threads for operations. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status load( + DynamicIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; +}; + +} // namespace v0 +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/src/ivf_index.cpp b/bindings/cpp/src/ivf_index.cpp new file mode 100644 index 000000000..5027e9bc1 --- /dev/null +++ b/bindings/cpp/src/ivf_index.cpp @@ -0,0 +1,276 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "svs/runtime/ivf_index.h" + +#include "ivf_index_impl.h" +#include "svs_runtime_utils.h" + +#include +#include +#include + +#include +#include +#include +#include + +namespace svs { +namespace runtime { + +namespace { + +// Manager class for Static IVF Index +struct StaticIVFIndexManager : public StaticIVFIndex { + std::unique_ptr impl_; + + StaticIVFIndexManager(std::unique_ptr impl) + : impl_{std::move(impl)} { + assert(impl_ != nullptr); + } + + ~StaticIVFIndexManager() override = default; + + Status search( + size_t n, + const float* x, + size_t k, + float* distances, + size_t* labels, + const SearchParams* params = nullptr + ) const noexcept override { + return runtime_error_wrapper([&] { + auto result = svs::QueryResultView{ + svs::MatrixView{svs::make_dims(n, k), labels}, + svs::MatrixView{svs::make_dims(n, k), distances}}; + auto queries = svs::data::ConstSimpleDataView(x, n, impl_->dimensions()); + impl_->search(result, queries, params); + }); + } + + Status save(std::ostream& out) const noexcept override { + return runtime_error_wrapper([&] { impl_->save(out); }); + } +}; + +// Manager class for Dynamic IVF Index +struct DynamicIVFIndexManager : public DynamicIVFIndex { + std::unique_ptr impl_; + + DynamicIVFIndexManager(std::unique_ptr impl) + : impl_{std::move(impl)} { + assert(impl_ != nullptr); + } + + ~DynamicIVFIndexManager() override = default; + + Status search( + size_t n, + const float* x, + size_t k, + float* distances, + size_t* labels, + const SearchParams* params = nullptr + ) const noexcept override { + return runtime_error_wrapper([&] { + auto result = svs::QueryResultView{ + svs::MatrixView{svs::make_dims(n, k), labels}, + svs::MatrixView{svs::make_dims(n, k), distances}}; + auto queries = svs::data::ConstSimpleDataView(x, n, impl_->dimensions()); + impl_->search(result, queries, params); + }); + } + + Status + add(size_t n, const size_t* labels, const float* x, bool reuse_empty + ) noexcept override { + return runtime_error_wrapper([&] { + svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; + std::span lbls(labels, n); + impl_->add(data, lbls, reuse_empty); + }); + } + + Status remove(size_t n, const size_t* labels) noexcept override { + return runtime_error_wrapper([&] { + std::span lbls(labels, n); + impl_->remove(lbls); + }); + } + + Status + remove_selected(size_t* num_removed, const IDFilter& selector) noexcept override { + return runtime_error_wrapper([&] { + *num_removed = impl_->remove_selected(selector); + }); + } + + Status has_id(bool* exists, size_t id) const noexcept override { + return runtime_error_wrapper([&] { *exists = impl_->has_id(id); }); + } + + Status consolidate() noexcept override { + return runtime_error_wrapper([&] { impl_->consolidate(); }); + } + + Status compact(size_t batchsize) noexcept override { + return runtime_error_wrapper([&] { impl_->compact(batchsize); }); + } + + Status save(std::ostream& out) const noexcept override { + return runtime_error_wrapper([&] { impl_->save(out); }); + } +}; + +} // namespace + +// IVFIndex interface implementation +IVFIndex::~IVFIndex() = default; + +// StaticIVFIndex interface implementation +Status StaticIVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { + if (ivf_storage::is_supported_storage_kind(storage_kind)) { + return Status_Ok; + } else { + return Status{ + ErrorCode::INVALID_ARGUMENT, + "StaticIVFIndex only supports FP32 and FP16 storage kinds"}; + } +} + +Status StaticIVFIndex::build( + StaticIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + auto impl = std::make_unique( + dim, + metric, + storage_kind, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + // Build with provided data + svs::data::ConstSimpleDataView data_view{data, n, dim}; + impl->build(data_view); + + *index = new StaticIVFIndexManager{std::move(impl)}; + }); +} + +Status StaticIVFIndex::destroy(StaticIVFIndex* index) noexcept { + return runtime_error_wrapper([&] { delete index; }); +} + +Status StaticIVFIndex::load( + StaticIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + std::unique_ptr impl{StaticIVFIndexImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )}; + *index = new StaticIVFIndexManager{std::move(impl)}; + }); +} + +// DynamicIVFIndex interface implementation +Status DynamicIVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { + if (ivf_storage::is_supported_storage_kind(storage_kind)) { + return Status_Ok; + } else { + return Status{ + ErrorCode::INVALID_ARGUMENT, + "DynamicIVFIndex only supports FP32 and FP16 storage kinds"}; + } +} + +Status DynamicIVFIndex::build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + auto impl = std::make_unique( + dim, + metric, + storage_kind, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + // Build with provided data if any + if (n > 0 && data != nullptr && labels != nullptr) { + svs::data::ConstSimpleDataView data_view{data, n, dim}; + std::span labels_span{labels, n}; + impl->build(data_view, labels_span); + } + + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} + +Status DynamicIVFIndex::destroy(DynamicIVFIndex* index) noexcept { + return runtime_error_wrapper([&] { delete index; }); +} + +Status DynamicIVFIndex::load( + DynamicIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + std::unique_ptr impl{DynamicIVFIndexImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )}; + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} + +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/src/ivf_index_impl.h b/bindings/cpp/src/ivf_index_impl.h new file mode 100644 index 000000000..c58684b14 --- /dev/null +++ b/bindings/cpp/src/ivf_index_impl.h @@ -0,0 +1,635 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "svs/runtime/ivf_index.h" +#include "svs_runtime_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace svs { +namespace runtime { + +// IVF storage kind support - IVF supports a subset of storage kinds +namespace ivf_storage { + +// IVF supports FP32 and FP16 storage kinds +inline bool is_supported_storage_kind(StorageKind kind) { + switch (kind) { + case StorageKind::FP32: + case StorageKind::FP16: + return true; + default: + return false; + } +} + +// IVF data type for static index (uses lib::Allocator) +template +using IVFDataType = svs::data::SimpleData>; + +// IVF data type for dynamic index (uses Blocked allocator) +template +using IVFBlockedDataType = + svs::data::SimpleData>>; + +// Dispatch on storage kind for IVF operations +template +auto dispatch_ivf_storage_kind(StorageKind kind, F&& f, Args&&... args) { + switch (kind) { + case StorageKind::FP32: + return f(svs::lib::Type>{}, std::forward(args)...); + case StorageKind::FP16: + return f( + svs::lib::Type>{}, std::forward(args)... + ); + default: + throw StatusException{ + ErrorCode::NOT_IMPLEMENTED, + "Requested storage kind is not supported for IVF index"}; + } +} + +// Dispatch on storage kind for Dynamic IVF operations (uses blocked allocator) +template +auto dispatch_ivf_blocked_storage_kind(StorageKind kind, F&& f, Args&&... args) { + switch (kind) { + case StorageKind::FP32: + return f( + svs::lib::Type>{}, std::forward(args)... + ); + case StorageKind::FP16: + return f( + svs::lib::Type>{}, + std::forward(args)... + ); + default: + throw StatusException{ + ErrorCode::NOT_IMPLEMENTED, + "Requested storage kind is not supported for Dynamic IVF index"}; + } +} + +} // namespace ivf_storage + +// Static IVF index implementation +class StaticIVFIndexImpl { + public: + StaticIVFIndexImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{dim} + , metric_type_{metric} + , storage_kind_{storage_kind} + , build_params_{params} + , default_search_params_{default_search_params} + , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} + , intra_query_threads_{intra_query_threads} { + if (!ivf_storage::is_supported_storage_kind(storage_kind)) { + throw StatusException{ + ErrorCode::INVALID_ARGUMENT, + "The specified storage kind is not compatible with StaticIVFIndex"}; + } + } + + size_t size() const { return impl_ ? impl_->size() : 0; } + + size_t dimensions() const { return dim_; } + + MetricType metric_type() const { return metric_type_; } + + StorageKind get_storage_kind() const { return storage_kind_; } + + void build(data::ConstSimpleDataView data) { + if (impl_) { + throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; + } + init_impl(data); + } + + void search( + svs::QueryResultView result, + svs::data::ConstSimpleDataView queries, + const IVFIndex::SearchParams* params = nullptr + ) const { + if (!impl_) { + auto& dists = result.distances(); + std::fill(dists.begin(), dists.end(), Unspecify()); + auto& inds = result.indices(); + std::fill(inds.begin(), inds.end(), Unspecify()); + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + + if (queries.size() == 0) { + return; + } + + const size_t k = result.n_neighbors(); + if (k == 0) { + throw StatusException{ErrorCode::INVALID_ARGUMENT, "k must be greater than 0"}; + } + + auto sp = make_search_parameters(params); + impl_->set_search_parameters(sp); + impl_->search(result, queries, {}); + } + + void save(std::ostream& out) const { + if (!impl_) { + throw StatusException{ + ErrorCode::NOT_INITIALIZED, "Cannot serialize: IVF index not initialized."}; + } + impl_->save(out); + } + + static StaticIVFIndexImpl* load( + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) { + if (num_threads == 0) { + num_threads = static_cast(omp_get_max_threads()); + } + + // Dispatch on storage kind to load with correct data type + return ivf_storage::dispatch_ivf_storage_kind( + storage_kind, + [&](svs::lib::Type) { + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::IVF::assemble( + in, + std::forward(distance), + num_threads, + intra_query_threads + ) + ); + return new StaticIVFIndexImpl( + std::move(impl), + metric, + storage_kind, + num_threads, + intra_query_threads + ); + }); + } + ); + } + + protected: + // Constructor used during loading + StaticIVFIndexImpl( + std::unique_ptr&& impl, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{impl->dimensions()} + , metric_type_{metric} + , storage_kind_{storage_kind} + , num_threads_{num_threads} + , intra_query_threads_{intra_query_threads} + , impl_{std::move(impl)} { + // Extract default search params from loaded index + auto loaded_params = impl_->get_search_parameters(); + default_search_params_ = {loaded_params.n_probes_, loaded_params.k_reorder_}; + } + + svs::index::ivf::IVFBuildParameters ivf_build_parameters() const { + svs::index::ivf::IVFBuildParameters result; + set_if_specified(result.num_centroids_, build_params_.num_centroids); + set_if_specified(result.minibatch_size_, build_params_.minibatch_size); + set_if_specified(result.num_iterations_, build_params_.num_iterations); + if (is_specified(build_params_.is_hierarchical)) { + result.is_hierarchical_ = build_params_.is_hierarchical.is_enabled(); + } + set_if_specified(result.training_fraction_, build_params_.training_fraction); + set_if_specified( + result.hierarchical_level1_clusters_, build_params_.hierarchical_level1_clusters + ); + set_if_specified(result.seed_, build_params_.seed); + return result; + } + + svs::index::ivf::IVFSearchParameters + make_search_parameters(const IVFIndex::SearchParams* params) const { + // Start with default parameters + svs::index::ivf::IVFSearchParameters result; + if (is_specified(default_search_params_.n_probes)) { + result.n_probes_ = default_search_params_.n_probes; + } + if (is_specified(default_search_params_.k_reorder)) { + result.k_reorder_ = default_search_params_.k_reorder; + } + + // Override with user-specified parameters + if (params) { + set_if_specified(result.n_probes_, params->n_probes); + set_if_specified(result.k_reorder_, params->k_reorder); + } + + return result; + } + + void init_impl(data::ConstSimpleDataView data) { + auto build_params = ivf_build_parameters(); + + // Single copy of data - required because IVF assembly deduces internal types from + // data type, and ConstSimpleDataView has const element type which breaks + // internal type deduction. This copy is also passed directly to assemble which + // partitions it into clusters (no additional copy for FP32 storage). + auto owned_data = svs::data::SimpleData(data.size(), data.dimensions()); + svs::data::copy(data, owned_data); + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); + + impl_.reset(distance_dispatcher([&](auto&& distance) { + // Build clustering using BFloat16 for efficiency (AMX support) + // Note: build_clustering takes const ref, doesn't consume data + auto clustering = svs::IVF::build_clustering( + build_params, + owned_data, + std::forward(distance), + num_threads_ + ); + + // Dispatch on storage kind to assemble with correct data type + return ivf_storage::dispatch_ivf_storage_kind( + storage_kind_, + [&](svs::lib::Type) { + using TargetElement = typename DataType::element_type; + + // For FP32: pass owned_data directly (moved into clusters) + // For FP16: convert from owned_data + if constexpr (std::is_same_v) { + return new svs::IVF(svs::IVF::assemble_from_clustering( + std::move(clustering), + owned_data, + std::forward(distance), + num_threads_, + intra_query_threads_ + )); + } else { + // Convert to target type (e.g., FP16) + DataType converted_data(owned_data.size(), owned_data.dimensions()); + svs::data::copy(owned_data, converted_data); + return new svs::IVF(svs::IVF::assemble_from_clustering( + std::move(clustering), + std::move(converted_data), + std::forward(distance), + num_threads_, + intra_query_threads_ + )); + } + } + ); + })); + } + + // Data members + size_t dim_; + MetricType metric_type_; + StorageKind storage_kind_; + IVFIndex::BuildParams build_params_; + IVFIndex::SearchParams default_search_params_; + size_t num_threads_; + size_t intra_query_threads_; + std::unique_ptr impl_; +}; + +// Dynamic IVF index implementation +class DynamicIVFIndexImpl { + public: + DynamicIVFIndexImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{dim} + , metric_type_{metric} + , storage_kind_{storage_kind} + , build_params_{params} + , default_search_params_{default_search_params} + , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} + , intra_query_threads_{intra_query_threads} { + if (!ivf_storage::is_supported_storage_kind(storage_kind)) { + throw StatusException{ + ErrorCode::INVALID_ARGUMENT, + "The specified storage kind is not compatible with DynamicIVFIndex"}; + } + } + + size_t size() const { return impl_ ? impl_->size() : 0; } + + size_t dimensions() const { return dim_; } + + MetricType metric_type() const { return metric_type_; } + + StorageKind get_storage_kind() const { return storage_kind_; } + + void build(data::ConstSimpleDataView data, std::span ids) { + if (impl_) { + throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; + } + init_impl(data, ids); + } + + void + add(data::ConstSimpleDataView data, + std::span ids, + bool reuse_empty = false) { + if (!impl_) { + // First add initializes the index + init_impl(data, ids); + return; + } + impl_->add_points(data, ids, reuse_empty); + } + + size_t remove(std::span ids) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + return impl_->delete_points(ids); + } + + size_t remove_selected(const IDFilter& selector) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + + auto ids = impl_->all_ids(); + std::vector ids_to_delete; + std::copy_if( + ids.begin(), + ids.end(), + std::back_inserter(ids_to_delete), + [&](size_t id) { return selector(id); } + ); + + return impl_->delete_points(ids_to_delete); + } + + bool has_id(size_t id) const { + if (!impl_) { + return false; + } + return impl_->has_id(id); + } + + void consolidate() { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + impl_->consolidate(); + } + + void compact(size_t batchsize = 1'000'000) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + impl_->compact(batchsize); + } + + void search( + svs::QueryResultView result, + svs::data::ConstSimpleDataView queries, + const IVFIndex::SearchParams* params = nullptr + ) const { + if (!impl_) { + auto& dists = result.distances(); + std::fill(dists.begin(), dists.end(), Unspecify()); + auto& inds = result.indices(); + std::fill(inds.begin(), inds.end(), Unspecify()); + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + + if (queries.size() == 0) { + return; + } + + const size_t k = result.n_neighbors(); + if (k == 0) { + throw StatusException{ErrorCode::INVALID_ARGUMENT, "k must be greater than 0"}; + } + + auto sp = make_search_parameters(params); + impl_->set_search_parameters(sp); + impl_->search(result, queries, {}); + } + + void save(std::ostream& out) const { + if (!impl_) { + throw StatusException{ + ErrorCode::NOT_INITIALIZED, + "Cannot serialize: DynamicIVF index not initialized."}; + } + impl_->save(out); + } + + static DynamicIVFIndexImpl* load( + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) { + if (num_threads == 0) { + num_threads = static_cast(omp_get_max_threads()); + } + + // Dispatch on storage kind to load with correct data type + return ivf_storage::dispatch_ivf_storage_kind( + storage_kind, + [&](svs::lib::Type) { + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::DynamicIVF::assemble( + in, + std::forward(distance), + num_threads, + intra_query_threads + ) + ); + return new DynamicIVFIndexImpl( + std::move(impl), + metric, + storage_kind, + num_threads, + intra_query_threads + ); + }); + } + ); + } + + protected: + // Constructor used during loading + DynamicIVFIndexImpl( + std::unique_ptr&& impl, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{impl->dimensions()} + , metric_type_{metric} + , storage_kind_{storage_kind} + , num_threads_{num_threads} + , intra_query_threads_{intra_query_threads} + , impl_{std::move(impl)} { + // Extract default search params from loaded index + auto loaded_params = impl_->get_search_parameters(); + default_search_params_ = {loaded_params.n_probes_, loaded_params.k_reorder_}; + } + + svs::index::ivf::IVFBuildParameters ivf_build_parameters() const { + svs::index::ivf::IVFBuildParameters result; + set_if_specified(result.num_centroids_, build_params_.num_centroids); + set_if_specified(result.minibatch_size_, build_params_.minibatch_size); + set_if_specified(result.num_iterations_, build_params_.num_iterations); + if (is_specified(build_params_.is_hierarchical)) { + result.is_hierarchical_ = build_params_.is_hierarchical.is_enabled(); + } + set_if_specified(result.training_fraction_, build_params_.training_fraction); + set_if_specified( + result.hierarchical_level1_clusters_, build_params_.hierarchical_level1_clusters + ); + set_if_specified(result.seed_, build_params_.seed); + return result; + } + + svs::index::ivf::IVFSearchParameters + make_search_parameters(const IVFIndex::SearchParams* params) const { + // Start with default parameters + svs::index::ivf::IVFSearchParameters result; + if (is_specified(default_search_params_.n_probes)) { + result.n_probes_ = default_search_params_.n_probes; + } + if (is_specified(default_search_params_.k_reorder)) { + result.k_reorder_ = default_search_params_.k_reorder; + } + + // Override with user-specified parameters + if (params) { + set_if_specified(result.n_probes_, params->n_probes); + set_if_specified(result.k_reorder_, params->k_reorder); + } + + return result; + } + + void init_impl(data::ConstSimpleDataView data, std::span ids) { + auto build_params = ivf_build_parameters(); + + // Single copy of data - required because IVF assembly deduces internal types from + // data type, and ConstSimpleDataView has const element type which breaks + // internal type deduction. This copy is also passed directly to assemble which + // partitions it into clusters (no additional copy for FP32 storage). + auto owned_data = svs::data::SimpleData(data.size(), data.dimensions()); + svs::data::copy(data, owned_data); + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); + + impl_.reset(distance_dispatcher([&](auto&& distance) { + // Build clustering using BFloat16 for efficiency (AMX support) + // Note: build_clustering takes const ref, doesn't consume data + auto clustering = svs::IVF::build_clustering( + build_params, + owned_data, + std::forward(distance), + num_threads_ + ); + + // Dispatch on storage kind to assemble with correct data type + return ivf_storage::dispatch_ivf_storage_kind( + storage_kind_, + [&](svs::lib::Type) { + using TargetElement = typename DataType::element_type; + + // For FP32: pass owned_data directly (moved into clusters) + // For FP16: convert from owned_data + if constexpr (std::is_same_v) { + return new svs::DynamicIVF( + svs::DynamicIVF::assemble_from_clustering( + std::move(clustering), + owned_data, + ids, + std::forward(distance), + num_threads_, + intra_query_threads_ + ) + ); + } else { + // Convert to target type (e.g., FP16) + DataType converted_data(owned_data.size(), owned_data.dimensions()); + svs::data::copy(owned_data, converted_data); + return new svs::DynamicIVF( + svs::DynamicIVF::assemble_from_clustering( + std::move(clustering), + std::move(converted_data), + ids, + std::forward(distance), + num_threads_, + intra_query_threads_ + ) + ); + } + } + ); + })); + } + + // Data members + size_t dim_; + MetricType metric_type_; + StorageKind storage_kind_; + IVFIndex::BuildParams build_params_; + IVFIndex::SearchParams default_search_params_; + size_t num_threads_; + size_t intra_query_threads_; + std::unique_ptr impl_; +}; + +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/tests/CMakeLists.txt b/bindings/cpp/tests/CMakeLists.txt index 77e7116eb..c87d31487 100644 --- a/bindings/cpp/tests/CMakeLists.txt +++ b/bindings/cpp/tests/CMakeLists.txt @@ -66,6 +66,11 @@ target_include_directories(svs_runtime_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../include ) +# Pass IVF enable flag to tests +if (SVS_RUNTIME_ENABLE_IVF) + target_compile_definitions(svs_runtime_test PRIVATE SVS_RUNTIME_ENABLE_IVF) +endif() + # Enable testing with CTest include(CTest) enable_testing() diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index ec6f309d9..fcded1006 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -17,6 +17,9 @@ #include "svs/runtime/api_defs.h" #include "svs/runtime/dynamic_vamana_index.h" #include "svs/runtime/flat_index.h" +#ifdef SVS_RUNTIME_ENABLE_IVF +#include "svs/runtime/ivf_index.h" +#endif #include "svs/runtime/training.h" #include "svs/runtime/vamana_index.h" @@ -675,3 +678,465 @@ CATCH_TEST_CASE("SetIfSpecifiedUtility", "[runtime]") { CATCH_REQUIRE(target == false); } } + +// +// IVF Index Tests +// + +#ifdef SVS_RUNTIME_ENABLE_IVF +CATCH_TEST_CASE("StaticIVFIndexBuildAndSearch", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::StaticIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; // Small number for test data + build_params.num_iterations = 5; + + svs::runtime::v0::IVFIndex::SearchParams search_params; + search_params.n_probes = 3; + search_params.k_reorder = 1.0f; + + svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params, + search_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Verify results are reasonable (at least some results found) + bool found_valid = false; + for (int i = 0; i < nq * k; ++i) { + if (result_labels[i] < test_n) { + found_valid = true; + break; + } + } + CATCH_REQUIRE(found_valid); + + svs::runtime::v0::StaticIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("StaticIVFIndexWriteAndRead", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::StaticIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "static_ivf_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::StaticIVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::StaticIVFIndex::load( + &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Test search on loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Clean up + svs::runtime::v0::StaticIVFIndex::destroy(index); + svs::runtime::v0::StaticIVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearch", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build dynamic IVF index with initial data + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::IVFIndex::SearchParams search_params; + search_params.n_probes = 3; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params, + search_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("DynamicIVFIndexAddAndRemove", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build empty dynamic IVF index first, then add data + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + // Create with initial data (needed for clustering) + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Check has_id for existing IDs + bool exists = false; + status = index->has_id(&exists, 0); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + status = index->has_id(&exists, test_n - 1); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + // Check has_id for non-existing ID + status = index->has_id(&exists, test_n + 100); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + + // Remove some IDs + std::vector ids_to_remove = {0, 1, 2}; + status = index->remove(ids_to_remove.size(), ids_to_remove.data()); + CATCH_REQUIRE(status.ok()); + + // Verify removed IDs no longer exist + status = index->has_id(&exists, 0); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + + // Consolidate and compact + status = index->consolidate(); + CATCH_REQUIRE(status.ok()); + + status = index->compact(); + CATCH_REQUIRE(status.ok()); + + // Search should still work + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("DynamicIVFIndexWriteAndRead", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build dynamic IVF index + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "dynamic_ivf_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::DynamicIVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::DynamicIVFIndex::load( + &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Test search on loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Clean up + svs::runtime::v0::DynamicIVFIndex::destroy(index); + svs::runtime::v0::DynamicIVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexRemoveSelected", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build dynamic IVF index + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Remove IDs in range [0, 20) using selector + size_t min_id = 0; + size_t max_id = 20; + test_utils::IDFilterRange selector(min_id, max_id); + + size_t num_removed = 0; + status = index->remove_selected(&num_removed, selector); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(num_removed == max_id - min_id); + + // Verify removed IDs no longer exist + bool exists = false; + for (size_t i = min_id; i < max_id; ++i) { + status = index->has_id(&exists, i); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + } + + // Verify IDs outside range still exist + status = index->has_id(&exists, max_id); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::StaticIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::IVFIndex::SearchParams default_search_params; + default_search_params.n_probes = 2; + + svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params, + default_search_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances1(nq * k); + std::vector result_labels1(nq * k); + + // Search with default params + status = index->search(nq, xq, k, distances1.data(), result_labels1.data()); + CATCH_REQUIRE(status.ok()); + + // Search with custom params (more probes should potentially give better results) + svs::runtime::v0::IVFIndex::SearchParams custom_params; + custom_params.n_probes = 5; + custom_params.k_reorder = 2.0f; + + std::vector distances2(nq * k); + std::vector result_labels2(nq * k); + + status = + index->search(nq, xq, k, distances2.data(), result_labels2.data(), &custom_params); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::StaticIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexCheckStorageKind", "[runtime][ivf]") { + // FP32 should be supported + CATCH_REQUIRE(svs::runtime::v0::StaticIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP32 + ) + .ok()); + CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP32 + ) + .ok()); + + // FP16 should be supported + CATCH_REQUIRE(svs::runtime::v0::StaticIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP16 + ) + .ok()); + CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP16 + ) + .ok()); + + // LVQ storage kinds should not be supported for IVF + CATCH_REQUIRE(!svs::runtime::v0::StaticIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LVQ4x4 + ) + .ok()); + CATCH_REQUIRE(!svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LVQ4x4 + ) + .ok()); +} + +CATCH_TEST_CASE("StaticIVFIndexInnerProduct", "[runtime][ivf]") { + const auto& test_data = get_test_data(); + + // Build static IVF index with inner product metric + svs::runtime::v0::StaticIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::INNER_PRODUCT, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::StaticIVFIndex::destroy(index); +} +#endif // SVS_RUNTIME_ENABLE_IVF From 8179316fb591632345e26c93119494cb1db301a8 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Tue, 10 Feb 2026 16:03:03 -0800 Subject: [PATCH 02/12] enable lvq/leanvec --- .../workflows/build-cpp-runtime-bindings.yml | 7 + bindings/cpp/CMakeLists.txt | 25 +- bindings/cpp/conda-recipe/build.sh | 1 + .../include/svs/runtime/dynamic_ivf_index.h | 166 ++++ bindings/cpp/include/svs/runtime/ivf_index.h | 156 +--- bindings/cpp/src/dynamic_ivf_index.cpp | 309 +++++++ bindings/cpp/src/dynamic_ivf_index_impl.h | 462 ++++++++++ bindings/cpp/src/dynamic_vamana_index.cpp | 8 +- .../src/dynamic_vamana_index_leanvec_impl.h | 2 +- bindings/cpp/src/ivf_index.cpp | 254 +++--- bindings/cpp/src/ivf_index_impl.h | 803 ++++++++++------- bindings/cpp/src/svs_runtime_utils.h | 19 +- bindings/cpp/src/training.cpp | 6 +- bindings/cpp/src/training_impl.h | 6 +- bindings/cpp/tests/CMakeLists.txt | 4 + bindings/cpp/tests/ivf_runtime_test.cpp | 843 ++++++++++++++++++ bindings/cpp/tests/runtime_test.cpp | 466 ---------- docker/x86_64/build-cpp-runtime-bindings.sh | 3 +- 18 files changed, 2455 insertions(+), 1085 deletions(-) create mode 100644 bindings/cpp/include/svs/runtime/dynamic_ivf_index.h create mode 100644 bindings/cpp/src/dynamic_ivf_index.cpp create mode 100644 bindings/cpp/src/dynamic_ivf_index_impl.h create mode 100644 bindings/cpp/tests/ivf_runtime_test.cpp diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index cd2ab3bc5..ae74101d9 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -37,10 +37,16 @@ jobs: include: - name: "with static library" enable_lvq_leanvec: "ON" + enable_ivf: "OFF" suffix: "" - name: "public only" enable_lvq_leanvec: "OFF" + enable_ivf: "OFF" suffix: "-public-only" + - name: "IVF public only" + enable_lvq_leanvec: "OFF" + enable_ivf: "ON" + suffix: "-ivf" fail-fast: false steps: @@ -56,6 +62,7 @@ jobs: -v ${{ github.workspace }}:/workspace \ -w /workspace \ -e ENABLE_LVQ_LEANVEC=${{ matrix.enable_lvq_leanvec }} \ + -e ENABLE_IVF=${{ matrix.enable_ivf }} \ -e SUFFIX=${{ matrix.suffix }} \ svs-manylinux228:latest \ /bin/bash -c "chmod +x docker/x86_64/build-cpp-runtime-bindings.sh && ./docker/x86_64/build-cpp-runtime-bindings.sh" diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 78a6323da..50d86e462 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -44,8 +44,16 @@ set(SVS_RUNTIME_SOURCES # Add IVF files if enabled if (SVS_RUNTIME_ENABLE_IVF) message(STATUS "SVS runtime will be built with IVF support (requires MKL)") - list(APPEND SVS_RUNTIME_HEADERS include/svs/runtime/ivf_index.h) - list(APPEND SVS_RUNTIME_SOURCES src/ivf_index_impl.h src/ivf_index.cpp) + list(APPEND SVS_RUNTIME_HEADERS + include/svs/runtime/ivf_index.h + include/svs/runtime/dynamic_ivf_index.h + ) + list(APPEND SVS_RUNTIME_SOURCES + src/ivf_index_impl.h + src/dynamic_ivf_index_impl.h + src/ivf_index.cpp + src/dynamic_ivf_index.cpp + ) else() message(STATUS "SVS runtime will be built without IVF support") endif() @@ -87,14 +95,6 @@ set_target_properties(${TARGET_NAME} PROPERTIES CXX_EXTENSIONS OFF) set_target_properties(${TARGET_NAME} PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION ${PROJECT_VERSION_MAJOR} ) if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC) - if(DEFINED SVS_LVQ_HEADER AND DEFINED SVS_LEANVEC_HEADER) - # expected that pre-defined headers are implementation headers - message(STATUS "Using pre-defined LVQ header: ${SVS_LVQ_HEADER}") - message(STATUS "Using pre-defined LeanVec header: ${SVS_LEANVEC_HEADER}") - else() - set(SVS_LVQ_HEADER "svs/extensions/vamana/lvq.h") - set(SVS_LEANVEC_HEADER "svs/extensions/vamana/leanvec.h") - endif() if(RUNTIME_BINDINGS_PRIVATE_SOURCE_BUILD) message(STATUS "Building directly from private sources") @@ -135,10 +135,7 @@ if (SVS_RUNTIME_ENABLE_LVQ_LEANVEC) svs::svs_static_library ) endif() - target_compile_definitions(${TARGET_NAME} PRIVATE - PUBLIC "SVS_LVQ_HEADER=\"${SVS_LVQ_HEADER}\"" - PUBLIC "SVS_LEANVEC_HEADER=\"${SVS_LEANVEC_HEADER}\"" - ) + target_compile_definitions(${TARGET_NAME} PUBLIC SVS_RUNTIME_HAVE_LVQ_LEANVEC) else() # Include the SVS library directly if needed. if (NOT TARGET svs::svs) diff --git a/bindings/cpp/conda-recipe/build.sh b/bindings/cpp/conda-recipe/build.sh index 8b4347019..f3ce6e295 100644 --- a/bindings/cpp/conda-recipe/build.sh +++ b/bindings/cpp/conda-recipe/build.sh @@ -30,6 +30,7 @@ CMAKE_ARGS=( "-DCMAKE_INSTALL_PREFIX=${PREFIX}" "-DSVS_BUILD_RUNTIME_TESTS=OFF" "-DSVS_RUNTIME_ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON}" + "-DSVS_RUNTIME_ENABLE_IVF=${ENABLE_IVF:-OFF}" ) # Add SVS_URL if specified (for fetching static library) diff --git a/bindings/cpp/include/svs/runtime/dynamic_ivf_index.h b/bindings/cpp/include/svs/runtime/dynamic_ivf_index.h new file mode 100644 index 000000000..39eda2af8 --- /dev/null +++ b/bindings/cpp/include/svs/runtime/dynamic_ivf_index.h @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +#include +#include +#include + +namespace svs { +namespace runtime { +namespace v0 { + +/// @brief Abstract interface for dynamic IVF indices (supports add/delete). +struct SVS_RUNTIME_API DynamicIVFIndex : public IVFIndex { + /// @brief Utility function to check storage kind support. + static Status check_storage_kind(StorageKind storage_kind) noexcept; + + /// @brief Build a dynamic IVF index. + /// + /// @param index Output pointer to the created index. + /// @param dim Dimensionality of vectors. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param n Number of initial vectors (can be 0 for empty index). + /// @param data Pointer to initial vector data (can be nullptr if n=0). + /// @param labels Pointer to labels for initial vectors (can be nullptr if n=0). + /// @param params Build parameters for clustering. + /// @param default_search_params Default search parameters. + /// @param num_threads Number of threads for operations. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; + + /// @brief Destroy a dynamic IVF index. + static Status destroy(DynamicIVFIndex* index) noexcept; + + /// @brief Add vectors to the index. + /// + /// @param n Number of vectors to add. + /// @param labels Pointer to labels for the new vectors. + /// @param x Pointer to vector data (row-major, n x dimensions). + /// @param reuse_empty Whether to reuse empty slots from deleted vectors. + /// @return Status indicating success or error. + virtual Status + add(size_t n, const size_t* labels, const float* x, bool reuse_empty = false + ) noexcept = 0; + + /// @brief Remove vectors from the index by ID. + /// + /// @param n Number of vectors to remove. + /// @param labels Pointer to labels of vectors to remove. + /// @return Status indicating success or error. + virtual Status remove(size_t n, const size_t* labels) noexcept = 0; + + /// @brief Remove vectors matching a selector. + /// + /// @param num_removed Output: number of vectors actually removed. + /// @param selector Filter to determine which vectors to remove. + /// @return Status indicating success or error. + virtual Status + remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; + + /// @brief Check if an ID exists in the index. + /// + /// @param exists Output: true if the ID exists. + /// @param id The ID to check. + /// @return Status indicating success or error. + virtual Status has_id(bool* exists, size_t id) const noexcept = 0; + + /// @brief Consolidate the index (clean up deleted entries). + virtual Status consolidate() noexcept = 0; + + /// @brief Compact the index (reclaim memory from deleted entries). + /// + /// @param batchsize Number of entries to process per batch. + /// @return Status indicating success or error. + virtual Status compact(size_t batchsize = 1'000'000) noexcept = 0; + + /// @brief Save the index to a stream. + virtual Status save(std::ostream& out) const noexcept = 0; + + /// @brief Load a dynamic IVF index from a stream. + /// + /// @param index Output pointer to the loaded index. + /// @param in Input stream containing the serialized index. + /// @param metric Distance metric to use. + /// @param storage_kind Storage type for the dataset. + /// @param num_threads Number of threads for operations. + /// @param intra_query_threads Number of threads for intra-query parallelism. + /// @return Status indicating success or error. + static Status load( + DynamicIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; +}; + +/// @brief Specialization for building LeanVec-based dynamic IVF indices. +struct SVS_RUNTIME_API DynamicIVFIndexLeanVec : public DynamicIVFIndex { + /// @brief Build a LeanVec dynamic IVF index with specified leanvec dimensions. + static Status build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + size_t leanvec_dims, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; + + /// @brief Build a LeanVec dynamic IVF index with provided training data. + static Status build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const LeanVecTrainingData* training_data, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; +}; + +} // namespace v0 +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/include/svs/runtime/ivf_index.h b/bindings/cpp/include/svs/runtime/ivf_index.h index b780571a3..796a59c55 100644 --- a/bindings/cpp/include/svs/runtime/ivf_index.h +++ b/bindings/cpp/include/svs/runtime/ivf_index.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -56,14 +57,6 @@ struct SVS_RUNTIME_API IVFIndex { }; /// @brief Perform k-NN search on the index. - /// - /// @param n Number of query vectors. - /// @param x Pointer to query vectors (row-major, n x dimensions). - /// @param k Number of nearest neighbors to find. - /// @param distances Output array for distances (n x k). - /// @param labels Output array for neighbor IDs (n x k). - /// @param params Optional search parameters (uses defaults if nullptr). - /// @return Status indicating success or error. virtual Status search( size_t n, const float* x, @@ -72,56 +65,46 @@ struct SVS_RUNTIME_API IVFIndex { size_t* labels, const SearchParams* params = nullptr ) const noexcept = 0; -}; -/// @brief Abstract interface for static IVF indices (read-only after construction). -struct SVS_RUNTIME_API StaticIVFIndex : public IVFIndex { /// @brief Utility function to check storage kind support. static Status check_storage_kind(StorageKind storage_kind) noexcept; - /// @brief Build a static IVF index from data. - /// - /// @param index Output pointer to the created index. - /// @param dim Dimensionality of vectors. - /// @param metric Distance metric to use. - /// @param storage_kind Storage type for the dataset. - /// @param n Number of vectors in the dataset. - /// @param data Pointer to vector data (row-major, n x dim). - /// @param params Build parameters for clustering. - /// @param default_search_params Default search parameters. - /// @param num_threads Number of threads for building and searching. - /// @param intra_query_threads Number of threads for intra-query parallelism. - /// @return Status indicating success or error. + /// @brief Build an IVF index from data. static Status build( - StaticIVFIndex** index, + IVFIndex** index, size_t dim, MetricType metric, StorageKind storage_kind, size_t n, const float* data, - const IVFIndex::BuildParams& params = {}, - const IVFIndex::SearchParams& default_search_params = {}, + const BuildParams& params, + const SearchParams& default_search_params, + size_t num_threads = 0, + size_t intra_query_threads = 1 + ) noexcept; + + /// @brief Build an IVF index from data (uses default search parameters). + static Status build( + IVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const BuildParams& params, size_t num_threads = 0, size_t intra_query_threads = 1 ) noexcept; - /// @brief Destroy a static IVF index. - static Status destroy(StaticIVFIndex* index) noexcept; + /// @brief Destroy an IVF index. + static Status destroy(IVFIndex* index) noexcept; /// @brief Save the index to a stream. virtual Status save(std::ostream& out) const noexcept = 0; - /// @brief Load a static IVF index from a stream. - /// - /// @param index Output pointer to the loaded index. - /// @param in Input stream containing the serialized index. - /// @param metric Distance metric to use. - /// @param storage_kind Storage type for the dataset. - /// @param num_threads Number of threads for searching. - /// @param intra_query_threads Number of threads for intra-query parallelism. - /// @return Status indicating success or error. + /// @brief Load an IVF index from a stream. static Status load( - StaticIVFIndex** index, + IVFIndex** index, std::istream& in, MetricType metric, StorageKind storage_kind, @@ -130,101 +113,34 @@ struct SVS_RUNTIME_API StaticIVFIndex : public IVFIndex { ) noexcept; }; -/// @brief Abstract interface for dynamic IVF indices (supports add/delete). -struct SVS_RUNTIME_API DynamicIVFIndex : public IVFIndex { - /// @brief Utility function to check storage kind support. - static Status check_storage_kind(StorageKind storage_kind) noexcept; - - /// @brief Build a dynamic IVF index. - /// - /// @param index Output pointer to the created index. - /// @param dim Dimensionality of vectors. - /// @param metric Distance metric to use. - /// @param storage_kind Storage type for the dataset. - /// @param n Number of initial vectors (can be 0 for empty index). - /// @param data Pointer to initial vector data (can be nullptr if n=0). - /// @param labels Pointer to labels for initial vectors (can be nullptr if n=0). - /// @param params Build parameters for clustering. - /// @param default_search_params Default search parameters. - /// @param num_threads Number of threads for operations. - /// @param intra_query_threads Number of threads for intra-query parallelism. - /// @return Status indicating success or error. +/// @brief Specialization for building LeanVec-based IVF indices. +struct SVS_RUNTIME_API IVFIndexLeanVec : public IVFIndex { + /// @brief Build a LeanVec IVF index with specified leanvec dimensions. static Status build( - DynamicIVFIndex** index, + IVFIndex** index, size_t dim, MetricType metric, StorageKind storage_kind, size_t n, const float* data, - const size_t* labels, + size_t leanvec_dims, const IVFIndex::BuildParams& params = {}, const IVFIndex::SearchParams& default_search_params = {}, size_t num_threads = 0, size_t intra_query_threads = 1 ) noexcept; - /// @brief Destroy a dynamic IVF index. - static Status destroy(DynamicIVFIndex* index) noexcept; - - /// @brief Add vectors to the index. - /// - /// @param n Number of vectors to add. - /// @param labels Pointer to labels for the new vectors. - /// @param x Pointer to vector data (row-major, n x dimensions). - /// @param reuse_empty Whether to reuse empty slots from deleted vectors. - /// @return Status indicating success or error. - virtual Status - add(size_t n, const size_t* labels, const float* x, bool reuse_empty = false - ) noexcept = 0; - - /// @brief Remove vectors from the index by ID. - /// - /// @param n Number of vectors to remove. - /// @param labels Pointer to labels of vectors to remove. - /// @return Status indicating success or error. - virtual Status remove(size_t n, const size_t* labels) noexcept = 0; - - /// @brief Remove vectors matching a selector. - /// - /// @param num_removed Output: number of vectors actually removed. - /// @param selector Filter to determine which vectors to remove. - /// @return Status indicating success or error. - virtual Status - remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; - - /// @brief Check if an ID exists in the index. - /// - /// @param exists Output: true if the ID exists. - /// @param id The ID to check. - /// @return Status indicating success or error. - virtual Status has_id(bool* exists, size_t id) const noexcept = 0; - - /// @brief Consolidate the index (clean up deleted entries). - virtual Status consolidate() noexcept = 0; - - /// @brief Compact the index (reclaim memory from deleted entries). - /// - /// @param batchsize Number of entries to process per batch. - /// @return Status indicating success or error. - virtual Status compact(size_t batchsize = 1'000'000) noexcept = 0; - - /// @brief Save the index to a stream. - virtual Status save(std::ostream& out) const noexcept = 0; - - /// @brief Load a dynamic IVF index from a stream. - /// - /// @param index Output pointer to the loaded index. - /// @param in Input stream containing the serialized index. - /// @param metric Distance metric to use. - /// @param storage_kind Storage type for the dataset. - /// @param num_threads Number of threads for operations. - /// @param intra_query_threads Number of threads for intra-query parallelism. - /// @return Status indicating success or error. - static Status load( - DynamicIVFIndex** index, - std::istream& in, + /// @brief Build a LeanVec IVF index with provided training data. + static Status build( + IVFIndex** index, + size_t dim, MetricType metric, StorageKind storage_kind, + size_t n, + const float* data, + const LeanVecTrainingData* training_data, + const IVFIndex::BuildParams& params = {}, + const IVFIndex::SearchParams& default_search_params = {}, size_t num_threads = 0, size_t intra_query_threads = 1 ) noexcept; diff --git a/bindings/cpp/src/dynamic_ivf_index.cpp b/bindings/cpp/src/dynamic_ivf_index.cpp new file mode 100644 index 000000000..91e89e0a6 --- /dev/null +++ b/bindings/cpp/src/dynamic_ivf_index.cpp @@ -0,0 +1,309 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "svs/runtime/dynamic_ivf_index.h" + +#include "dynamic_ivf_index_impl.h" +#include "svs_runtime_utils.h" + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +#include "training_impl.h" +#endif + +#include +#include +#include + +#include +#include +#include +#include + +namespace svs { +namespace runtime { + +namespace { + +// Manager class for Dynamic IVF Index +struct DynamicIVFIndexManager : public DynamicIVFIndex { + std::unique_ptr impl_; + + DynamicIVFIndexManager(std::unique_ptr impl) + : impl_{std::move(impl)} { + assert(impl_ != nullptr); + } + + ~DynamicIVFIndexManager() override = default; + + Status search( + size_t n, + const float* x, + size_t k, + float* distances, + size_t* labels, + const SearchParams* params = nullptr + ) const noexcept override { + return runtime_error_wrapper([&] { + auto result = svs::QueryResultView{ + svs::MatrixView{svs::make_dims(n, k), labels}, + svs::MatrixView{svs::make_dims(n, k), distances}}; + auto queries = svs::data::ConstSimpleDataView(x, n, impl_->dimensions()); + impl_->search(result, queries, params); + }); + } + + Status + add(size_t n, const size_t* labels, const float* x, bool reuse_empty + ) noexcept override { + return runtime_error_wrapper([&] { + svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; + std::span lbls(labels, n); + impl_->add(data, lbls, reuse_empty); + }); + } + + Status remove(size_t n, const size_t* labels) noexcept override { + return runtime_error_wrapper([&] { + std::span lbls(labels, n); + impl_->remove(lbls); + }); + } + + Status + remove_selected(size_t* num_removed, const IDFilter& selector) noexcept override { + return runtime_error_wrapper([&] { + *num_removed = impl_->remove_selected(selector); + }); + } + + Status has_id(bool* exists, size_t id) const noexcept override { + return runtime_error_wrapper([&] { *exists = impl_->has_id(id); }); + } + + Status consolidate() noexcept override { + return runtime_error_wrapper([&] { impl_->consolidate(); }); + } + + Status compact(size_t batchsize) noexcept override { + return runtime_error_wrapper([&] { impl_->compact(batchsize); }); + } + + Status save(std::ostream& out) const noexcept override { + return runtime_error_wrapper([&] { impl_->save(out); }); + } +}; + +} // namespace + +// DynamicIVFIndex interface implementation +Status DynamicIVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { + if (ivf_storage::is_supported_storage_kind(storage_kind)) { + return Status_Ok; + } else { + return Status{ + ErrorCode::INVALID_ARGUMENT, + "DynamicIVFIndex only supports FP32, FP16, SQI8, and LVQ storage kinds"}; + } +} + +Status DynamicIVFIndex::build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + auto impl = std::make_unique( + dim, + metric, + storage_kind, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + // Build with provided data if any + if (n > 0 && data != nullptr && labels != nullptr) { + svs::data::ConstSimpleDataView data_view{data, n, dim}; + std::span labels_span{labels, n}; + impl->build(data_view, labels_span); + } + + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} + +Status DynamicIVFIndex::destroy(DynamicIVFIndex* index) noexcept { + return runtime_error_wrapper([&] { delete index; }); +} + +Status DynamicIVFIndex::load( + DynamicIVFIndex** index, + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + std::unique_ptr impl; +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC + if (ivf_storage::is_leanvec_storage_kind(storage_kind)) { + impl.reset(DynamicIVFIndexLeanVecImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )); + } else +#endif + { + impl.reset(DynamicIVFIndexImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )); + } + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} + +// DynamicIVFIndexLeanVec implementations +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +Status DynamicIVFIndexLeanVec::build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + size_t leanvec_dims, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + auto impl = std::make_unique( + dim, + metric, + storage_kind, + leanvec_dims, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + if (n > 0 && data != nullptr && labels != nullptr) { + svs::data::ConstSimpleDataView data_view{data, n, dim}; + std::span labels_span{labels, n}; + impl->build(data_view, labels_span); + } + + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} + +Status DynamicIVFIndexLeanVec::build( + DynamicIVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const size_t* labels, + const LeanVecTrainingData* training_data, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + *index = nullptr; + return runtime_error_wrapper([&] { + auto training_data_impl = + static_cast(training_data)->impl_; + auto impl = std::make_unique( + dim, + metric, + storage_kind, + training_data_impl, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + if (n > 0 && data != nullptr && labels != nullptr) { + svs::data::ConstSimpleDataView data_view{data, n, dim}; + std::span labels_span{labels, n}; + impl->build(data_view, labels_span); + } + + *index = new DynamicIVFIndexManager{std::move(impl)}; + }); +} +#else // SVS_RUNTIME_HAVE_LVQ_LEANVEC +Status DynamicIVFIndexLeanVec::build( + DynamicIVFIndex**, + size_t, + MetricType, + StorageKind, + size_t, + const float*, + const size_t*, + size_t, + const IVFIndex::BuildParams&, + const IVFIndex::SearchParams&, + size_t, + size_t +) noexcept { + return Status( + ErrorCode::NOT_IMPLEMENTED, + "DynamicIVFIndexLeanVec is not supported in this build configuration." + ); +} + +Status DynamicIVFIndexLeanVec::build( + DynamicIVFIndex**, + size_t, + MetricType, + StorageKind, + size_t, + const float*, + const size_t*, + const LeanVecTrainingData*, + const IVFIndex::BuildParams&, + const IVFIndex::SearchParams&, + size_t, + size_t +) noexcept { + return Status( + ErrorCode::NOT_IMPLEMENTED, + "DynamicIVFIndexLeanVec is not supported in this build configuration." + ); +} +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/src/dynamic_ivf_index_impl.h b/bindings/cpp/src/dynamic_ivf_index_impl.h new file mode 100644 index 000000000..8bbd15c2a --- /dev/null +++ b/bindings/cpp/src/dynamic_ivf_index_impl.h @@ -0,0 +1,462 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "ivf_index_impl.h" + +namespace svs { +namespace runtime { + +// Dynamic IVF index implementation (non-LeanVec storage kinds) +class DynamicIVFIndexImpl { + public: + DynamicIVFIndexImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{dim} + , metric_type_{metric} + , storage_kind_{storage_kind} + , build_params_{params} + , default_search_params_{default_search_params} + , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} + , intra_query_threads_{intra_query_threads} { + if (!ivf_storage::is_supported_non_leanvec_storage_kind(storage_kind)) { + throw StatusException{ + ErrorCode::INVALID_ARGUMENT, + "The specified storage kind is not compatible with DynamicIVFIndex. " + "Use DynamicIVFIndexLeanVecImpl for LeanVec storage kinds."}; + } + } + + size_t size() const { return impl_ ? impl_->size() : 0; } + + size_t dimensions() const { return dim_; } + + MetricType metric_type() const { return metric_type_; } + + StorageKind get_storage_kind() const { return storage_kind_; } + + void build(data::ConstSimpleDataView data, std::span ids) { + if (impl_) { + throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; + } + init_impl(data, ids); + } + + void + add(data::ConstSimpleDataView data, + std::span ids, + bool reuse_empty = false) { + if (!impl_) { + // First add initializes the index + init_impl(data, ids); + return; + } + impl_->add_points(data, ids, reuse_empty); + } + + size_t remove(std::span ids) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + return impl_->delete_points(ids); + } + + size_t remove_selected(const IDFilter& selector) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + + auto ids = impl_->all_ids(); + std::vector ids_to_delete; + std::copy_if( + ids.begin(), + ids.end(), + std::back_inserter(ids_to_delete), + [&](size_t id) { return selector(id); } + ); + + return impl_->delete_points(ids_to_delete); + } + + bool has_id(size_t id) const { + if (!impl_) { + return false; + } + return impl_->has_id(id); + } + + void consolidate() { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + impl_->consolidate(); + } + + void compact(size_t batchsize = 1'000'000) { + if (!impl_) { + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + impl_->compact(batchsize); + } + + void search( + svs::QueryResultView result, + svs::data::ConstSimpleDataView queries, + const IVFIndex::SearchParams* params = nullptr + ) const { + if (!impl_) { + auto& dists = result.distances(); + std::fill(dists.begin(), dists.end(), Unspecify()); + auto& inds = result.indices(); + std::fill(inds.begin(), inds.end(), Unspecify()); + throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; + } + + if (queries.size() == 0) { + return; + } + + const size_t k = result.n_neighbors(); + if (k == 0) { + throw StatusException{ErrorCode::INVALID_ARGUMENT, "k must be greater than 0"}; + } + + auto sp = make_search_parameters(params); + impl_->set_search_parameters(sp); + impl_->search(result, queries, {}); + } + + void save(std::ostream& out) const { + if (!impl_) { + throw StatusException{ + ErrorCode::NOT_INITIALIZED, + "Cannot serialize: DynamicIVF index not initialized."}; + } + impl_->save(out); + } + + static DynamicIVFIndexImpl* load( + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) { + if (num_threads == 0) { + num_threads = static_cast(omp_get_max_threads()); + } + + // Dispatch on storage kind to load with correct data type + return ivf_storage::dispatch_ivf_storage_kind(storage_kind, [&](auto tag) { + using Tag = decltype(tag); + using DataType = ivf_storage::IVFBlockedStorageType_t; + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::DynamicIVF::assemble( + in, + std::forward(distance), + num_threads, + intra_query_threads + ) + ); + return new DynamicIVFIndexImpl( + std::move(impl), metric, storage_kind, num_threads, intra_query_threads + ); + }); + }); + } + + protected: + // Constructor used during loading + DynamicIVFIndexImpl( + std::unique_ptr&& impl, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) + : dim_{impl->dimensions()} + , metric_type_{metric} + , storage_kind_{storage_kind} + , num_threads_{num_threads} + , intra_query_threads_{intra_query_threads} + , impl_{std::move(impl)} { + // Extract default search params from loaded index + auto loaded_params = impl_->get_search_parameters(); + default_search_params_.n_probes = loaded_params.n_probes_; + default_search_params_.k_reorder = loaded_params.k_reorder_; + } + + // Constructor used by subclasses (LeanVec) that handle their own validation + DynamicIVFIndexImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads, + bool /*skip_validation*/ + ) + : dim_{dim} + , metric_type_{metric} + , storage_kind_{storage_kind} + , build_params_{params} + , default_search_params_{default_search_params} + , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} + , intra_query_threads_{intra_query_threads} { + // Subclasses handle their own storage kind validation + } + + svs::index::ivf::IVFBuildParameters ivf_build_parameters() const { + svs::index::ivf::IVFBuildParameters result; + set_if_specified(result.num_centroids_, build_params_.num_centroids); + set_if_specified(result.minibatch_size_, build_params_.minibatch_size); + set_if_specified(result.num_iterations_, build_params_.num_iterations); + if (is_specified(build_params_.is_hierarchical)) { + result.is_hierarchical_ = build_params_.is_hierarchical.is_enabled(); + } + set_if_specified(result.training_fraction_, build_params_.training_fraction); + set_if_specified( + result.hierarchical_level1_clusters_, build_params_.hierarchical_level1_clusters + ); + set_if_specified(result.seed_, build_params_.seed); + return result; + } + + svs::index::ivf::IVFSearchParameters + make_search_parameters(const IVFIndex::SearchParams* params) const { + // Start with default parameters + svs::index::ivf::IVFSearchParameters result; + if (is_specified(default_search_params_.n_probes)) { + result.n_probes_ = default_search_params_.n_probes; + } + if (is_specified(default_search_params_.k_reorder)) { + result.k_reorder_ = default_search_params_.k_reorder; + } + + // Override with user-specified parameters + if (params) { + set_if_specified(result.n_probes_, params->n_probes); + set_if_specified(result.k_reorder_, params->k_reorder); + } + + return result; + } + + void init_impl(data::ConstSimpleDataView data, std::span ids) { + auto build_params = ivf_build_parameters(); + auto threadpool = default_threadpool(); + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); + + impl_.reset(distance_dispatcher([&](auto&& distance) { + // Build clustering using BFloat16 for efficiency (AMX support) + auto clustering = svs::IVF::build_clustering( + build_params, data, std::forward(distance), num_threads_ + ); + + // Dispatch on storage kind to compress and assemble + return ivf_storage::dispatch_ivf_storage_kind(storage_kind_, [&](auto tag) { + // Compress data to target storage type using the factory + auto compressed_data = + ivf_storage::make_ivf_blocked_storage(tag, data, threadpool); + + return new svs::DynamicIVF(svs::DynamicIVF::assemble_from_clustering( + std::move(clustering), + std::move(compressed_data), + ids, + std::forward(distance), + num_threads_, + intra_query_threads_ + )); + }); + })); + } + + // Data members + size_t dim_; + MetricType metric_type_; + StorageKind storage_kind_; + IVFIndex::BuildParams build_params_; + IVFIndex::SearchParams default_search_params_; + size_t num_threads_; + size_t intra_query_threads_; + std::unique_ptr impl_; +}; + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// Dynamic IVF index implementation for LeanVec storage kinds +class DynamicIVFIndexLeanVecImpl : public DynamicIVFIndexImpl { + public: + using LeanVecMatricesType = LeanVecTrainingDataImpl::LeanVecMatricesType; + + // Constructor for building with training data + DynamicIVFIndexLeanVecImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const LeanVecTrainingDataImpl& training_data, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : DynamicIVFIndexImpl{dim, metric, storage_kind, params, default_search_params, num_threads, intra_query_threads, /*skip_validation=*/true} + , leanvec_dims_{training_data.get_leanvec_dims()} + , leanvec_matrices_{training_data.get_leanvec_matrices()} { + check_leanvec_storage_kind(storage_kind); + } + + // Constructor for building without pre-computed matrices (will compute during build) + DynamicIVFIndexLeanVecImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t leanvec_dims, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : DynamicIVFIndexImpl{dim, metric, storage_kind, params, default_search_params, num_threads, intra_query_threads, /*skip_validation=*/true} + , leanvec_dims_{leanvec_dims} + , leanvec_matrices_{std::nullopt} { + check_leanvec_storage_kind(storage_kind); + } + + // Constructor for loading + DynamicIVFIndexLeanVecImpl( + std::unique_ptr&& impl, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) + : DynamicIVFIndexImpl{std::move(impl), metric, storage_kind, num_threads, intra_query_threads} + , leanvec_dims_{0} + , leanvec_matrices_{std::nullopt} { + check_leanvec_storage_kind(storage_kind); + } + + void build(data::ConstSimpleDataView data, std::span ids) { + if (impl_) { + throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; + } + init_leanvec_impl(data, ids); + } + + static DynamicIVFIndexLeanVecImpl* load( + std::istream& in, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) { + if (num_threads == 0) { + num_threads = static_cast(omp_get_max_threads()); + } + + return ivf_storage::dispatch_ivf_leanvec_storage_kind(storage_kind, [&](auto tag) { + using Tag = decltype(tag); + using DataType = ivf_storage::IVFBlockedStorageType_t; + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::DynamicIVF::assemble( + in, + std::forward(distance), + num_threads, + intra_query_threads + ) + ); + return new DynamicIVFIndexLeanVecImpl( + std::move(impl), metric, storage_kind, num_threads, intra_query_threads + ); + }); + }); + } + + protected: + size_t leanvec_dims_; + std::optional leanvec_matrices_; + + void check_leanvec_storage_kind(StorageKind kind) { + if (!ivf_storage::is_leanvec_storage_kind(kind)) { + throw StatusException( + ErrorCode::INVALID_ARGUMENT, "LeanVec storage kind required" + ); + } + if (!svs::detail::lvq_leanvec_enabled()) { + throw StatusException( + ErrorCode::NOT_IMPLEMENTED, + "LeanVec storage kind requested but not supported by CPU" + ); + } + } + + void + init_leanvec_impl(data::ConstSimpleDataView data, std::span ids) { + auto build_params = ivf_build_parameters(); + auto threadpool = default_threadpool(); + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); + + impl_.reset(distance_dispatcher([&](auto&& distance) { + // Build clustering using BFloat16 for efficiency (AMX support) + auto clustering = svs::IVF::build_clustering( + build_params, data, std::forward(distance), num_threads_ + ); + + // Dispatch on LeanVec storage kind to compress and assemble + return ivf_storage::dispatch_ivf_leanvec_storage_kind( + storage_kind_, + [&](auto tag) { + // Compress data to LeanVec storage type using the factory with matrices + auto compressed_data = ivf_storage::make_ivf_blocked_leanvec_storage( + tag, data, threadpool, leanvec_dims_, leanvec_matrices_ + ); + + return new svs::DynamicIVF( + svs::DynamicIVF::assemble_from_clustering( + std::move(clustering), + std::move(compressed_data), + ids, + std::forward(distance), + num_threads_, + intra_query_threads_ + ) + ); + } + ); + })); + } +}; +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +} // namespace runtime +} // namespace svs diff --git a/bindings/cpp/src/dynamic_vamana_index.cpp b/bindings/cpp/src/dynamic_vamana_index.cpp index 73d78807d..0c1a6a890 100644 --- a/bindings/cpp/src/dynamic_vamana_index.cpp +++ b/bindings/cpp/src/dynamic_vamana_index.cpp @@ -19,7 +19,7 @@ #include "dynamic_vamana_index_impl.h" #include "svs_runtime_utils.h" -#ifdef SVS_LEANVEC_HEADER +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC #include "dynamic_vamana_index_leanvec_impl.h" #endif @@ -261,7 +261,7 @@ Status DynamicVamanaIndexLeanVec::build( ); } -#ifdef SVS_LEANVEC_HEADER +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC // Specialization to build LeanVec-based Vamana index with specified leanvec dims Status DynamicVamanaIndexLeanVec::build( DynamicVamanaIndex** index, @@ -328,7 +328,7 @@ Status DynamicVamanaIndexLeanVec::build( }); } -#else // SVS_LEANVEC_HEADER +#else // SVS_RUNTIME_HAVE_LVQ_LEANVEC // LeanVec storage kind is not supported in this build configuration Status DynamicVamanaIndexLeanVec:: build(DynamicVamanaIndex**, size_t, MetricType, StorageKind, size_t, const DynamicVamanaIndex::BuildParams&, const DynamicVamanaIndex::SearchParams&, const DynamicVamanaIndex::DynamicIndexParams&) noexcept { @@ -345,6 +345,6 @@ Status DynamicVamanaIndexLeanVec:: "DynamicVamanaIndexLeanVec is not supported in this build configuration." ); } -#endif // SVS_LEANVEC_HEADER +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC } // namespace runtime } // namespace svs diff --git a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h index 0ba764d5c..74819174f 100644 --- a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h @@ -24,7 +24,7 @@ #include #include -#include SVS_LEANVEC_HEADER +#include #include #include diff --git a/bindings/cpp/src/ivf_index.cpp b/bindings/cpp/src/ivf_index.cpp index 5027e9bc1..12706627e 100644 --- a/bindings/cpp/src/ivf_index.cpp +++ b/bindings/cpp/src/ivf_index.cpp @@ -19,6 +19,10 @@ #include "ivf_index_impl.h" #include "svs_runtime_utils.h" +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +#include "training_impl.h" +#endif + #include #include #include @@ -33,16 +37,16 @@ namespace runtime { namespace { -// Manager class for Static IVF Index -struct StaticIVFIndexManager : public StaticIVFIndex { - std::unique_ptr impl_; +// Manager class for IVF Index +struct IVFIndexManager : public IVFIndex { + std::unique_ptr impl_; - StaticIVFIndexManager(std::unique_ptr impl) + IVFIndexManager(std::unique_ptr impl) : impl_{std::move(impl)} { assert(impl_ != nullptr); } - ~StaticIVFIndexManager() override = default; + ~IVFIndexManager() override = default; Status search( size_t n, @@ -66,93 +70,24 @@ struct StaticIVFIndexManager : public StaticIVFIndex { } }; -// Manager class for Dynamic IVF Index -struct DynamicIVFIndexManager : public DynamicIVFIndex { - std::unique_ptr impl_; - - DynamicIVFIndexManager(std::unique_ptr impl) - : impl_{std::move(impl)} { - assert(impl_ != nullptr); - } - - ~DynamicIVFIndexManager() override = default; - - Status search( - size_t n, - const float* x, - size_t k, - float* distances, - size_t* labels, - const SearchParams* params = nullptr - ) const noexcept override { - return runtime_error_wrapper([&] { - auto result = svs::QueryResultView{ - svs::MatrixView{svs::make_dims(n, k), labels}, - svs::MatrixView{svs::make_dims(n, k), distances}}; - auto queries = svs::data::ConstSimpleDataView(x, n, impl_->dimensions()); - impl_->search(result, queries, params); - }); - } - - Status - add(size_t n, const size_t* labels, const float* x, bool reuse_empty - ) noexcept override { - return runtime_error_wrapper([&] { - svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; - std::span lbls(labels, n); - impl_->add(data, lbls, reuse_empty); - }); - } - - Status remove(size_t n, const size_t* labels) noexcept override { - return runtime_error_wrapper([&] { - std::span lbls(labels, n); - impl_->remove(lbls); - }); - } - - Status - remove_selected(size_t* num_removed, const IDFilter& selector) noexcept override { - return runtime_error_wrapper([&] { - *num_removed = impl_->remove_selected(selector); - }); - } - - Status has_id(bool* exists, size_t id) const noexcept override { - return runtime_error_wrapper([&] { *exists = impl_->has_id(id); }); - } - - Status consolidate() noexcept override { - return runtime_error_wrapper([&] { impl_->consolidate(); }); - } - - Status compact(size_t batchsize) noexcept override { - return runtime_error_wrapper([&] { impl_->compact(batchsize); }); - } - - Status save(std::ostream& out) const noexcept override { - return runtime_error_wrapper([&] { impl_->save(out); }); - } -}; - } // namespace // IVFIndex interface implementation IVFIndex::~IVFIndex() = default; -// StaticIVFIndex interface implementation -Status StaticIVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { +// IVFIndex interface implementation +Status IVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { if (ivf_storage::is_supported_storage_kind(storage_kind)) { return Status_Ok; } else { return Status{ ErrorCode::INVALID_ARGUMENT, - "StaticIVFIndex only supports FP32 and FP16 storage kinds"}; + "IVFIndex only supports FP32, FP16, SQI8, and LVQ storage kinds"}; } } -Status StaticIVFIndex::build( - StaticIVFIndex** index, +Status IVFIndex::build( + IVFIndex** index, size_t dim, MetricType metric, StorageKind storage_kind, @@ -165,7 +100,7 @@ Status StaticIVFIndex::build( ) noexcept { *index = nullptr; return runtime_error_wrapper([&] { - auto impl = std::make_unique( + auto impl = std::make_unique( dim, metric, storage_kind, @@ -179,16 +114,42 @@ Status StaticIVFIndex::build( svs::data::ConstSimpleDataView data_view{data, n, dim}; impl->build(data_view); - *index = new StaticIVFIndexManager{std::move(impl)}; + *index = new IVFIndexManager{std::move(impl)}; }); } -Status StaticIVFIndex::destroy(StaticIVFIndex* index) noexcept { +Status IVFIndex::build( + IVFIndex** index, + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t n, + const float* data, + const IVFIndex::BuildParams& params, + size_t num_threads, + size_t intra_query_threads +) noexcept { + SearchParams default_search_params; + return build( + index, + dim, + metric, + storage_kind, + n, + data, + params, + default_search_params, + num_threads, + intra_query_threads + ); +} + +Status IVFIndex::destroy(IVFIndex* index) noexcept { return runtime_error_wrapper([&] { delete index; }); } -Status StaticIVFIndex::load( - StaticIVFIndex** index, +Status IVFIndex::load( + IVFIndex** index, std::istream& in, MetricType metric, StorageKind storage_kind, @@ -197,32 +158,33 @@ Status StaticIVFIndex::load( ) noexcept { *index = nullptr; return runtime_error_wrapper([&] { - std::unique_ptr impl{StaticIVFIndexImpl::load( - in, metric, storage_kind, num_threads, intra_query_threads - )}; - *index = new StaticIVFIndexManager{std::move(impl)}; + std::unique_ptr impl; +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC + if (ivf_storage::is_leanvec_storage_kind(storage_kind)) { + impl.reset(IVFIndexLeanVecImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )); + } else +#endif + { + impl.reset(IVFIndexImpl::load( + in, metric, storage_kind, num_threads, intra_query_threads + )); + } + *index = new IVFIndexManager{std::move(impl)}; }); } -// DynamicIVFIndex interface implementation -Status DynamicIVFIndex::check_storage_kind(StorageKind storage_kind) noexcept { - if (ivf_storage::is_supported_storage_kind(storage_kind)) { - return Status_Ok; - } else { - return Status{ - ErrorCode::INVALID_ARGUMENT, - "DynamicIVFIndex only supports FP32 and FP16 storage kinds"}; - } -} - -Status DynamicIVFIndex::build( - DynamicIVFIndex** index, +// IVFIndexLeanVec implementations +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +Status IVFIndexLeanVec::build( + IVFIndex** index, size_t dim, MetricType metric, StorageKind storage_kind, size_t n, const float* data, - const size_t* labels, + size_t leanvec_dims, const IVFIndex::BuildParams& params, const IVFIndex::SearchParams& default_search_params, size_t num_threads, @@ -230,47 +192,97 @@ Status DynamicIVFIndex::build( ) noexcept { *index = nullptr; return runtime_error_wrapper([&] { - auto impl = std::make_unique( + auto impl = std::make_unique( dim, metric, storage_kind, + leanvec_dims, params, default_search_params, num_threads, intra_query_threads ); - // Build with provided data if any - if (n > 0 && data != nullptr && labels != nullptr) { - svs::data::ConstSimpleDataView data_view{data, n, dim}; - std::span labels_span{labels, n}; - impl->build(data_view, labels_span); - } + svs::data::ConstSimpleDataView data_view{data, n, dim}; + impl->build(data_view); - *index = new DynamicIVFIndexManager{std::move(impl)}; + *index = new IVFIndexManager{std::move(impl)}; }); } -Status DynamicIVFIndex::destroy(DynamicIVFIndex* index) noexcept { - return runtime_error_wrapper([&] { delete index; }); -} - -Status DynamicIVFIndex::load( - DynamicIVFIndex** index, - std::istream& in, +Status IVFIndexLeanVec::build( + IVFIndex** index, + size_t dim, MetricType metric, StorageKind storage_kind, + size_t n, + const float* data, + const LeanVecTrainingData* training_data, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, size_t num_threads, size_t intra_query_threads ) noexcept { *index = nullptr; return runtime_error_wrapper([&] { - std::unique_ptr impl{DynamicIVFIndexImpl::load( - in, metric, storage_kind, num_threads, intra_query_threads - )}; - *index = new DynamicIVFIndexManager{std::move(impl)}; + auto training_data_impl = + static_cast(training_data)->impl_; + auto impl = std::make_unique( + dim, + metric, + storage_kind, + training_data_impl, + params, + default_search_params, + num_threads, + intra_query_threads + ); + + svs::data::ConstSimpleDataView data_view{data, n, dim}; + impl->build(data_view); + + *index = new IVFIndexManager{std::move(impl)}; }); } +#else // SVS_RUNTIME_HAVE_LVQ_LEANVEC +Status IVFIndexLeanVec::build( + IVFIndex**, + size_t, + MetricType, + StorageKind, + size_t, + const float*, + size_t, + const IVFIndex::BuildParams&, + const IVFIndex::SearchParams&, + size_t, + size_t +) noexcept { + return Status( + ErrorCode::NOT_IMPLEMENTED, + "IVFIndexLeanVec is not supported in this build configuration." + ); +} + +Status IVFIndexLeanVec::build( + IVFIndex**, + size_t, + MetricType, + StorageKind, + size_t, + const float*, + const LeanVecTrainingData*, + const IVFIndex::BuildParams&, + const IVFIndex::SearchParams&, + size_t, + size_t +) noexcept { + return Status( + ErrorCode::NOT_IMPLEMENTED, + "IVFIndexLeanVec is not supported in this build configuration." + ); +} +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC } // namespace runtime } // namespace svs diff --git a/bindings/cpp/src/ivf_index_impl.h b/bindings/cpp/src/ivf_index_impl.h index c58684b14..0aeaa3774 100644 --- a/bindings/cpp/src/ivf_index_impl.h +++ b/bindings/cpp/src/ivf_index_impl.h @@ -27,6 +27,17 @@ #include #include +// Include scalar quantization support +#include +#include + +// Conditionally include LVQ/LeanVec headers +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +#include "training_impl.h" +#include +#include +#endif + #include #include #include @@ -35,39 +46,312 @@ namespace svs { namespace runtime { -// IVF storage kind support - IVF supports a subset of storage kinds +// IVF storage kind support - following the Vamana storage pattern namespace ivf_storage { -// IVF supports FP32 and FP16 storage kinds -inline bool is_supported_storage_kind(StorageKind kind) { +// Check if storage kind is LeanVec (requires training data) +inline bool is_leanvec_storage_kind(StorageKind kind) { +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC + return kind == StorageKind::LeanVec4x4 || kind == StorageKind::LeanVec4x8 || + kind == StorageKind::LeanVec8x8; +#else + (void)kind; + return false; +#endif +} + +// Check if storage kind is supported for IVF (non-LeanVec) +inline bool is_supported_non_leanvec_storage_kind(StorageKind kind) { switch (kind) { case StorageKind::FP32: case StorageKind::FP16: + case StorageKind::SQI8: + return true; +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC + case StorageKind::LVQ4x0: + case StorageKind::LVQ8x0: + case StorageKind::LVQ4x4: + case StorageKind::LVQ4x8: return true; +#endif default: return false; } } -// IVF data type for static index (uses lib::Allocator) +// Check if any storage kind is supported for IVF (including LeanVec) +inline bool is_supported_storage_kind(StorageKind kind) { + return is_supported_non_leanvec_storage_kind(kind) || is_leanvec_storage_kind(kind); +} + +///// IVF Data Types ///// + +// Simple uncompressed data types template -using IVFDataType = svs::data::SimpleData>; +using IVFSimpleDataType = svs::data::SimpleData>; -// IVF data type for dynamic index (uses Blocked allocator) template -using IVFBlockedDataType = +using IVFBlockedSimpleDataType = svs::data::SimpleData>>; -// Dispatch on storage kind for IVF operations +// Scalar Quantization data types +template +using IVFSQDataType = + svs::quantization::scalar::SQDataset>; + +template +using IVFBlockedSQDataType = svs::quantization::scalar:: + SQDataset>>; + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// LVQ data types +template +using IVFLVQDataType = svs::quantization::lvq:: + LVQDataset>; + +template +using IVFBlockedLVQDataType = svs::quantization::lvq::LVQDataset< + Primary, + Residual, + svs::Dynamic, + Strategy, + svs::data::Blocked>>; + +using Sequential = svs::quantization::lvq::Sequential; +using Turbo16x8 = svs::quantization::lvq::Turbo<16, 8>; + +// LeanVec data types +template +using IVFLeanVecDataType = svs::leanvec::LeanDataset< + svs::leanvec::UsingLVQ, + svs::leanvec::UsingLVQ, + svs::Dynamic, + svs::Dynamic, + svs::lib::Allocator>; + +template +using IVFBlockedLeanVecDataType = svs::leanvec::LeanDataset< + svs::leanvec::UsingLVQ, + svs::leanvec::UsingLVQ, + svs::Dynamic, + svs::Dynamic, + svs::data::Blocked>>; +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +///// Storage Type Mapping ///// + +// Map StorageKind to data type using storage tags +template struct IVFStorageType { + using type = storage::UnsupportedStorageType; +}; + +template struct IVFBlockedStorageType { + using type = storage::UnsupportedStorageType; +}; + +template +using IVFStorageType_t = typename IVFStorageType::type; + +template +using IVFBlockedStorageType_t = typename IVFBlockedStorageType::type; + +// clang-format off +template <> struct IVFStorageType { using type = IVFSimpleDataType; }; +template <> struct IVFStorageType { using type = IVFSimpleDataType; }; +template <> struct IVFStorageType { using type = IVFSQDataType; }; + +template <> struct IVFBlockedStorageType { using type = IVFBlockedSimpleDataType; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedSimpleDataType; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedSQDataType; }; +// clang-format on + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// clang-format off +template <> struct IVFStorageType { using type = IVFLVQDataType<4, 0, Turbo16x8>; }; +template <> struct IVFStorageType { using type = IVFLVQDataType<8, 0, Sequential>; }; +template <> struct IVFStorageType { using type = IVFLVQDataType<4, 4, Turbo16x8>; }; +template <> struct IVFStorageType { using type = IVFLVQDataType<4, 8, Turbo16x8>; }; + +template <> struct IVFBlockedStorageType { using type = IVFBlockedLVQDataType<4, 0, Turbo16x8>; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedLVQDataType<8, 0, Sequential>; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedLVQDataType<4, 4, Turbo16x8>; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedLVQDataType<4, 8, Turbo16x8>; }; +// clang-format on + +// clang-format off +template <> struct IVFStorageType { using type = IVFLeanVecDataType<4, 4>; }; +template <> struct IVFStorageType { using type = IVFLeanVecDataType<4, 8>; }; +template <> struct IVFStorageType { using type = IVFLeanVecDataType<8, 8>; }; + +template <> struct IVFBlockedStorageType { using type = IVFBlockedLeanVecDataType<4, 4>; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedLeanVecDataType<4, 8>; }; +template <> struct IVFBlockedStorageType { using type = IVFBlockedLeanVecDataType<8, 8>; }; +// clang-format on +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +///// Storage Factory ///// + +template struct IVFStorageFactory; + +// Unsupported storage factory +template <> struct IVFStorageFactory { + using DataType = IVFSimpleDataType; + + template + static DataType + compress(const svs::data::ConstSimpleDataView&, Pool&, size_t = 0) { + throw StatusException( + ErrorCode::NOT_IMPLEMENTED, "Requested storage kind is not supported for IVF" + ); + } +}; + +// Simple data factory (FP32, FP16) +template +struct IVFStorageFactory> { + using DataType = svs::data::SimpleData; + + template + static DataType + compress(const svs::data::ConstSimpleDataView& data, Pool& pool, size_t = 0) { + DataType result(data.size(), data.dimensions()); + svs::threads::parallel_for( + pool, + svs::threads::StaticPartition(result.size()), + [&](auto is, auto) { + for (auto i : is) { + result.set_datum(i, data.get_datum(i)); + } + } + ); + return result; + } +}; + +// Scalar Quantization factory +template +struct IVFStorageFactory> { + using DataType = svs::quantization::scalar::SQDataset; + + template + static DataType + compress(const svs::data::ConstSimpleDataView& data, Pool& pool, size_t = 0) { + return DataType::compress(data, pool); + } +}; + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// LVQ factory +template +struct IVFStorageFactory< + svs::quantization::lvq::LVQDataset> { + using DataType = + svs::quantization::lvq::LVQDataset; + + template + static DataType + compress(const svs::data::ConstSimpleDataView& data, Pool& pool, size_t = 0) { + return DataType::compress(data, pool, 0); + } +}; + +// LeanVec factory - requires optional matrices for proper training +template +struct IVFStorageFactory> { + using DataType = svs::leanvec::LeanDataset; + using LeanVecMatricesType = svs::leanvec::LeanVecMatrices; + + template + static DataType compress( + const svs::data::ConstSimpleDataView& data, + Pool& pool, + size_t leanvec_d = 0, + std::optional matrices = std::nullopt + ) { + if (leanvec_d == 0) { + leanvec_d = (data.dimensions() + 1) / 2; + } + return DataType::reduce( + data, std::move(matrices), pool, 0, svs::lib::MaybeStatic{leanvec_d} + ); + } +}; +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +// Helper to make compressed data (non-LeanVec) +template + requires storage::StorageTag> +auto make_ivf_storage( + Tag&&, const svs::data::ConstSimpleDataView& data, Pool& pool, size_t arg = 0 +) { + using TagDecay = std::decay_t; + return IVFStorageFactory>::compress(data, pool, arg); +} + +template + requires storage::StorageTag> +auto make_ivf_blocked_storage( + Tag&&, const svs::data::ConstSimpleDataView& data, Pool& pool, size_t arg = 0 +) { + using TagDecay = std::decay_t; + return IVFStorageFactory>::compress(data, pool, arg); +} + +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// LeanVec-specific make functions with matrices parameter +template + requires storage::StorageTag> +auto make_ivf_leanvec_storage( + Tag&&, + const svs::data::ConstSimpleDataView& data, + Pool& pool, + size_t leanvec_d, + std::optional> matrices +) { + using TagDecay = std::decay_t; + return IVFStorageFactory>::compress( + data, pool, leanvec_d, std::move(matrices) + ); +} + +template + requires storage::StorageTag> +auto make_ivf_blocked_leanvec_storage( + Tag&&, + const svs::data::ConstSimpleDataView& data, + Pool& pool, + size_t leanvec_d, + std::optional> matrices +) { + using TagDecay = std::decay_t; + return IVFStorageFactory>::compress( + data, pool, leanvec_d, std::move(matrices) + ); +} +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC + +///// Dispatch Functions ///// + +// Dispatch on storage kind for IVF operations (excludes LeanVec - handled separately) template auto dispatch_ivf_storage_kind(StorageKind kind, F&& f, Args&&... args) { switch (kind) { case StorageKind::FP32: - return f(svs::lib::Type>{}, std::forward(args)...); + return f(storage::FP32Tag{}, std::forward(args)...); case StorageKind::FP16: - return f( - svs::lib::Type>{}, std::forward(args)... - ); + return f(storage::FP16Tag{}, std::forward(args)...); + case StorageKind::SQI8: + return f(storage::SQI8Tag{}, std::forward(args)...); +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC + case StorageKind::LVQ4x0: + return f(storage::LVQ4x0Tag{}, std::forward(args)...); + case StorageKind::LVQ8x0: + return f(storage::LVQ8x0Tag{}, std::forward(args)...); + case StorageKind::LVQ4x4: + return f(storage::LVQ4x4Tag{}, std::forward(args)...); + case StorageKind::LVQ4x8: + return f(storage::LVQ4x8Tag{}, std::forward(args)...); +#endif default: throw StatusException{ ErrorCode::NOT_IMPLEMENTED, @@ -75,32 +359,30 @@ auto dispatch_ivf_storage_kind(StorageKind kind, F&& f, Args&&... args) { } } -// Dispatch on storage kind for Dynamic IVF operations (uses blocked allocator) +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// Dispatch on LeanVec storage kinds only template -auto dispatch_ivf_blocked_storage_kind(StorageKind kind, F&& f, Args&&... args) { +auto dispatch_ivf_leanvec_storage_kind(StorageKind kind, F&& f, Args&&... args) { switch (kind) { - case StorageKind::FP32: - return f( - svs::lib::Type>{}, std::forward(args)... - ); - case StorageKind::FP16: - return f( - svs::lib::Type>{}, - std::forward(args)... - ); + case StorageKind::LeanVec4x4: + return f(storage::LeanVec4x4Tag{}, std::forward(args)...); + case StorageKind::LeanVec4x8: + return f(storage::LeanVec4x8Tag{}, std::forward(args)...); + case StorageKind::LeanVec8x8: + return f(storage::LeanVec8x8Tag{}, std::forward(args)...); default: throw StatusException{ - ErrorCode::NOT_IMPLEMENTED, - "Requested storage kind is not supported for Dynamic IVF index"}; + ErrorCode::INVALID_ARGUMENT, "LeanVec storage kind required"}; } } +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC } // namespace ivf_storage -// Static IVF index implementation -class StaticIVFIndexImpl { +// Static IVF index implementation (non-LeanVec storage kinds) +class IVFIndexImpl { public: - StaticIVFIndexImpl( + IVFIndexImpl( size_t dim, MetricType metric, StorageKind storage_kind, @@ -116,10 +398,11 @@ class StaticIVFIndexImpl { , default_search_params_{default_search_params} , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} , intra_query_threads_{intra_query_threads} { - if (!ivf_storage::is_supported_storage_kind(storage_kind)) { + if (!ivf_storage::is_supported_non_leanvec_storage_kind(storage_kind)) { throw StatusException{ ErrorCode::INVALID_ARGUMENT, - "The specified storage kind is not compatible with StaticIVFIndex"}; + "The specified storage kind is not compatible with IVFIndex. " + "Use IVFIndexLeanVecImpl for LeanVec storage kinds."}; } } @@ -173,7 +456,7 @@ class StaticIVFIndexImpl { impl_->save(out); } - static StaticIVFIndexImpl* load( + static IVFIndexImpl* load( std::istream& in, MetricType metric, StorageKind storage_kind, @@ -185,34 +468,30 @@ class StaticIVFIndexImpl { } // Dispatch on storage kind to load with correct data type - return ivf_storage::dispatch_ivf_storage_kind( - storage_kind, - [&](svs::lib::Type) { - svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); - return distance_dispatcher([&](auto&& distance) { - auto impl = std::make_unique( - svs::IVF::assemble( - in, - std::forward(distance), - num_threads, - intra_query_threads - ) - ); - return new StaticIVFIndexImpl( - std::move(impl), - metric, - storage_kind, + return ivf_storage::dispatch_ivf_storage_kind(storage_kind, [&](auto tag) { + using Tag = decltype(tag); + using DataType = ivf_storage::IVFStorageType_t; + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::IVF::assemble( + in, + std::forward(distance), num_threads, intra_query_threads - ); - }); - } - ); + ) + ); + return new IVFIndexImpl( + std::move(impl), metric, storage_kind, num_threads, intra_query_threads + ); + }); + }); } protected: // Constructor used during loading - StaticIVFIndexImpl( + IVFIndexImpl( std::unique_ptr&& impl, MetricType metric, StorageKind storage_kind, @@ -227,7 +506,29 @@ class StaticIVFIndexImpl { , impl_{std::move(impl)} { // Extract default search params from loaded index auto loaded_params = impl_->get_search_parameters(); - default_search_params_ = {loaded_params.n_probes_, loaded_params.k_reorder_}; + default_search_params_.n_probes = loaded_params.n_probes_; + default_search_params_.k_reorder = loaded_params.k_reorder_; + } + + // Constructor used by subclasses (LeanVec) that handle their own validation + IVFIndexImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads, + bool /*skip_validation*/ + ) + : dim_{dim} + , metric_type_{metric} + , storage_kind_{storage_kind} + , build_params_{params} + , default_search_params_{default_search_params} + , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} + , intra_query_threads_{intra_query_threads} { + // Subclasses handle their own storage kind validation } svs::index::ivf::IVFBuildParameters ivf_build_parameters() const { @@ -268,56 +569,29 @@ class StaticIVFIndexImpl { void init_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - - // Single copy of data - required because IVF assembly deduces internal types from - // data type, and ConstSimpleDataView has const element type which breaks - // internal type deduction. This copy is also passed directly to assemble which - // partitions it into clusters (no additional copy for FP32 storage). - auto owned_data = svs::data::SimpleData(data.size(), data.dimensions()); - svs::data::copy(data, owned_data); + auto threadpool = default_threadpool(); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); impl_.reset(distance_dispatcher([&](auto&& distance) { // Build clustering using BFloat16 for efficiency (AMX support) - // Note: build_clustering takes const ref, doesn't consume data auto clustering = svs::IVF::build_clustering( - build_params, - owned_data, - std::forward(distance), - num_threads_ + build_params, data, std::forward(distance), num_threads_ ); - // Dispatch on storage kind to assemble with correct data type - return ivf_storage::dispatch_ivf_storage_kind( - storage_kind_, - [&](svs::lib::Type) { - using TargetElement = typename DataType::element_type; - - // For FP32: pass owned_data directly (moved into clusters) - // For FP16: convert from owned_data - if constexpr (std::is_same_v) { - return new svs::IVF(svs::IVF::assemble_from_clustering( - std::move(clustering), - owned_data, - std::forward(distance), - num_threads_, - intra_query_threads_ - )); - } else { - // Convert to target type (e.g., FP16) - DataType converted_data(owned_data.size(), owned_data.dimensions()); - svs::data::copy(owned_data, converted_data); - return new svs::IVF(svs::IVF::assemble_from_clustering( - std::move(clustering), - std::move(converted_data), - std::forward(distance), - num_threads_, - intra_query_threads_ - )); - } - } - ); + // Dispatch on storage kind to compress and assemble + return ivf_storage::dispatch_ivf_storage_kind(storage_kind_, [&](auto tag) { + // Compress data to target storage type using the factory + auto compressed_data = ivf_storage::make_ivf_storage(tag, data, threadpool); + + return new svs::IVF(svs::IVF::assemble_from_clustering( + std::move(clustering), + std::move(compressed_data), + std::forward(distance), + num_threads_, + intra_query_threads_ + )); + }); })); } @@ -332,141 +606,68 @@ class StaticIVFIndexImpl { std::unique_ptr impl_; }; -// Dynamic IVF index implementation -class DynamicIVFIndexImpl { +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +// Static IVF index implementation for LeanVec storage kinds +class IVFIndexLeanVecImpl : public IVFIndexImpl { public: - DynamicIVFIndexImpl( + using LeanVecMatricesType = LeanVecTrainingDataImpl::LeanVecMatricesType; + + // Constructor for building with training data + IVFIndexLeanVecImpl( size_t dim, MetricType metric, StorageKind storage_kind, + const LeanVecTrainingDataImpl& training_data, const IVFIndex::BuildParams& params, const IVFIndex::SearchParams& default_search_params, size_t num_threads, size_t intra_query_threads ) - : dim_{dim} - , metric_type_{metric} - , storage_kind_{storage_kind} - , build_params_{params} - , default_search_params_{default_search_params} - , num_threads_{num_threads == 0 ? static_cast(omp_get_max_threads()) : num_threads} - , intra_query_threads_{intra_query_threads} { - if (!ivf_storage::is_supported_storage_kind(storage_kind)) { - throw StatusException{ - ErrorCode::INVALID_ARGUMENT, - "The specified storage kind is not compatible with DynamicIVFIndex"}; - } - } - - size_t size() const { return impl_ ? impl_->size() : 0; } - - size_t dimensions() const { return dim_; } - - MetricType metric_type() const { return metric_type_; } - - StorageKind get_storage_kind() const { return storage_kind_; } - - void build(data::ConstSimpleDataView data, std::span ids) { - if (impl_) { - throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; - } - init_impl(data, ids); - } - - void - add(data::ConstSimpleDataView data, - std::span ids, - bool reuse_empty = false) { - if (!impl_) { - // First add initializes the index - init_impl(data, ids); - return; - } - impl_->add_points(data, ids, reuse_empty); - } - - size_t remove(std::span ids) { - if (!impl_) { - throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; - } - return impl_->delete_points(ids); - } - - size_t remove_selected(const IDFilter& selector) { - if (!impl_) { - throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; - } - - auto ids = impl_->all_ids(); - std::vector ids_to_delete; - std::copy_if( - ids.begin(), - ids.end(), - std::back_inserter(ids_to_delete), - [&](size_t id) { return selector(id); } - ); - - return impl_->delete_points(ids_to_delete); - } - - bool has_id(size_t id) const { - if (!impl_) { - return false; - } - return impl_->has_id(id); - } - - void consolidate() { - if (!impl_) { - throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; - } - impl_->consolidate(); + : IVFIndexImpl{dim, metric, storage_kind, params, default_search_params, num_threads, intra_query_threads, /*skip_validation=*/true} + , leanvec_dims_{training_data.get_leanvec_dims()} + , leanvec_matrices_{training_data.get_leanvec_matrices()} { + check_leanvec_storage_kind(storage_kind); } - void compact(size_t batchsize = 1'000'000) { - if (!impl_) { - throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; - } - impl_->compact(batchsize); + // Constructor for building without pre-computed matrices (will compute during build) + IVFIndexLeanVecImpl( + size_t dim, + MetricType metric, + StorageKind storage_kind, + size_t leanvec_dims, + const IVFIndex::BuildParams& params, + const IVFIndex::SearchParams& default_search_params, + size_t num_threads, + size_t intra_query_threads + ) + : IVFIndexImpl{dim, metric, storage_kind, params, default_search_params, num_threads, intra_query_threads, /*skip_validation=*/true} + , leanvec_dims_{leanvec_dims} + , leanvec_matrices_{std::nullopt} { + check_leanvec_storage_kind(storage_kind); } - void search( - svs::QueryResultView result, - svs::data::ConstSimpleDataView queries, - const IVFIndex::SearchParams* params = nullptr - ) const { - if (!impl_) { - auto& dists = result.distances(); - std::fill(dists.begin(), dists.end(), Unspecify()); - auto& inds = result.indices(); - std::fill(inds.begin(), inds.end(), Unspecify()); - throw StatusException{ErrorCode::NOT_INITIALIZED, "Index not initialized"}; - } - - if (queries.size() == 0) { - return; - } - - const size_t k = result.n_neighbors(); - if (k == 0) { - throw StatusException{ErrorCode::INVALID_ARGUMENT, "k must be greater than 0"}; - } - - auto sp = make_search_parameters(params); - impl_->set_search_parameters(sp); - impl_->search(result, queries, {}); + // Constructor for loading + IVFIndexLeanVecImpl( + std::unique_ptr&& impl, + MetricType metric, + StorageKind storage_kind, + size_t num_threads, + size_t intra_query_threads + ) + : IVFIndexImpl{std::move(impl), metric, storage_kind, num_threads, intra_query_threads} + , leanvec_dims_{0} + , leanvec_matrices_{std::nullopt} { + check_leanvec_storage_kind(storage_kind); } - void save(std::ostream& out) const { - if (!impl_) { - throw StatusException{ - ErrorCode::NOT_INITIALIZED, - "Cannot serialize: DynamicIVF index not initialized."}; + void build(data::ConstSimpleDataView data) { + if (impl_) { + throw StatusException{ErrorCode::RUNTIME_ERROR, "Index already initialized"}; } - impl_->save(out); + init_leanvec_impl(data); } - static DynamicIVFIndexImpl* load( + static IVFIndexLeanVecImpl* load( std::istream& in, MetricType metric, StorageKind storage_kind, @@ -477,159 +678,79 @@ class DynamicIVFIndexImpl { num_threads = static_cast(omp_get_max_threads()); } - // Dispatch on storage kind to load with correct data type - return ivf_storage::dispatch_ivf_storage_kind( - storage_kind, - [&](svs::lib::Type) { - svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); - return distance_dispatcher([&](auto&& distance) { - auto impl = std::make_unique( - svs::DynamicIVF::assemble( - in, - std::forward(distance), - num_threads, - intra_query_threads - ) - ); - return new DynamicIVFIndexImpl( - std::move(impl), - metric, - storage_kind, + return ivf_storage::dispatch_ivf_leanvec_storage_kind(storage_kind, [&](auto tag) { + using Tag = decltype(tag); + using DataType = ivf_storage::IVFStorageType_t; + + svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric)); + return distance_dispatcher([&](auto&& distance) { + auto impl = std::make_unique( + svs::IVF::assemble( + in, + std::forward(distance), num_threads, intra_query_threads - ); - }); - } - ); + ) + ); + return new IVFIndexLeanVecImpl( + std::move(impl), metric, storage_kind, num_threads, intra_query_threads + ); + }); + }); } protected: - // Constructor used during loading - DynamicIVFIndexImpl( - std::unique_ptr&& impl, - MetricType metric, - StorageKind storage_kind, - size_t num_threads, - size_t intra_query_threads - ) - : dim_{impl->dimensions()} - , metric_type_{metric} - , storage_kind_{storage_kind} - , num_threads_{num_threads} - , intra_query_threads_{intra_query_threads} - , impl_{std::move(impl)} { - // Extract default search params from loaded index - auto loaded_params = impl_->get_search_parameters(); - default_search_params_ = {loaded_params.n_probes_, loaded_params.k_reorder_}; - } - - svs::index::ivf::IVFBuildParameters ivf_build_parameters() const { - svs::index::ivf::IVFBuildParameters result; - set_if_specified(result.num_centroids_, build_params_.num_centroids); - set_if_specified(result.minibatch_size_, build_params_.minibatch_size); - set_if_specified(result.num_iterations_, build_params_.num_iterations); - if (is_specified(build_params_.is_hierarchical)) { - result.is_hierarchical_ = build_params_.is_hierarchical.is_enabled(); - } - set_if_specified(result.training_fraction_, build_params_.training_fraction); - set_if_specified( - result.hierarchical_level1_clusters_, build_params_.hierarchical_level1_clusters - ); - set_if_specified(result.seed_, build_params_.seed); - return result; - } + size_t leanvec_dims_; + std::optional leanvec_matrices_; - svs::index::ivf::IVFSearchParameters - make_search_parameters(const IVFIndex::SearchParams* params) const { - // Start with default parameters - svs::index::ivf::IVFSearchParameters result; - if (is_specified(default_search_params_.n_probes)) { - result.n_probes_ = default_search_params_.n_probes; - } - if (is_specified(default_search_params_.k_reorder)) { - result.k_reorder_ = default_search_params_.k_reorder; + void check_leanvec_storage_kind(StorageKind kind) { + if (!ivf_storage::is_leanvec_storage_kind(kind)) { + throw StatusException( + ErrorCode::INVALID_ARGUMENT, "LeanVec storage kind required" + ); } - - // Override with user-specified parameters - if (params) { - set_if_specified(result.n_probes_, params->n_probes); - set_if_specified(result.k_reorder_, params->k_reorder); + if (!svs::detail::lvq_leanvec_enabled()) { + throw StatusException( + ErrorCode::NOT_IMPLEMENTED, + "LeanVec storage kind requested but not supported by CPU" + ); } - - return result; } - void init_impl(data::ConstSimpleDataView data, std::span ids) { + void init_leanvec_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - - // Single copy of data - required because IVF assembly deduces internal types from - // data type, and ConstSimpleDataView has const element type which breaks - // internal type deduction. This copy is also passed directly to assemble which - // partitions it into clusters (no additional copy for FP32 storage). - auto owned_data = svs::data::SimpleData(data.size(), data.dimensions()); - svs::data::copy(data, owned_data); + auto threadpool = default_threadpool(); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); impl_.reset(distance_dispatcher([&](auto&& distance) { // Build clustering using BFloat16 for efficiency (AMX support) - // Note: build_clustering takes const ref, doesn't consume data auto clustering = svs::IVF::build_clustering( - build_params, - owned_data, - std::forward(distance), - num_threads_ + build_params, data, std::forward(distance), num_threads_ ); - // Dispatch on storage kind to assemble with correct data type - return ivf_storage::dispatch_ivf_storage_kind( + // Dispatch on LeanVec storage kind to compress and assemble + return ivf_storage::dispatch_ivf_leanvec_storage_kind( storage_kind_, - [&](svs::lib::Type) { - using TargetElement = typename DataType::element_type; - - // For FP32: pass owned_data directly (moved into clusters) - // For FP16: convert from owned_data - if constexpr (std::is_same_v) { - return new svs::DynamicIVF( - svs::DynamicIVF::assemble_from_clustering( - std::move(clustering), - owned_data, - ids, - std::forward(distance), - num_threads_, - intra_query_threads_ - ) - ); - } else { - // Convert to target type (e.g., FP16) - DataType converted_data(owned_data.size(), owned_data.dimensions()); - svs::data::copy(owned_data, converted_data); - return new svs::DynamicIVF( - svs::DynamicIVF::assemble_from_clustering( - std::move(clustering), - std::move(converted_data), - ids, - std::forward(distance), - num_threads_, - intra_query_threads_ - ) - ); - } + [&](auto tag) { + // Compress data to LeanVec storage type using the factory with matrices + auto compressed_data = ivf_storage::make_ivf_leanvec_storage( + tag, data, threadpool, leanvec_dims_, leanvec_matrices_ + ); + + return new svs::IVF(svs::IVF::assemble_from_clustering( + std::move(clustering), + std::move(compressed_data), + std::forward(distance), + num_threads_, + intra_query_threads_ + )); } ); })); } - - // Data members - size_t dim_; - MetricType metric_type_; - StorageKind storage_kind_; - IVFIndex::BuildParams build_params_; - IVFIndex::SearchParams default_search_params_; - size_t num_threads_; - size_t intra_query_threads_; - std::unique_ptr impl_; }; +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC } // namespace runtime } // namespace svs diff --git a/bindings/cpp/src/svs_runtime_utils.h b/bindings/cpp/src/svs_runtime_utils.h index b8d7e98f3..52f4ff5e3 100644 --- a/bindings/cpp/src/svs_runtime_utils.h +++ b/bindings/cpp/src/svs_runtime_utils.h @@ -27,16 +27,10 @@ #include #include -#ifdef SVS_LVQ_HEADER -#include SVS_LVQ_HEADER -#endif - -#ifdef SVS_LEANVEC_HEADER -#include SVS_LEANVEC_HEADER -#endif - -#if defined(SVS_LEANVEC_HEADER) || defined(SVS_LVQ_HEADER) +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC #include +#include +#include #else namespace svs::detail { inline bool lvq_leanvec_enabled() { return false; } @@ -249,7 +243,7 @@ struct StorageFactory { }; // LVQ Storage support -#ifdef SVS_LVQ_HEADER +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC template using LVQDatasetType = svs::quantization::lvq::LVQDataset< Primary, @@ -288,9 +282,8 @@ struct StorageFactory { return svs::lib::load_from_disk(path, SVS_FWD(args)...); } }; -#endif // SVS_LVQ_HEADER -#ifdef SVS_LEANVEC_HEADER +// LeanVec Storage support template using LeanDatasetType = svs::leanvec::LeanDataset< svs::leanvec::UsingLVQ, @@ -332,7 +325,7 @@ struct StorageFactory { return svs::lib::load_from_disk(path, SVS_FWD(args)...); } }; -#endif // SVS_LEANVEC_HEADER +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC template auto make_storage(Tag&& SVS_UNUSED(tag), Args&&... args) { diff --git a/bindings/cpp/src/training.cpp b/bindings/cpp/src/training.cpp index fdcb30b62..23afd3321 100644 --- a/bindings/cpp/src/training.cpp +++ b/bindings/cpp/src/training.cpp @@ -18,7 +18,7 @@ #include "svs_runtime_utils.h" -#ifdef SVS_LEANVEC_HEADER +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC #include "training_impl.h" namespace svs { @@ -74,7 +74,7 @@ LeanVecTrainingData::load(LeanVecTrainingData** training_data, std::istream& in) } // namespace runtime } // namespace svs -#else // SVS_LEANVEC_HEADER +#else // SVS_RUNTIME_HAVE_LVQ_LEANVEC namespace svs { namespace runtime { LeanVecTrainingData::~LeanVecTrainingData() = default; @@ -107,4 +107,4 @@ Status LeanVecTrainingData::load( } } // namespace runtime } // namespace svs -#endif // SVS_LEANVEC_HEADER +#endif // SVS_RUNTIME_HAVE_LVQ_LEANVEC diff --git a/bindings/cpp/src/training_impl.h b/bindings/cpp/src/training_impl.h index 3795ac81f..adc9e80d7 100644 --- a/bindings/cpp/src/training_impl.h +++ b/bindings/cpp/src/training_impl.h @@ -24,7 +24,11 @@ #include #include #include -#include SVS_LEANVEC_HEADER + +// Include the core LeanVec header for LeanVecMatrices and related types. +#ifdef SVS_RUNTIME_HAVE_LVQ_LEANVEC +#include +#endif #include #include diff --git a/bindings/cpp/tests/CMakeLists.txt b/bindings/cpp/tests/CMakeLists.txt index c87d31487..dd70352c6 100644 --- a/bindings/cpp/tests/CMakeLists.txt +++ b/bindings/cpp/tests/CMakeLists.txt @@ -44,6 +44,10 @@ set(TEST_SOURCES ${CMAKE_CURRENT_LIST_DIR}/runtime_test.cpp ) +if (SVS_RUNTIME_ENABLE_IVF) + list(APPEND TEST_SOURCES ${CMAKE_CURRENT_LIST_DIR}/ivf_runtime_test.cpp) +endif() + add_executable(svs_runtime_test ${TEST_SOURCES}) # Link with the runtime library diff --git a/bindings/cpp/tests/ivf_runtime_test.cpp b/bindings/cpp/tests/ivf_runtime_test.cpp new file mode 100644 index 000000000..692f1d404 --- /dev/null +++ b/bindings/cpp/tests/ivf_runtime_test.cpp @@ -0,0 +1,843 @@ +/* + * Copyright 2026 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "svs/runtime/api_defs.h" +#include "svs/runtime/dynamic_ivf_index.h" +#include "svs/runtime/ivf_index.h" + +#include + +#include +#include +#include +#include +#include + +#include "utils.h" + +namespace { + +// Generate test data +std::vector create_test_data(size_t n, size_t d, unsigned int seed = 123) { + std::vector data(n * d); + std::mt19937 gen(seed); + std::uniform_real_distribution dis(0.0f, 1.0f); + for (size_t i = 0; i < data.size(); ++i) { + data[i] = dis(gen); + } + return data; +} + +constexpr size_t test_d = 64; +constexpr size_t test_n = 100; + +// Global test data - generated once and reused across all tests +inline const std::vector& get_test_data() { + static const std::vector test_data = create_test_data(test_n, test_d, 123); + return test_data; +} + +} // namespace + +CATCH_TEST_CASE("IVFIndexBuildAndSearch", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexBuildAndSearch..." << std::endl; + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; // Small number for test data + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Verify results are reasonable (at least some results found) + bool found_valid = false; + for (int i = 0; i < nq * k; ++i) { + if (result_labels[i] < test_n) { + found_valid = true; + break; + } + } + CATCH_REQUIRE(found_valid); + + svs::runtime::v0::IVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexWriteAndRead", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexWriteAndRead..." << std::endl; + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "static_ivf_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::IVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::IVFIndex::load( + &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Test search on loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Clean up + svs::runtime::v0::IVFIndex::destroy(index); + svs::runtime::v0::IVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearch", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexBuildAndSearch..." << std::endl; + const auto& test_data = get_test_data(); + + // Build dynamic IVF index with initial data + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::IVFIndex::SearchParams search_params; + search_params.n_probes = 3; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params, + search_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("DynamicIVFIndexAddAndRemove", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexAddAndRemove..." << std::endl; + const auto& test_data = get_test_data(); + + // Build empty dynamic IVF index first, then add data + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + // Create with initial data (needed for clustering) + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Check has_id for existing IDs + bool exists = false; + status = index->has_id(&exists, 0); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + status = index->has_id(&exists, test_n - 1); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + // Check has_id for non-existing ID + status = index->has_id(&exists, test_n + 100); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + + // Remove some IDs + std::vector ids_to_remove = {0, 1, 2}; + status = index->remove(ids_to_remove.size(), ids_to_remove.data()); + CATCH_REQUIRE(status.ok()); + + // Verify removed IDs no longer exist + status = index->has_id(&exists, 0); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + + // Consolidate and compact + status = index->consolidate(); + CATCH_REQUIRE(status.ok()); + + status = index->compact(); + CATCH_REQUIRE(status.ok()); + + // Search should still work + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("DynamicIVFIndexWriteAndRead", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexWriteAndRead..." << std::endl; + const auto& test_data = get_test_data(); + + // Build dynamic IVF index + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "dynamic_ivf_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::DynamicIVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::DynamicIVFIndex::load( + &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Test search on loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + // Clean up + svs::runtime::v0::DynamicIVFIndex::destroy(index); + svs::runtime::v0::DynamicIVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexRemoveSelected", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexRemoveSelected..." << std::endl; + const auto& test_data = get_test_data(); + + // Build dynamic IVF index + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + labels.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Remove IDs in range [0, 20) using selector + size_t min_id = 0; + size_t max_id = 20; + test_utils::IDFilterRange selector(min_id, max_id); + + size_t num_removed = 0; + status = index->remove_selected(&num_removed, selector); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(num_removed == max_id - min_id); + + // Verify removed IDs no longer exist + bool exists = false; + for (size_t i = min_id; i < max_id; ++i) { + status = index->has_id(&exists, i); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(!exists); + } + + // Verify IDs outside range still exist + status = index->has_id(&exists, max_id); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(exists); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexSearchWithParams..." << std::endl; + const auto& test_data = get_test_data(); + + // Build static IVF index + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::IVFIndex::SearchParams default_search_params; + default_search_params.n_probes = 2; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params, + default_search_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances1(nq * k); + std::vector result_labels1(nq * k); + + // Search with default params + status = index->search(nq, xq, k, distances1.data(), result_labels1.data()); + CATCH_REQUIRE(status.ok()); + + // Search with custom params (more probes should potentially give better results) + svs::runtime::v0::IVFIndex::SearchParams custom_params; + custom_params.n_probes = 5; + custom_params.k_reorder = 2.0f; + + std::vector distances2(nq * k); + std::vector result_labels2(nq * k); + + status = + index->search(nq, xq, k, distances2.data(), result_labels2.data(), &custom_params); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexCheckStorageKind", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexCheckStorageKind..." << std::endl; + // FP32 should always be supported + CATCH_REQUIRE( + svs::runtime::v0::IVFIndex::check_storage_kind(svs::runtime::v0::StorageKind::FP32) + .ok() + ); + CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP32 + ) + .ok()); + + // FP16 should always be supported + CATCH_REQUIRE( + svs::runtime::v0::IVFIndex::check_storage_kind(svs::runtime::v0::StorageKind::FP16) + .ok() + ); + CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::FP16 + ) + .ok()); + + // SQI8 should always be supported + CATCH_REQUIRE( + svs::runtime::v0::IVFIndex::check_storage_kind(svs::runtime::v0::StorageKind::SQI8) + .ok() + ); + CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::SQI8 + ) + .ok()); + + // LVQ and LeanVec support depends on build configuration + // check_storage_kind will return ok when built with LVQ/LeanVec support + auto lvq_status = + svs::runtime::v0::IVFIndex::check_storage_kind(svs::runtime::v0::StorageKind::LVQ4x4 + ); + auto leanvec_status = svs::runtime::v0::IVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LeanVec4x4 + ); + // Just verify the calls don't crash - actual support depends on build flags + (void)lvq_status; + (void)leanvec_status; +} + +CATCH_TEST_CASE("IVFIndexBuildAndSearchLVQ", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexBuildAndSearchLVQ..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LVQ4x4, + test_n, + test_data.data(), + build_params + ); + + if (!svs::runtime::v0::IVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LVQ4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LVQ storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexWriteAndReadLVQ", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexWriteAndReadLVQ..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LVQ4x4, + test_n, + test_data.data(), + build_params + ); + + if (!svs::runtime::v0::IVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LVQ4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LVQ storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "ivf_lvq_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::IVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::IVFIndex::load( + &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::LVQ4x4 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Search loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); + svs::runtime::v0::IVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearchLVQ", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexBuildAndSearchLVQ..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LVQ4x4, + test_n, + test_data.data(), + labels.data(), + build_params + ); + + if (!svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LVQ4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LVQ storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexBuildAndSearchLeanVec", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexBuildAndSearchLeanVec..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndexLeanVec::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LeanVec4x4, + test_n, + test_data.data(), + 32, // leanvec_dims + build_params + ); + + if (!svs::runtime::v0::IVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LeanVec4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LeanVec storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexWriteAndReadLeanVec", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexWriteAndReadLeanVec..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndexLeanVec::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LeanVec4x4, + test_n, + test_data.data(), + 32, // leanvec_dims + build_params + ); + + if (!svs::runtime::v0::IVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LeanVec4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LeanVec storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + svs_test::prepare_temp_directory(); + auto temp_dir = svs_test::temp_directory(); + auto filename = temp_dir / "ivf_leanvec_test.bin"; + + // Serialize + std::ofstream out(filename, std::ios::binary); + CATCH_REQUIRE(out.is_open()); + status = index->save(out); + CATCH_REQUIRE(status.ok()); + out.close(); + + // Deserialize + svs::runtime::v0::IVFIndex* loaded = nullptr; + std::ifstream in(filename, std::ios::binary); + CATCH_REQUIRE(in.is_open()); + status = svs::runtime::v0::IVFIndex::load( + &loaded, + in, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LeanVec4x4 + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(loaded != nullptr); + in.close(); + + // Search loaded index + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); + svs::runtime::v0::IVFIndex::destroy(loaded); +} + +CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearchLeanVec", "[runtime][ivf]") { + std::cout << "[IVF] Running DynamicIVFIndexBuildAndSearchLeanVec..." << std::endl; + const auto& test_data = get_test_data(); + + svs::runtime::v0::DynamicIVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndexLeanVec::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LeanVec4x4, + test_n, + test_data.data(), + labels.data(), + 32, // leanvec_dims + build_params + ); + + if (!svs::runtime::v0::DynamicIVFIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LeanVec4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("LeanVec storage kind is not supported in this build configuration."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicIVFIndex::destroy(index); +} + +CATCH_TEST_CASE("IVFIndexInnerProduct", "[runtime][ivf]") { + std::cout << "[IVF] Running IVFIndexInnerProduct..." << std::endl; + const auto& test_data = get_test_data(); + + // Build static IVF index with inner product metric + svs::runtime::v0::IVFIndex* index = nullptr; + svs::runtime::v0::IVFIndex::BuildParams build_params; + build_params.num_centroids = 10; + build_params.num_iterations = 5; + + svs::runtime::v0::Status status = svs::runtime::v0::IVFIndex::build( + &index, + test_d, + svs::runtime::v0::MetricType::INNER_PRODUCT, + svs::runtime::v0::StorageKind::FP32, + test_n, + test_data.data(), + build_params + ); + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Search + const int nq = 5; + const float* xq = test_data.data(); + const int k = 10; + + std::vector distances(nq * k); + std::vector result_labels(nq * k); + + status = index->search(nq, xq, k, distances.data(), result_labels.data()); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::IVFIndex::destroy(index); +} diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index fcded1006..093a6075f 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -17,9 +17,6 @@ #include "svs/runtime/api_defs.h" #include "svs/runtime/dynamic_vamana_index.h" #include "svs/runtime/flat_index.h" -#ifdef SVS_RUNTIME_ENABLE_IVF -#include "svs/runtime/ivf_index.h" -#endif #include "svs/runtime/training.h" #include "svs/runtime/vamana_index.h" @@ -80,7 +77,6 @@ struct UsageInfo { } // namespace -// Template function to write and read an index template void write_and_read_index( BuildFunc build_func, @@ -678,465 +674,3 @@ CATCH_TEST_CASE("SetIfSpecifiedUtility", "[runtime]") { CATCH_REQUIRE(target == false); } } - -// -// IVF Index Tests -// - -#ifdef SVS_RUNTIME_ENABLE_IVF -CATCH_TEST_CASE("StaticIVFIndexBuildAndSearch", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build static IVF index - svs::runtime::v0::StaticIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; // Small number for test data - build_params.num_iterations = 5; - - svs::runtime::v0::IVFIndex::SearchParams search_params; - search_params.n_probes = 3; - search_params.k_reorder = 1.0f; - - svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - build_params, - search_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - // Search - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = index->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - // Verify results are reasonable (at least some results found) - bool found_valid = false; - for (int i = 0; i < nq * k; ++i) { - if (result_labels[i] < test_n) { - found_valid = true; - break; - } - } - CATCH_REQUIRE(found_valid); - - svs::runtime::v0::StaticIVFIndex::destroy(index); -} - -CATCH_TEST_CASE("StaticIVFIndexWriteAndRead", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build static IVF index - svs::runtime::v0::StaticIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - build_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - svs_test::prepare_temp_directory(); - auto temp_dir = svs_test::temp_directory(); - auto filename = temp_dir / "static_ivf_test.bin"; - - // Serialize - std::ofstream out(filename, std::ios::binary); - CATCH_REQUIRE(out.is_open()); - status = index->save(out); - CATCH_REQUIRE(status.ok()); - out.close(); - - // Deserialize - svs::runtime::v0::StaticIVFIndex* loaded = nullptr; - std::ifstream in(filename, std::ios::binary); - CATCH_REQUIRE(in.is_open()); - status = svs::runtime::v0::StaticIVFIndex::load( - &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(loaded != nullptr); - in.close(); - - // Test search on loaded index - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - // Clean up - svs::runtime::v0::StaticIVFIndex::destroy(index); - svs::runtime::v0::StaticIVFIndex::destroy(loaded); -} - -CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearch", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build dynamic IVF index with initial data - svs::runtime::v0::DynamicIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - svs::runtime::v0::IVFIndex::SearchParams search_params; - search_params.n_probes = 3; - - std::vector labels(test_n); - std::iota(labels.begin(), labels.end(), 0); - - svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - labels.data(), - build_params, - search_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - // Search - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = index->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - svs::runtime::v0::DynamicIVFIndex::destroy(index); -} - -CATCH_TEST_CASE("DynamicIVFIndexAddAndRemove", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build empty dynamic IVF index first, then add data - svs::runtime::v0::DynamicIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - // Create with initial data (needed for clustering) - std::vector labels(test_n); - std::iota(labels.begin(), labels.end(), 0); - - svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - labels.data(), - build_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - // Check has_id for existing IDs - bool exists = false; - status = index->has_id(&exists, 0); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(exists); - - status = index->has_id(&exists, test_n - 1); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(exists); - - // Check has_id for non-existing ID - status = index->has_id(&exists, test_n + 100); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(!exists); - - // Remove some IDs - std::vector ids_to_remove = {0, 1, 2}; - status = index->remove(ids_to_remove.size(), ids_to_remove.data()); - CATCH_REQUIRE(status.ok()); - - // Verify removed IDs no longer exist - status = index->has_id(&exists, 0); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(!exists); - - // Consolidate and compact - status = index->consolidate(); - CATCH_REQUIRE(status.ok()); - - status = index->compact(); - CATCH_REQUIRE(status.ok()); - - // Search should still work - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = index->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - svs::runtime::v0::DynamicIVFIndex::destroy(index); -} - -CATCH_TEST_CASE("DynamicIVFIndexWriteAndRead", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build dynamic IVF index - svs::runtime::v0::DynamicIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - std::vector labels(test_n); - std::iota(labels.begin(), labels.end(), 0); - - svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - labels.data(), - build_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - svs_test::prepare_temp_directory(); - auto temp_dir = svs_test::temp_directory(); - auto filename = temp_dir / "dynamic_ivf_test.bin"; - - // Serialize - std::ofstream out(filename, std::ios::binary); - CATCH_REQUIRE(out.is_open()); - status = index->save(out); - CATCH_REQUIRE(status.ok()); - out.close(); - - // Deserialize - svs::runtime::v0::DynamicIVFIndex* loaded = nullptr; - std::ifstream in(filename, std::ios::binary); - CATCH_REQUIRE(in.is_open()); - status = svs::runtime::v0::DynamicIVFIndex::load( - &loaded, in, svs::runtime::v0::MetricType::L2, svs::runtime::v0::StorageKind::FP32 - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(loaded != nullptr); - in.close(); - - // Test search on loaded index - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = loaded->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - // Clean up - svs::runtime::v0::DynamicIVFIndex::destroy(index); - svs::runtime::v0::DynamicIVFIndex::destroy(loaded); -} - -CATCH_TEST_CASE("DynamicIVFIndexRemoveSelected", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build dynamic IVF index - svs::runtime::v0::DynamicIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - std::vector labels(test_n); - std::iota(labels.begin(), labels.end(), 0); - - svs::runtime::v0::Status status = svs::runtime::v0::DynamicIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - labels.data(), - build_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - // Remove IDs in range [0, 20) using selector - size_t min_id = 0; - size_t max_id = 20; - test_utils::IDFilterRange selector(min_id, max_id); - - size_t num_removed = 0; - status = index->remove_selected(&num_removed, selector); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(num_removed == max_id - min_id); - - // Verify removed IDs no longer exist - bool exists = false; - for (size_t i = min_id; i < max_id; ++i) { - status = index->has_id(&exists, i); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(!exists); - } - - // Verify IDs outside range still exist - status = index->has_id(&exists, max_id); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(exists); - - svs::runtime::v0::DynamicIVFIndex::destroy(index); -} - -CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build static IVF index - svs::runtime::v0::StaticIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - svs::runtime::v0::IVFIndex::SearchParams default_search_params; - default_search_params.n_probes = 2; - - svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - build_params, - default_search_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances1(nq * k); - std::vector result_labels1(nq * k); - - // Search with default params - status = index->search(nq, xq, k, distances1.data(), result_labels1.data()); - CATCH_REQUIRE(status.ok()); - - // Search with custom params (more probes should potentially give better results) - svs::runtime::v0::IVFIndex::SearchParams custom_params; - custom_params.n_probes = 5; - custom_params.k_reorder = 2.0f; - - std::vector distances2(nq * k); - std::vector result_labels2(nq * k); - - status = - index->search(nq, xq, k, distances2.data(), result_labels2.data(), &custom_params); - CATCH_REQUIRE(status.ok()); - - svs::runtime::v0::StaticIVFIndex::destroy(index); -} - -CATCH_TEST_CASE("IVFIndexCheckStorageKind", "[runtime][ivf]") { - // FP32 should be supported - CATCH_REQUIRE(svs::runtime::v0::StaticIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::FP32 - ) - .ok()); - CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::FP32 - ) - .ok()); - - // FP16 should be supported - CATCH_REQUIRE(svs::runtime::v0::StaticIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::FP16 - ) - .ok()); - CATCH_REQUIRE(svs::runtime::v0::DynamicIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::FP16 - ) - .ok()); - - // LVQ storage kinds should not be supported for IVF - CATCH_REQUIRE(!svs::runtime::v0::StaticIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::LVQ4x4 - ) - .ok()); - CATCH_REQUIRE(!svs::runtime::v0::DynamicIVFIndex::check_storage_kind( - svs::runtime::v0::StorageKind::LVQ4x4 - ) - .ok()); -} - -CATCH_TEST_CASE("StaticIVFIndexInnerProduct", "[runtime][ivf]") { - const auto& test_data = get_test_data(); - - // Build static IVF index with inner product metric - svs::runtime::v0::StaticIVFIndex* index = nullptr; - svs::runtime::v0::IVFIndex::BuildParams build_params; - build_params.num_centroids = 10; - build_params.num_iterations = 5; - - svs::runtime::v0::Status status = svs::runtime::v0::StaticIVFIndex::build( - &index, - test_d, - svs::runtime::v0::MetricType::INNER_PRODUCT, - svs::runtime::v0::StorageKind::FP32, - test_n, - test_data.data(), - build_params - ); - CATCH_REQUIRE(status.ok()); - CATCH_REQUIRE(index != nullptr); - - // Search - const int nq = 5; - const float* xq = test_data.data(); - const int k = 10; - - std::vector distances(nq * k); - std::vector result_labels(nq * k); - - status = index->search(nq, xq, k, distances.data(), result_labels.data()); - CATCH_REQUIRE(status.ok()); - - svs::runtime::v0::StaticIVFIndex::destroy(index); -} -#endif // SVS_RUNTIME_ENABLE_IVF diff --git a/docker/x86_64/build-cpp-runtime-bindings.sh b/docker/x86_64/build-cpp-runtime-bindings.sh index 1597382f1..7b37dfc57 100644 --- a/docker/x86_64/build-cpp-runtime-bindings.sh +++ b/docker/x86_64/build-cpp-runtime-bindings.sh @@ -31,6 +31,7 @@ CMAKE_ARGS=( "-DCMAKE_INSTALL_PREFIX=/workspace/install_cpp_bindings" "-DCMAKE_INSTALL_LIBDIR=lib" "-DSVS_RUNTIME_ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON}" + "-DSVS_RUNTIME_ENABLE_IVF=${ENABLE_IVF:-OFF}" ) if [ -n "$SVS_URL" ]; then @@ -45,7 +46,7 @@ cmake --install . # Build conda package for cpp runtime bindings source /opt/conda/etc/profile.d/conda.sh cd /workspace -ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld +ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} ENABLE_IVF=${ENABLE_IVF:-OFF} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld # Create tarball with symlink for compatibility cd /workspace/install_cpp_bindings && \ From 423193c09fe2827beac15a2c0bdd2046c6344b7a Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 11 Feb 2026 09:18:47 -0800 Subject: [PATCH 03/12] Install MKL in Docker flows for IVF --- .github/scripts/build-cpp-runtime-bindings.sh | 13 ++++++++++++- .github/workflows/build-cpp-runtime-bindings.yml | 5 ++++- docker/x86_64/manylinux228/Dockerfile | 5 +++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/scripts/build-cpp-runtime-bindings.sh b/.github/scripts/build-cpp-runtime-bindings.sh index 7b37dfc57..cfdaa79f0 100644 --- a/.github/scripts/build-cpp-runtime-bindings.sh +++ b/.github/scripts/build-cpp-runtime-bindings.sh @@ -15,9 +15,20 @@ set -e # Exit on error -# Source environment setup (for compiler and MKL) +# Source environment setup (for compiler) source /etc/bashrc || true +# Source MKL environment if IVF is enabled +if [ "${ENABLE_IVF:-OFF}" = "ON" ]; then + if [ -f /opt/intel/oneapi/setvars.sh ]; then + source /opt/intel/oneapi/setvars.sh --include-intel-llvm 2>/dev/null || true + echo "MKL sourced: MKLROOT=${MKLROOT}" + else + echo "ERROR: IVF enabled but MKL setvars.sh not found" + exit 1 + fi +fi + # Create build+install directories for cpp runtime bindings rm -rf /workspace/bindings/cpp/build_cpp_bindings /workspace/install_cpp_bindings mkdir -p /workspace/bindings/cpp/build_cpp_bindings /workspace/install_cpp_bindings diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index 7c05137db..66a4f5b7b 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -87,8 +87,11 @@ jobs: docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ + -e ENABLE_IVF=${{ matrix.enable_ivf }} \ svs-manylinux228:latest \ - /bin/bash -c "source /etc/bashrc || true && ctest --test-dir bindings/cpp/build_cpp_bindings/tests --output-on-failure --no-tests=error --verbose" + /bin/bash -c "source /etc/bashrc || true && \ + if [ \"\${ENABLE_IVF}\" = 'ON' ] && [ -f /opt/intel/oneapi/setvars.sh ]; then source /opt/intel/oneapi/setvars.sh 2>/dev/null || true; fi && \ + ctest --test-dir bindings/cpp/build_cpp_bindings/tests --output-on-failure --no-tests=error --verbose" # Run full test script using the built artifacts test: diff --git a/docker/x86_64/manylinux228/Dockerfile b/docker/x86_64/manylinux228/Dockerfile index 627327156..0818982e2 100644 --- a/docker/x86_64/manylinux228/Dockerfile +++ b/docker/x86_64/manylinux228/Dockerfile @@ -29,6 +29,11 @@ RUN echo '# Configure gcc-11' > /etc/profile.d/01-gcc.sh && \ echo 'source scl_source enable gcc-toolset-11' >> /etc/profile.d/01-gcc.sh && \ chmod +x /etc/profile.d/01-gcc.sh +# Download and Install MKL 2025.3 (needed for IVF support) +RUN wget -nv https://registrationcenter-download.intel.com/akdlm/IRC_NAS/6caa93ca-e10a-4cc5-b210-68f385feea9e/intel-oneapi-base-toolkit-2025.3.1.36.sh -O /tmp/oneapi-installer.sh && \ + bash /tmp/oneapi-installer.sh -a --components intel.oneapi.lin.mkl.devel --action install --eula accept -s --install-dir /opt/intel/oneapi && \ + rm -f /tmp/oneapi-installer.sh + # Download and install Miniforge RUN wget -nv https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh -O /tmp/miniforge.sh && \ bash /tmp/miniforge.sh -b -p /opt/conda && \ From 9e6bd90613c20d9c29c5529a06563269ce26ded7 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 11 Feb 2026 11:04:24 -0800 Subject: [PATCH 04/12] Fix a sporadically failing ivf save/load test --- tests/svs/index/ivf/index.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/svs/index/ivf/index.cpp b/tests/svs/index/ivf/index.cpp index 6724dc3b3..7c78a3add 100644 --- a/tests/svs/index/ivf/index.cpp +++ b/tests/svs/index/ivf/index.cpp @@ -182,7 +182,8 @@ CATCH_TEST_CASE("IVF Index Save and Load", "[ivf][index][saveload]") { size_t num_clusters = 10; size_t num_threads = 2; - size_t num_inner_threads = 2; + // Use 1 inner thread to avoid non-deterministic tie-breaking in search results. + size_t num_inner_threads = 1; auto distance = svs::distance::DistanceL2(); // Build clustering From de67083f6c1b07b5c81d70cac45f5feeb51f8c50 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Wed, 11 Feb 2026 15:59:52 -0800 Subject: [PATCH 05/12] Support ivf runtime in conda flows --- bindings/cpp/conda-recipe/meta.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bindings/cpp/conda-recipe/meta.yaml b/bindings/cpp/conda-recipe/meta.yaml index a62f56033..8f7230124 100644 --- a/bindings/cpp/conda-recipe/meta.yaml +++ b/bindings/cpp/conda-recipe/meta.yaml @@ -29,6 +29,11 @@ build: string: {{ GIT_DESCRIBE_HASH }}_{{ buildnumber }}{{ variant_suffix }} skip: true # [not linux] include_recipe: False + script_env: + - ENABLE_IVF + - ENABLE_LVQ_LEANVEC + - SUFFIX + - SVS_URL requirements: build: From 032023ec000e10fc9f7eb219c1bac15f3adb2000 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 12 Feb 2026 06:19:27 -0800 Subject: [PATCH 06/12] Use my faiss fork branch for runtime testing --- .github/scripts/test-cpp-runtime-bindings.sh | 2 +- .github/workflows/build-cpp-runtime-bindings.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test-cpp-runtime-bindings.sh b/.github/scripts/test-cpp-runtime-bindings.sh index 0209b3c71..91ce533dc 100644 --- a/.github/scripts/test-cpp-runtime-bindings.sh +++ b/.github/scripts/test-cpp-runtime-bindings.sh @@ -35,7 +35,7 @@ conda install -y mkl=2022.2.1 mkl-devel=2022.2.1 conda install -y /runtime_conda/libsvs-runtime-*.conda # Validate python and C++ tests against FAISS CI -git clone https://github.com/facebookresearch/faiss.git +git clone --branch ib/svs_ivf https://github.com/ibhati/faiss.git cd faiss echo "===============================================" diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index 66a4f5b7b..2899406cd 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -105,6 +105,8 @@ jobs: suffix: "" - name: "public only" suffix: "-public-only" + - name: "IVF public only" + suffix: "-ivf" steps: - uses: actions/checkout@v6 From 03d7d941459681c6178aa727c7b1ad7bf077afbe Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 12 Feb 2026 08:41:28 -0800 Subject: [PATCH 07/12] Use faiss fork for IVF only --- .github/scripts/test-cpp-runtime-bindings.sh | 9 ++++++++- .github/workflows/build-cpp-runtime-bindings.yml | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/scripts/test-cpp-runtime-bindings.sh b/.github/scripts/test-cpp-runtime-bindings.sh index 91ce533dc..e3b47ce52 100644 --- a/.github/scripts/test-cpp-runtime-bindings.sh +++ b/.github/scripts/test-cpp-runtime-bindings.sh @@ -35,7 +35,14 @@ conda install -y mkl=2022.2.1 mkl-devel=2022.2.1 conda install -y /runtime_conda/libsvs-runtime-*.conda # Validate python and C++ tests against FAISS CI -git clone --branch ib/svs_ivf https://github.com/ibhati/faiss.git +ENABLE_IVF="${ENABLE_IVF:-OFF}" +if [ "${ENABLE_IVF}" = "ON" ]; then + echo "IVF enabled: cloning forked FAISS with IVF support" + git clone --branch ib/svs_ivf https://github.com/ibhati/faiss.git +else + echo "IVF disabled: cloning upstream FAISS" + git clone https://github.com/facebookresearch/faiss.git +fi cd faiss echo "===============================================" diff --git a/.github/workflows/build-cpp-runtime-bindings.yml b/.github/workflows/build-cpp-runtime-bindings.yml index 2899406cd..9d8ba9f3d 100644 --- a/.github/workflows/build-cpp-runtime-bindings.yml +++ b/.github/workflows/build-cpp-runtime-bindings.yml @@ -103,10 +103,13 @@ jobs: include: - name: "with static library" suffix: "" + enable_ivf: "OFF" - name: "public only" suffix: "-public-only" + enable_ivf: "OFF" - name: "IVF public only" suffix: "-ivf" + enable_ivf: "ON" steps: - uses: actions/checkout@v6 @@ -133,5 +136,6 @@ jobs: -v ${{ github.workspace }}/runtime_conda:/runtime_conda \ -w /workspace \ -e SUFFIX=${{ matrix.suffix }} \ + -e ENABLE_IVF=${{ matrix.enable_ivf }} \ svs-manylinux228:latest \ /bin/bash .github/scripts/test-cpp-runtime-bindings.sh From e8df9128ccdb76d67bf77196e6a91c4d56d3e698 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 12 Feb 2026 11:11:05 -0800 Subject: [PATCH 08/12] bug fix --- bindings/cpp/src/dynamic_ivf_index_impl.h | 3 +- bindings/cpp/src/ivf_index_impl.h | 3 +- bindings/cpp/tests/ivf_runtime_test.cpp | 106 +++++++++++++++++++--- 3 files changed, 97 insertions(+), 15 deletions(-) diff --git a/bindings/cpp/src/dynamic_ivf_index_impl.h b/bindings/cpp/src/dynamic_ivf_index_impl.h index 8bbd15c2a..5d83884a2 100644 --- a/bindings/cpp/src/dynamic_ivf_index_impl.h +++ b/bindings/cpp/src/dynamic_ivf_index_impl.h @@ -143,8 +143,7 @@ class DynamicIVFIndexImpl { } auto sp = make_search_parameters(params); - impl_->set_search_parameters(sp); - impl_->search(result, queries, {}); + impl_->search(result, queries, sp); } void save(std::ostream& out) const { diff --git a/bindings/cpp/src/ivf_index_impl.h b/bindings/cpp/src/ivf_index_impl.h index 0aeaa3774..2e3417b01 100644 --- a/bindings/cpp/src/ivf_index_impl.h +++ b/bindings/cpp/src/ivf_index_impl.h @@ -444,8 +444,7 @@ class IVFIndexImpl { } auto sp = make_search_parameters(params); - impl_->set_search_parameters(sp); - impl_->search(result, queries, {}); + impl_->search(result, queries, sp); } void save(std::ostream& out) const { diff --git a/bindings/cpp/tests/ivf_runtime_test.cpp b/bindings/cpp/tests/ivf_runtime_test.cpp index 692f1d404..2d3b45c12 100644 --- a/bindings/cpp/tests/ivf_runtime_test.cpp +++ b/bindings/cpp/tests/ivf_runtime_test.cpp @@ -50,6 +50,29 @@ inline const std::vector& get_test_data() { return test_data; } +// Compute recall@k: fraction of ground-truth neighbors found in the result set, +// averaged over all queries. +double compute_recall( + const std::vector& result_labels, + const std::vector& gt_labels, + size_t nq, + size_t k +) { + size_t total_found = 0; + for (size_t q = 0; q < nq; ++q) { + for (size_t i = 0; i < k; ++i) { + size_t gt_id = gt_labels[q * k + i]; + for (size_t j = 0; j < k; ++j) { + if (result_labels[q * k + j] == gt_id) { + ++total_found; + break; + } + } + } + } + return static_cast(total_found) / static_cast(nq * k); +} + } // namespace CATCH_TEST_CASE("IVFIndexBuildAndSearch", "[runtime][ivf]") { @@ -199,6 +222,44 @@ CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearch", "[runtime][ivf]") { status = index->search(nq, xq, k, distances.data(), result_labels.data()); CATCH_REQUIRE(status.ok()); + // Verify that increasing n_probes improves recall for dynamic IVF + // Ground truth: search with all centroids probed + svs::runtime::v0::IVFIndex::SearchParams exhaustive_params; + exhaustive_params.n_probes = 10; + std::vector gt_distances(nq * k); + std::vector gt_labels(nq * k); + status = index->search( + nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params + ); + CATCH_REQUIRE(status.ok()); + + // Low n_probes + svs::runtime::v0::IVFIndex::SearchParams low_params; + low_params.n_probes = 1; + std::vector dist_low(nq * k); + std::vector labels_low(nq * k); + status = index->search(nq, xq, k, dist_low.data(), labels_low.data(), &low_params); + CATCH_REQUIRE(status.ok()); + + // High n_probes + svs::runtime::v0::IVFIndex::SearchParams high_params; + high_params.n_probes = 5; + std::vector dist_high(nq * k); + std::vector labels_high(nq * k); + status = index->search(nq, xq, k, dist_high.data(), labels_high.data(), &high_params); + CATCH_REQUIRE(status.ok()); + + double recall_low = compute_recall(labels_low, gt_labels, nq, k); + double recall_high = compute_recall(labels_high, gt_labels, nq, k); + + std::cout << " [Dynamic] recall@" << k << " with n_probes=1: " << recall_low + << std::endl; + std::cout << " [Dynamic] recall@" << k << " with n_probes=5: " << recall_high + << std::endl; + + CATCH_REQUIRE(recall_high >= recall_low); + CATCH_REQUIRE(recall_high > 0.0); + svs::runtime::v0::DynamicIVFIndex::destroy(index); } @@ -421,25 +482,48 @@ CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { const float* xq = test_data.data(); const int k = 10; - std::vector distances1(nq * k); - std::vector result_labels1(nq * k); + // Step 1: Get ground-truth by searching with all centroids probed + svs::runtime::v0::IVFIndex::SearchParams exhaustive_params; + exhaustive_params.n_probes = 10; // all centroids - // Search with default params - status = index->search(nq, xq, k, distances1.data(), result_labels1.data()); + std::vector gt_distances(nq * k); + std::vector gt_labels(nq * k); + status = index->search( + nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params + ); CATCH_REQUIRE(status.ok()); - // Search with custom params (more probes should potentially give better results) - svs::runtime::v0::IVFIndex::SearchParams custom_params; - custom_params.n_probes = 5; - custom_params.k_reorder = 2.0f; + // Step 2: Search with low n_probes + svs::runtime::v0::IVFIndex::SearchParams low_params; + low_params.n_probes = 1; + + std::vector distances_low(nq * k); + std::vector labels_low(nq * k); + status = + index->search(nq, xq, k, distances_low.data(), labels_low.data(), &low_params); + CATCH_REQUIRE(status.ok()); - std::vector distances2(nq * k); - std::vector result_labels2(nq * k); + // Step 3: Search with high n_probes + svs::runtime::v0::IVFIndex::SearchParams high_params; + high_params.n_probes = 5; + std::vector distances_high(nq * k); + std::vector labels_high(nq * k); status = - index->search(nq, xq, k, distances2.data(), result_labels2.data(), &custom_params); + index->search(nq, xq, k, distances_high.data(), labels_high.data(), &high_params); CATCH_REQUIRE(status.ok()); + // Step 4: Compute recall for both and verify higher n_probes gives >= recall + double recall_low = compute_recall(labels_low, gt_labels, nq, k); + double recall_high = compute_recall(labels_high, gt_labels, nq, k); + + std::cout << " recall@" << k << " with n_probes=1: " << recall_low << std::endl; + std::cout << " recall@" << k << " with n_probes=5: " << recall_high << std::endl; + + CATCH_REQUIRE(recall_high >= recall_low); + // With 5 out of 10 centroids probed, recall should be reasonably high + CATCH_REQUIRE(recall_high > 0.0); + svs::runtime::v0::IVFIndex::destroy(index); } From b8e4bbfac335dcad06661cb1d57b67942881f27e Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Thu, 12 Feb 2026 14:13:12 -0800 Subject: [PATCH 09/12] conda path --- bindings/cpp/conda-recipe/build.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bindings/cpp/conda-recipe/build.sh b/bindings/cpp/conda-recipe/build.sh index f3ce6e295..0dfe71ec6 100644 --- a/bindings/cpp/conda-recipe/build.sh +++ b/bindings/cpp/conda-recipe/build.sh @@ -25,6 +25,17 @@ else echo "WARNING: gcc-toolset-11 not found, proceeding without sourcing it" fi +# Source MKL environment if IVF is enabled (IVF requires MKL) +if [ "${ENABLE_IVF:-OFF}" = "ON" ]; then + if [ -f /opt/intel/oneapi/setvars.sh ]; then + source /opt/intel/oneapi/setvars.sh --include-intel-llvm 2>/dev/null || true + echo "MKL sourced for IVF build: MKLROOT=${MKLROOT}" + else + echo "ERROR: IVF enabled but MKL setvars.sh not found" + exit 1 + fi +fi + # build runtime tests flag? CMAKE_ARGS=( "-DCMAKE_INSTALL_PREFIX=${PREFIX}" From e07f97cce26d1c294c65d30f2da9f13a43e2e663 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Fri, 13 Feb 2026 09:32:07 -0800 Subject: [PATCH 10/12] fix CI and threading --- .github/scripts/build-cpp-runtime-bindings.sh | 4 +++- bindings/cpp/src/dynamic_ivf_index_impl.h | 4 ++-- bindings/cpp/src/ivf_index_impl.h | 4 ++-- include/svs/index/ivf/dynamic_ivf.h | 4 ++-- include/svs/index/ivf/index.h | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/scripts/build-cpp-runtime-bindings.sh b/.github/scripts/build-cpp-runtime-bindings.sh index cfdaa79f0..b2ebb30bb 100644 --- a/.github/scripts/build-cpp-runtime-bindings.sh +++ b/.github/scripts/build-cpp-runtime-bindings.sh @@ -57,7 +57,9 @@ cmake --install . # Build conda package for cpp runtime bindings source /opt/conda/etc/profile.d/conda.sh cd /workspace -ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} ENABLE_IVF=${ENABLE_IVF:-OFF} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld +# Use /workspace for temp files to avoid filling up /tmp during LTO compilation +mkdir -p /workspace/tmp +TMPDIR=/workspace/tmp ENABLE_LVQ_LEANVEC=${ENABLE_LVQ_LEANVEC:-ON} ENABLE_IVF=${ENABLE_IVF:-OFF} SVS_URL="${SVS_URL}" SUFFIX="${SUFFIX}" conda build bindings/cpp/conda-recipe --output-folder /workspace/conda-bld # Create tarball with symlink for compatibility cd /workspace/install_cpp_bindings && \ diff --git a/bindings/cpp/src/dynamic_ivf_index_impl.h b/bindings/cpp/src/dynamic_ivf_index_impl.h index 5d83884a2..f7ced39b6 100644 --- a/bindings/cpp/src/dynamic_ivf_index_impl.h +++ b/bindings/cpp/src/dynamic_ivf_index_impl.h @@ -268,7 +268,7 @@ class DynamicIVFIndexImpl { void init_impl(data::ConstSimpleDataView data, std::span ids) { auto build_params = ivf_build_parameters(); - auto threadpool = default_threadpool(); + auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); @@ -421,7 +421,7 @@ class DynamicIVFIndexLeanVecImpl : public DynamicIVFIndexImpl { void init_leanvec_impl(data::ConstSimpleDataView data, std::span ids) { auto build_params = ivf_build_parameters(); - auto threadpool = default_threadpool(); + auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); diff --git a/bindings/cpp/src/ivf_index_impl.h b/bindings/cpp/src/ivf_index_impl.h index 2e3417b01..2e5ea73eb 100644 --- a/bindings/cpp/src/ivf_index_impl.h +++ b/bindings/cpp/src/ivf_index_impl.h @@ -568,7 +568,7 @@ class IVFIndexImpl { void init_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - auto threadpool = default_threadpool(); + auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); @@ -718,7 +718,7 @@ class IVFIndexLeanVecImpl : public IVFIndexImpl { void init_leanvec_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - auto threadpool = default_threadpool(); + auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); diff --git a/include/svs/index/ivf/dynamic_ivf.h b/include/svs/index/ivf/dynamic_ivf.h index 9aefbfcc4..366c511f6 100644 --- a/include/svs/index/ivf/dynamic_ivf.h +++ b/include/svs/index/ivf/dynamic_ivf.h @@ -81,7 +81,7 @@ class DynamicIVFIndex { // Thread-related type aliases using InterQueryThreadPool = threads::ThreadPoolHandle; - using IntraQueryThreadPool = threads::DefaultThreadPool; + using IntraQueryThreadPool = threads::ThreadPoolHandle; // Reuse scratchspace types from static IVF using buffer_centroids_type = SortedBuffer; @@ -775,7 +775,7 @@ class DynamicIVFIndex { void initialize_thread_pools() { for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { intra_query_threadpools_.push_back( - threads::as_threadpool(intra_query_thread_count_) + threads::ThreadPoolHandle(threads::DefaultThreadPool(intra_query_thread_count_)) ); } } diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h index dc983921d..2775ff777 100644 --- a/include/svs/index/ivf/index.h +++ b/include/svs/index/ivf/index.h @@ -116,7 +116,7 @@ class IVFIndex { // Thread-related type aliases for clarity using InterQueryThreadPool = threads::ThreadPoolHandle; // For inter-query parallelism - using IntraQueryThreadPool = threads::DefaultThreadPool; // For intra-query parallelism + using IntraQueryThreadPool = threads::ThreadPoolHandle; // For intra-query parallelism // Scratchspace type for external threading using buffer_centroids_type = SortedBuffer>; @@ -548,7 +548,7 @@ class IVFIndex { // Create thread pools for intra-query (cluster-level) parallelism for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { intra_query_threadpools_.push_back( - threads::as_threadpool(intra_query_thread_count_) + threads::ThreadPoolHandle(threads::DefaultThreadPool(intra_query_thread_count_)) ); } } From 31ef81051ce26b2a756756ddd676794b95f861ee Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Fri, 13 Feb 2026 09:34:20 -0800 Subject: [PATCH 11/12] format --- include/svs/index/ivf/dynamic_ivf.h | 6 +++--- include/svs/index/ivf/index.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/svs/index/ivf/dynamic_ivf.h b/include/svs/index/ivf/dynamic_ivf.h index 366c511f6..73bba6b82 100644 --- a/include/svs/index/ivf/dynamic_ivf.h +++ b/include/svs/index/ivf/dynamic_ivf.h @@ -774,9 +774,9 @@ class DynamicIVFIndex { void initialize_thread_pools() { for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { - intra_query_threadpools_.push_back( - threads::ThreadPoolHandle(threads::DefaultThreadPool(intra_query_thread_count_)) - ); + intra_query_threadpools_.push_back(threads::ThreadPoolHandle( + threads::DefaultThreadPool(intra_query_thread_count_) + )); } } diff --git a/include/svs/index/ivf/index.h b/include/svs/index/ivf/index.h index 2775ff777..4a0bbdd6e 100644 --- a/include/svs/index/ivf/index.h +++ b/include/svs/index/ivf/index.h @@ -115,7 +115,7 @@ class IVFIndex { using search_parameters_type = IVFSearchParameters; // Thread-related type aliases for clarity - using InterQueryThreadPool = threads::ThreadPoolHandle; // For inter-query parallelism + using InterQueryThreadPool = threads::ThreadPoolHandle; // For inter-query parallelism using IntraQueryThreadPool = threads::ThreadPoolHandle; // For intra-query parallelism // Scratchspace type for external threading @@ -547,9 +547,9 @@ class IVFIndex { void initialize_thread_pools() { // Create thread pools for intra-query (cluster-level) parallelism for (size_t i = 0; i < inter_query_threadpool_.size(); i++) { - intra_query_threadpools_.push_back( - threads::ThreadPoolHandle(threads::DefaultThreadPool(intra_query_thread_count_)) - ); + intra_query_threadpools_.push_back(threads::ThreadPoolHandle( + threads::DefaultThreadPool(intra_query_thread_count_) + )); } } From c01af3d3b53cebf859d808e2b0643312ff52bc13 Mon Sep 17 00:00:00 2001 From: Ishwar Bhati Date: Fri, 13 Feb 2026 09:39:17 -0800 Subject: [PATCH 12/12] format --- bindings/cpp/src/dynamic_ivf_index_impl.h | 6 ++++-- bindings/cpp/src/ivf_index_impl.h | 6 ++++-- bindings/cpp/tests/ivf_runtime_test.cpp | 13 +++++-------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/bindings/cpp/src/dynamic_ivf_index_impl.h b/bindings/cpp/src/dynamic_ivf_index_impl.h index f7ced39b6..b0d01916f 100644 --- a/bindings/cpp/src/dynamic_ivf_index_impl.h +++ b/bindings/cpp/src/dynamic_ivf_index_impl.h @@ -268,7 +268,8 @@ class DynamicIVFIndexImpl { void init_impl(data::ConstSimpleDataView data, std::span ids) { auto build_params = ivf_build_parameters(); - auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); + auto threadpool = + svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); @@ -421,7 +422,8 @@ class DynamicIVFIndexLeanVecImpl : public DynamicIVFIndexImpl { void init_leanvec_impl(data::ConstSimpleDataView data, std::span ids) { auto build_params = ivf_build_parameters(); - auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); + auto threadpool = + svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); diff --git a/bindings/cpp/src/ivf_index_impl.h b/bindings/cpp/src/ivf_index_impl.h index 2e5ea73eb..5e6de1299 100644 --- a/bindings/cpp/src/ivf_index_impl.h +++ b/bindings/cpp/src/ivf_index_impl.h @@ -568,7 +568,8 @@ class IVFIndexImpl { void init_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); + auto threadpool = + svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); @@ -718,7 +719,8 @@ class IVFIndexLeanVecImpl : public IVFIndexImpl { void init_leanvec_impl(data::ConstSimpleDataView data) { auto build_params = ivf_build_parameters(); - auto threadpool = svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); + auto threadpool = + svs::threads::ThreadPoolHandle(svs::threads::DefaultThreadPool(num_threads_)); svs::DistanceDispatcher distance_dispatcher(to_svs_distance(metric_type_)); diff --git a/bindings/cpp/tests/ivf_runtime_test.cpp b/bindings/cpp/tests/ivf_runtime_test.cpp index 2d3b45c12..ad6a275d1 100644 --- a/bindings/cpp/tests/ivf_runtime_test.cpp +++ b/bindings/cpp/tests/ivf_runtime_test.cpp @@ -228,9 +228,8 @@ CATCH_TEST_CASE("DynamicIVFIndexBuildAndSearch", "[runtime][ivf]") { exhaustive_params.n_probes = 10; std::vector gt_distances(nq * k); std::vector gt_labels(nq * k); - status = index->search( - nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params - ); + status = + index->search(nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params); CATCH_REQUIRE(status.ok()); // Low n_probes @@ -488,9 +487,8 @@ CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { std::vector gt_distances(nq * k); std::vector gt_labels(nq * k); - status = index->search( - nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params - ); + status = + index->search(nq, xq, k, gt_distances.data(), gt_labels.data(), &exhaustive_params); CATCH_REQUIRE(status.ok()); // Step 2: Search with low n_probes @@ -499,8 +497,7 @@ CATCH_TEST_CASE("IVFIndexSearchWithParams", "[runtime][ivf]") { std::vector distances_low(nq * k); std::vector labels_low(nq * k); - status = - index->search(nq, xq, k, distances_low.data(), labels_low.data(), &low_params); + status = index->search(nq, xq, k, distances_low.data(), labels_low.data(), &low_params); CATCH_REQUIRE(status.ok()); // Step 3: Search with high n_probes