Merge remote-tracking branch 'origin/main' into omer-add-save-load-check

Author: lerman25

cmake/svs.cmake

@@ -42,17 +42,34 @@ if(USE_SVS)
 
     # detect if build environment is using glibc
     include(CheckSymbolExists)
-    unset(GLIBC_FOUND CACHE)
     check_symbol_exists(__GLIBC__ "features.h" GLIBC_FOUND)
-    if(NOT GLIBC_FOUND)
-        message(STATUS "GLIBC is not detected - SVS shared library is not supported")
+    if(GLIBC_FOUND)
+        include(CheckCXXSourceRuns)
+        check_cxx_source_runs("#include <features.h>
+            int main(){ return __GLIBC__ == 2 && __GLIBC_MINOR__ >= 28 ?0:1; }"
+            GLIBC_2_28_FOUND)
+        check_cxx_source_runs("#include <features.h>
+            int main(){ return __GLIBC__ == 2 && __GLIBC_MINOR__ >= 26 ?0:1; }"
+            GLIBC_2_26_FOUND)
     endif()
 
     cmake_dependent_option(SVS_SHARED_LIB "Use SVS pre-compiled shared library" ON "USE_SVS AND GLIBC_FOUND AND SVS_LVQ_SUPPORTED" OFF)
     if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-        set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250629-clang.tar.gz" CACHE STRING "SVS URL")
+        if (GLIBC_2_28_FOUND)
+            set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250629-clang.tar.gz" CACHE STRING "SVS URL")
+        else()
+            message(STATUS "GLIBC>=2.28 is required for Clang build - disabling SVS_SHARED_LIB")
+            set(SVS_SHARED_LIB OFF)
+        endif()
     else()
-        set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250630.tar.gz" CACHE STRING "SVS URL")
+        if (GLIBC_2_28_FOUND)
+            set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250630.tar.gz" CACHE STRING "SVS URL")
+        elseif(GLIBC_2_26_FOUND)
+            set(SVS_URL "https://github.com/intel/ScalableVectorSearch/releases/download/v0.0.8-dev/svs-shared-library-0.0.8-NIGHTLY-20250701-glibc-2_26.tar.gz" CACHE STRING "SVS URL")
+        else()
+            message(STATUS "GLIBC>=2.26 is required for SVS shared library - disabling SVS_SHARED_LIB")
+            set(SVS_SHARED_LIB OFF)
+        endif()
     endif()
 
     if(SVS_SHARED_LIB)

src/VecSim/algorithms/svs/svs.h

@@ -83,8 +83,16 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
 
     // Index search parameters
     size_t search_window_size;
+    size_t search_buffer_capacity;
+    // LeanVec dataset dimension
+    // This parameter allows to tune LeanVec dimension if LeanVec is enabled
+    size_t leanvec_dim;
     double epsilon;
 
+    // Check if the dataset is Two-level LVQ
+    // This allows to tune default window capacity during search
+    bool is_two_level_lvq;
+
     // SVS thread pool
     VecSimSVSThreadPool threadpool_;
     svs::logging::logger_ptr logger_;
@@ -137,7 +145,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
 
         // Construct SVS index initial storage with compression if needed
         auto data = storage_traits_t::create_storage(points, this->blockSize, threadpool_handle,
-                                                     this->getAllocator());
+                                                     this->getAllocator(), this->leanvec_dim);
         // Compute the entry point.
         auto entry_point =
             svs::index::vamana::extensions::compute_entry_point(data, threadpool_handle);
@@ -164,7 +172,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
 
         // Configure default search parameters
         auto sp = impl_->get_search_parameters();
-        sp.buffer_config({this->search_window_size});
+        sp.buffer_config({this->search_window_size, this->search_buffer_capacity});
         impl_->set_search_parameters(sp);
         impl_->reset_performance_parameters();
     }
@@ -289,14 +297,31 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
         }
     }
 
+    bool isTwoLevelLVQ(const VecSimSvsQuantBits &qbits) {
+        switch (qbits) {
+        case VecSimSvsQuant_4x4:
+        case VecSimSvsQuant_4x8:
+        case VecSimSvsQuant_4x8_LeanVec:
+        case VecSimSvsQuant_8x8_LeanVec:
+            return true;
+        default:
+            return false;
+        }
+    }
+
 public:
     SVSIndex(const SVSParams &params, const AbstractIndexInitParams &abstractInitParams,
              const index_component_t &components, bool force_preprocessing)
         : Base{abstractInitParams, components}, forcePreprocessing{force_preprocessing},
           changes_num{0}, buildParams{svs_details::makeVamanaBuildParameters(params)},
           search_window_size{svs_details::getOrDefault(params.search_window_size,
                                                        SVS_VAMANA_DEFAULT_SEARCH_WINDOW_SIZE)},
+          search_buffer_capacity{
+              svs_details::getOrDefault(params.search_buffer_capacity, search_window_size)},
+          leanvec_dim{
+              svs_details::getOrDefault(params.leanvec_dim, SVS_VAMANA_DEFAULT_LEANVEC_DIM)},
           epsilon{svs_details::getOrDefault(params.epsilon, SVS_VAMANA_DEFAULT_EPSILON)},
+          is_two_level_lvq{isTwoLevelLVQ(params.quantBits)},
           threadpool_{std::max(size_t{SVS_VAMANA_DEFAULT_NUM_THREADS}, params.num_threads)},
           impl_{nullptr} {
         logger_ = makeLogger();
@@ -351,6 +376,8 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
                           .numThreads = this->getNumThreads(),
                           .numberOfMarkedDeletedNodes = this->changes_num,
                           .searchWindowSize = this->search_window_size,
+                          .searchBufferCapacity = this->search_buffer_capacity,
+                          .leanvecDim = this->leanvec_dim,
                           .epsilon = this->epsilon};
         return info;
     }
@@ -427,7 +454,8 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
         auto query = svs::data::ConstSimpleDataView<DataType>{
             static_cast<const DataType *>(processed_query), 1, this->dim};
         auto result = svs::QueryResult<size_t>{query.size(), k};
-        auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams);
+        auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams,
+                                                is_two_level_lvq);
 
         auto timeoutCtx = queryParams ? queryParams->timeoutCtx : nullptr;
         auto cancel = [timeoutCtx]() { return VECSIM_TIMEOUT(timeoutCtx); };
@@ -472,7 +500,8 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
                                          this->dim};
 
         // Base search parameters for the SVS iterator schedule.
-        auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams);
+        auto sp = svs_details::joinSearchParams(impl_->get_search_parameters(), queryParams,
+                                                is_two_level_lvq);
         // SVS BatchIterator handles the search in batches
         // The batch size is set to the index search window size by default
         const size_t batch_size = sp.buffer_config_.get_search_window_size();
@@ -535,7 +564,7 @@ class SVSIndex : public VecSimIndexAbstract<svs_details::vecsim_dt<DataType>, fl
                 NullSVS_BatchIterator(queryBlobCopyPtr, queryParams, this->getAllocator());
         } else {
             return new (this->getAllocator()) SVS_BatchIterator<impl_type, data_type>(
-                queryBlobCopyPtr, impl_.get(), queryParams, this->getAllocator());
+                queryBlobCopyPtr, impl_.get(), queryParams, this->getAllocator(), is_two_level_lvq);
         }
     }
 

src/VecSim/algorithms/svs/svs_batch_iterator.h

@@ -71,14 +71,15 @@ class SVS_BatchIterator : public VecSimBatchIterator {
 
 public:
     SVS_BatchIterator(void *query_vector, const Index *index, const VecSimQueryParams *queryParams,
-                      std::shared_ptr<VecSimAllocator> allocator)
+                      std::shared_ptr<VecSimAllocator> allocator, bool is_two_level_lvq)
         : VecSimBatchIterator{query_vector, queryParams ? queryParams->timeoutCtx : nullptr,
                               std::move(allocator)},
           done{false}, dim{index->dimensions()}, index_{index},
           impl_{std::make_unique<impl_type>(index->make_batch_iterator(
               std::span{static_cast<const DataType *>(query_vector), dim}))},
           curr_it{impl_->begin()} {
-        auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams);
+        auto sp = svs_details::joinSearchParams(index->get_search_parameters(), queryParams,
+                                                is_two_level_lvq);
         batch_size = queryParams && queryParams->batchSize
                          ? queryParams->batchSize
                          : sp.buffer_config_.get_search_window_size();