Generalize Micro-benchmarks [MOD-10160] (#703)

* add array and index

* add docs
fix "$BM_TYPE" = "bm-updated-fp32-single"

* unify Memory benchmarks function to take index type parameter: Replace Memory_FLAT/HNSW/Tiered with single Memory function taking IndexTypeIndex to reduce code duplication.

replace INDEX_TYPE_SVS_QUANTIZED with INDEX_TYPE_SVS_COMPRESSED

* refactor: improve index ownership management in benchmarks
- Replaced manual reference counting with `IndexPtr` for safer and simpler ownership management.
- Introduced `IndexPtr` as a smart pointer wrapper for `VecSimIndex` to:
  1. Ensure proper cleanup using RAII principles.
  2. Enable safe ownership transfer via `release_ownership()`.
  3. Support shared ownership with thread pool using `get_shared()`.

introducee GET_INDEX to access an index in the indices array

refactor: improve benchmark ownership and initialization logic

- Ensured one-time initialization using `is_initialized` flags.

* RunTopK_HNSW: remove unused is_tiered

rename IndexTypeFlags etries from INDEX_TYPE to INDEX_MASK

initialize threadpool mock only if needed

* initizlie bf after hnsw

* fix mockethread pool deletion

* introduce DEFAULT_BM_INDEXES_MASK that defines bf, hnsw and tierred

address CR

---------

Co-authored-by: meiravgri <[email protected]>

Author: dor-forer

tests/benchmark/bm_batch_iterator.h

@@ -44,8 +44,8 @@ void BM_BatchIterator<index_type_t>::RunBatchedSearch_HNSW(
     benchmark::State &st, std::atomic_int &correct, size_t iter, size_t num_batches,
     size_t batch_size, size_t &total_res_num, size_t batch_increase_factor, size_t index_memory,
     double &memory_delta) {
-    VecSimBatchIterator *batchIterator = VecSimBatchIterator_New(
-        INDICES.at(VecSimAlgo_HNSWLIB), QUERIES[iter % N_QUERIES].data(), nullptr);
+    VecSimBatchIterator *batchIterator =
+        VecSimBatchIterator_New(GET_INDEX(INDEX_HNSW), QUERIES[iter % N_QUERIES].data(), nullptr);
     VecSimQueryReply *accumulated_results[num_batches];
     size_t batch_num = 0;
     total_res_num = 0;
@@ -62,13 +62,13 @@ void BM_BatchIterator<index_type_t>::RunBatchedSearch_HNSW(
     }
     st.PauseTiming();
     // Update the memory delta as a result of using the batch iterator.
-    size_t curr_memory = VecSimIndex_StatsInfo(INDICES.at(VecSimAlgo_HNSWLIB)).memory;
+    size_t curr_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_HNSW)).memory;
     memory_delta += (double)(curr_memory - index_memory);
     VecSimBatchIterator_Free(batchIterator);
 
     // Measure recall - compare every result that was collected in some batch to the BF results.
-    auto bf_results = VecSimIndex_TopKQuery(
-        INDICES[VecSimAlgo_BF], QUERIES[iter % N_QUERIES].data(), total_res_num, nullptr, BY_SCORE);
+    auto bf_results = VecSimIndex_TopKQuery(GET_INDEX(INDEX_BF), QUERIES[iter % N_QUERIES].data(),
+                                            total_res_num, nullptr, BY_SCORE);
     for (size_t i = 0; i < batch_num; i++) {
         auto hnsw_results = accumulated_results[i];
         BM_VecSimGeneral::MeasureRecall(hnsw_results, bf_results, correct);
@@ -83,12 +83,12 @@ void BM_BatchIterator<index_type_t>::BF_FixedBatchSize(benchmark::State &st) {
     size_t batch_size = st.range(0);
     size_t num_batches = st.range(1);
     size_t iter = 0;
-    size_t index_memory = VecSimIndex_StatsInfo(INDICES[VecSimAlgo_BF]).memory;
+    size_t index_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_BF)).memory;
     double memory_delta = 0.0;
 
     for (auto _ : st) {
-        VecSimBatchIterator *batchIterator = VecSimBatchIterator_New(
-            INDICES[VecSimAlgo_BF], QUERIES[iter % N_QUERIES].data(), nullptr);
+        VecSimBatchIterator *batchIterator =
+            VecSimBatchIterator_New(GET_INDEX(INDEX_BF), QUERIES[iter % N_QUERIES].data(), nullptr);
         size_t batches_counter = 0;
         while (VecSimBatchIterator_HasNext(batchIterator)) {
             VecSimQueryReply *res = VecSimBatchIterator_Next(batchIterator, batch_size, BY_ID);
@@ -98,7 +98,7 @@ void BM_BatchIterator<index_type_t>::BF_FixedBatchSize(benchmark::State &st) {
                 break;
             }
         }
-        size_t curr_memory = VecSimIndex_StatsInfo(INDICES[VecSimAlgo_BF]).memory;
+        size_t curr_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_BF)).memory;
         memory_delta += (double)(curr_memory - index_memory);
         VecSimBatchIterator_Free(batchIterator);
         iter++;
@@ -112,8 +112,8 @@ void BM_BatchIterator<index_type_t>::BF_VariableBatchSize(benchmark::State &st)
     size_t num_batches = st.range(1);
     size_t iter = 0;
     for (auto _ : st) {
-        VecSimBatchIterator *batchIterator = VecSimBatchIterator_New(
-            INDICES[VecSimAlgo_BF], QUERIES[iter % N_QUERIES].data(), nullptr);
+        VecSimBatchIterator *batchIterator =
+            VecSimBatchIterator_New(GET_INDEX(INDEX_BF), QUERIES[iter % N_QUERIES].data(), nullptr);
         size_t batches_counter = 0;
         while (VecSimBatchIterator_HasNext(batchIterator)) {
             VecSimQueryReply *res = VecSimBatchIterator_Next(batchIterator, batch_size, BY_ID);
@@ -136,8 +136,8 @@ void BM_BatchIterator<index_type_t>::BF_BatchesToAdhocBF(benchmark::State &st) {
     size_t batch_size = 10;
     size_t iter = 0;
     for (auto _ : st) {
-        VecSimBatchIterator *batchIterator = VecSimBatchIterator_New(
-            INDICES[VecSimAlgo_BF], QUERIES[iter % N_QUERIES].data(), nullptr);
+        VecSimBatchIterator *batchIterator =
+            VecSimBatchIterator_New(GET_INDEX(INDEX_BF), QUERIES[iter % N_QUERIES].data(), nullptr);
         size_t batches_counter = 0;
         while (VecSimBatchIterator_HasNext(batchIterator)) {
             if (batches_counter == num_batches) {
@@ -151,7 +151,7 @@ void BM_BatchIterator<index_type_t>::BF_BatchesToAdhocBF(benchmark::State &st) {
         VecSimBatchIterator_Free(batchIterator);
         // Switch to ad-hoc BF
         for (size_t i = 0; i < N_VECTORS; i += step) {
-            VecSimIndex_GetDistanceFrom_Unsafe(INDICES[VecSimAlgo_BF], i,
+            VecSimIndex_GetDistanceFrom_Unsafe(GET_INDEX(INDEX_BF), i,
                                                QUERIES[iter % N_QUERIES].data());
         }
         iter++;
@@ -166,7 +166,7 @@ void BM_BatchIterator<index_type_t>::HNSW_FixedBatchSize(benchmark::State &st) {
     size_t total_res_num = num_batches * batch_size;
     size_t iter = 0;
     std::atomic_int correct = 0;
-    size_t index_memory = VecSimIndex_StatsInfo(INDICES[VecSimAlgo_HNSWLIB]).memory;
+    size_t index_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_HNSW)).memory;
     double memory_delta = 0.0;
 
     for (auto _ : st) {
@@ -185,7 +185,7 @@ void BM_BatchIterator<index_type_t>::HNSW_VariableBatchSize(benchmark::State &st
     size_t total_res_num;
     size_t iter = 0;
     std::atomic_int correct = 0;
-    size_t index_memory = VecSimIndex_StatsInfo(INDICES[VecSimAlgo_HNSWLIB]).memory;
+    size_t index_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_HNSW)).memory;
     double memory_delta = 0.0;
 
     for (auto _ : st) {
@@ -204,15 +204,15 @@ void BM_BatchIterator<index_type_t>::HNSW_BatchesToAdhocBF(benchmark::State &st)
     size_t total_res_num;
     size_t iter = 0;
     std::atomic_int correct = 0;
-    size_t index_memory = VecSimIndex_StatsInfo(INDICES[VecSimAlgo_HNSWLIB]).memory;
+    size_t index_memory = VecSimIndex_StatsInfo(GET_INDEX(INDEX_HNSW)).memory;
     double memory_delta = 0.0;
 
     for (auto _ : st) {
         RunBatchedSearch_HNSW(st, correct, iter, num_batches, 10, total_res_num, 2, index_memory,
                               memory_delta);
         // Switch to ad-hoc BF
         for (size_t i = 0; i < N_VECTORS; i += step) {
-            VecSimIndex_GetDistanceFrom_Unsafe(INDICES[VecSimAlgo_HNSWLIB], i,
+            VecSimIndex_GetDistanceFrom_Unsafe(GET_INDEX(INDEX_HNSW), i,
                                                QUERIES[iter % N_QUERIES].data());
         }
         iter++;

tests/benchmark/bm_common.h

@@ -23,9 +23,10 @@ class BM_VecSimCommon : public BM_VecSimIndex<index_type_t> {
     BM_VecSimCommon() = default;
     ~BM_VecSimCommon() = default;
 
+    // index_offset: Offset added to base index types to access variants (0=original, 1=updated)
+
     static void RunTopK_HNSW(benchmark::State &st, size_t ef, size_t iter, size_t k,
-                             std::atomic_int &correct, unsigned short index_offset = 0,
-                             bool is_tiered = false);
+                             std::atomic_int &correct, unsigned short index_offset = 0);
 
     // Search for the K closest vectors to the query in the index. K is defined in the
     // test registration (initialization file).
@@ -36,24 +37,22 @@ class BM_VecSimCommon : public BM_VecSimIndex<index_type_t> {
     static void TopK_Tiered(benchmark::State &st, unsigned short index_offset = 0);
 
     // Does nothing but returning the index memory.
-    static void Memory_FLAT(benchmark::State &st, unsigned short index_offset = 0);
-    static void Memory_HNSW(benchmark::State &st, unsigned short index_offset = 0);
-    static void Memory_Tiered(benchmark::State &st, unsigned short index_offset = 0);
+    static void Memory(benchmark::State &st, IndexTypeIndex index_type);
 };
 
 template <typename index_type_t>
 void BM_VecSimCommon<index_type_t>::RunTopK_HNSW(benchmark::State &st, size_t ef, size_t iter,
                                                  size_t k, std::atomic_int &correct,
-                                                 unsigned short index_offset, bool is_tiered) {
+                                                 unsigned short index_offset) {
     HNSWRuntimeParams hnswRuntimeParams = {.efRuntime = ef};
     auto query_params = BM_VecSimGeneral::CreateQueryParams(hnswRuntimeParams);
-    auto hnsw_results = VecSimIndex_TopKQuery(
-        INDICES[is_tiered ? VecSimAlgo_TIERED : VecSimAlgo_HNSWLIB + index_offset],
-        QUERIES[iter % N_QUERIES].data(), k, &query_params, BY_SCORE);
+    auto hnsw_results =
+        VecSimIndex_TopKQuery(GET_INDEX(INDEX_HNSW + index_offset),
+                              QUERIES[iter % N_QUERIES].data(), k, &query_params, BY_SCORE);
     st.PauseTiming();
 
     // Measure recall:
-    auto bf_results = VecSimIndex_TopKQuery(INDICES[VecSimAlgo_BF + index_offset],
+    auto bf_results = VecSimIndex_TopKQuery(GET_INDEX(INDEX_BF + index_offset),
                                             QUERIES[iter % N_QUERIES].data(), k, nullptr, BY_SCORE);
 
     BM_VecSimGeneral::MeasureRecall(hnsw_results, bf_results, correct);
@@ -64,35 +63,15 @@ void BM_VecSimCommon<index_type_t>::RunTopK_HNSW(benchmark::State &st, size_t ef
 }
 
 template <typename index_type_t>
-void BM_VecSimCommon<index_type_t>::Memory_FLAT(benchmark::State &st, unsigned short index_offset) {
-    auto index = INDICES[VecSimAlgo_BF + index_offset];
-    index->fitMemory();
-
-    for (auto _ : st) {
-        // Do nothing...
-    }
-    st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
-}
-template <typename index_type_t>
-void BM_VecSimCommon<index_type_t>::Memory_HNSW(benchmark::State &st, unsigned short index_offset) {
-    auto index = INDICES[VecSimAlgo_HNSWLIB + index_offset];
+void BM_VecSimCommon<index_type_t>::Memory(benchmark::State &st, IndexTypeIndex index_type) {
+    auto index = GET_INDEX(index_type);
     index->fitMemory();
 
     for (auto _ : st) {
         // Do nothing...
     }
     st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
 }
-template <typename index_type_t>
-void BM_VecSimCommon<index_type_t>::Memory_Tiered(benchmark::State &st,
-                                                  unsigned short index_offset) {
-    auto index = INDICES[VecSimAlgo_TIERED + index_offset];
-    index->fitMemory();
-    for (auto _ : st) {
-        // Do nothing...
-    }
-    st.counters["memory"] = (double)VecSimIndex_StatsInfo(index).memory;
-}
 
 // TopK search BM
 
@@ -101,8 +80,8 @@ void BM_VecSimCommon<index_type_t>::TopK_BF(benchmark::State &st, unsigned short
     size_t k = st.range(0);
     size_t iter = 0;
     for (auto _ : st) {
-        VecSimIndex_TopKQuery(INDICES[VecSimAlgo_BF + index_offset],
-                              QUERIES[iter % N_QUERIES].data(), k, nullptr, BY_SCORE);
+        VecSimIndex_TopKQuery(GET_INDEX(INDEX_BF + index_offset), QUERIES[iter % N_QUERIES].data(),
+                              k, nullptr, BY_SCORE);
         iter++;
     }
 }
@@ -126,8 +105,8 @@ void BM_VecSimCommon<index_type_t>::TopK_Tiered(benchmark::State &st, unsigned s
     size_t k = st.range(1);
     std::atomic_int correct = 0;
     std::atomic_int iter = 0;
-    auto *tiered_index =
-        dynamic_cast<TieredHNSWIndex<data_t, dist_t> *>(INDICES[VecSimAlgo_TIERED]);
+    auto tiered_index =
+        dynamic_cast<TieredHNSWIndex<data_t, dist_t> *>(GET_INDEX(INDEX_TIERED_HNSW));
     size_t total_iters = 50;
     VecSimQueryReply *all_results[total_iters];
 
@@ -136,9 +115,9 @@ void BM_VecSimCommon<index_type_t>::TopK_Tiered(benchmark::State &st, unsigned s
         HNSWRuntimeParams hnswRuntimeParams = {.efRuntime = search_job->ef};
         auto query_params = BM_VecSimGeneral::CreateQueryParams(hnswRuntimeParams);
         size_t cur_iter = search_job->iter;
-        auto hnsw_results =
-            VecSimIndex_TopKQuery(INDICES[VecSimAlgo_TIERED], QUERIES[cur_iter % N_QUERIES].data(),
-                                  search_job->k, &query_params, BY_SCORE);
+        auto hnsw_results = VecSimIndex_TopKQuery(GET_INDEX(INDEX_TIERED_HNSW),
+                                                  QUERIES[cur_iter % N_QUERIES].data(),
+                                                  search_job->k, &query_params, BY_SCORE);
         search_job->all_results[cur_iter] = hnsw_results;
         delete job;
     };
@@ -149,14 +128,14 @@ void BM_VecSimCommon<index_type_t>::TopK_Tiered(benchmark::State &st, unsigned s
                                            tiered_index, k, ef, iter++, all_results);
         tiered_index->submitSingleJob(search_job);
         if (iter == total_iters) {
-            BM_VecSimGeneral::mock_thread_pool.thread_pool_wait();
+            BM_VecSimGeneral::mock_thread_pool->thread_pool_wait();
         }
     }
 
     // Measure recall
     for (iter = 0; iter < total_iters; iter++) {
         auto bf_results =
-            VecSimIndex_TopKQuery(INDICES[VecSimAlgo_BF + index_offset],
+            VecSimIndex_TopKQuery(GET_INDEX(INDEX_BF + index_offset),
                                   QUERIES[iter % N_QUERIES].data(), k, nullptr, BY_SCORE);
         BM_VecSimGeneral::MeasureRecall(all_results[iter], bf_results, correct);
 
@@ -165,7 +144,7 @@ void BM_VecSimCommon<index_type_t>::TopK_Tiered(benchmark::State &st, unsigned s
     }
 
     st.counters["Recall"] = (float)correct / (float)(k * iter);
-    st.counters["num_threads"] = (double)BM_VecSimGeneral::mock_thread_pool.thread_pool_size;
+    st.counters["num_threads"] = (double)BM_VecSimGeneral::mock_thread_pool->thread_pool_size;
 }
 
 #define REGISTER_TopK_BF(BM_CLASS, BM_FUNC)                                                        \

tests/benchmark/bm_definitions.h

@@ -21,6 +21,80 @@ struct IndexType {
     typedef DistType dist_t;
 };
 
+// Array indices for accessing different index types in the indices array
+// Note: Updated variants are offset by 1 from their base types (e.g., INDEX_BF_UPDATED = INDEX_BF +
+// 1)
+enum IndexTypeIndex {
+    INDEX_BF = 0,
+    INDEX_BF_UPDATED,
+    INDEX_HNSW,
+    INDEX_HNSW_UPDATED,
+    INDEX_TIERED_HNSW,
+    INDEX_SVS,
+    INDEX_TIERED_SVS,
+    INDEX_SVS_QUANTIZED,
+    NUMBER_OF_INDEX_TYPES // Keep last
+};
+
+// Bit flags for selectively enabling index types in benchmarks via
+// BM_VecSimGeneral::enabled_index_types bitmask Limited to 32 index types because IndexTypeFlags is
+// stored in a 32-bit mask (uint32_t)
+// Note: Bit positions currently match IndexTypeIndex values but this is not required by the code
+enum IndexTypeFlags {
+    INDEX_MASK_BF = 1 << 0,
+    INDEX_MASK_BF_UPDATED = 1 << 1,
+    INDEX_MASK_HNSW = 1 << 2,
+    INDEX_MASK_HNSW_UPDATED = 1 << 3,
+    INDEX_MASK_TIERED_HNSW = 1 << 4,
+    INDEX_MASK_SVS = 1 << 5,
+    INDEX_MASK_TIERED_SVS = 1 << 6,
+    INDEX_MASK_SVS_COMPRESSED = 1 << 7
+};
+
+// Smart pointer wrapper for VecSimIndex with configurable ownership
+// Supports:
+// 1. Ownership control via release_ownership()
+// 2. Sharing with thread pool's shared_ptr
+// 3. Safe transfer of ownership to tiered index
+class IndexPtr {
+private:
+    std::shared_ptr<VecSimIndex> ptr;
+    std::shared_ptr<bool> owns_ptr;
+
+public:
+    // Default constructor - creates empty pointer with ownership
+    IndexPtr() : ptr(nullptr), owns_ptr(std::make_shared<bool>(true)) {}
+
+    // Constructor - always starts with ownership
+    explicit IndexPtr(VecSimIndex *p) : owns_ptr(std::make_shared<bool>(true)) {
+        if (p) {
+            ptr = std::shared_ptr<VecSimIndex>(p, [owns_ptr = this->owns_ptr](VecSimIndex *p) {
+                if (*owns_ptr) {
+                    VecSimIndex_Free(p);
+                }
+            });
+        }
+    }
+
+    // Prevent copying to ensure clear ownership
+    IndexPtr(const IndexPtr &) = delete;
+    IndexPtr &operator=(const IndexPtr &) = delete;
+
+    // Allow moving
+    IndexPtr(IndexPtr &&) = default;
+    IndexPtr &operator=(IndexPtr &&) = default;
+
+    // Access methods
+    VecSimIndex *get() const { return ptr.get(); }
+    std::shared_ptr<VecSimIndex> get_shared() { return ptr; }
+
+    // Implicit conversion to raw pointer for ease of use
+    operator VecSimIndex *() const { return ptr.get(); }
+
+    // Ownership control
+    void release_ownership() { *owns_ptr = false; }
+};
+
 using fp32_index_t = IndexType<VecSimType_FLOAT32, float, float>;
 using fp64_index_t = IndexType<VecSimType_FLOAT64, double, double>;
 using bf16_index_t = IndexType<VecSimType_BFLOAT16, vecsim_types::bfloat16, float>;
@@ -29,9 +103,13 @@ using int8_index_t = IndexType<VecSimType_INT8, int8_t, float>;
 using uint8_index_t = IndexType<VecSimType_UINT8, uint8_t, float>;
 
 #define INDICES   BM_VecSimIndex<index_type_t>::indices
+#define GET_INDEX BM_VecSimIndex<index_type_t>::get_index
 #define QUERIES   BM_VecSimIndex<index_type_t>::queries
 #define N_QUERIES BM_VecSimGeneral::n_queries
 #define N_VECTORS BM_VecSimGeneral::n_vectors
 #define DIM       BM_VecSimGeneral::dim
 #define IS_MULTI  BM_VecSimGeneral::is_multi
-#define REF_COUNT BM_VecSimIndex<index_type_t>::ref_count
+
+constexpr uint32_t DEFAULT_BM_INDEXES_MASK = IndexTypeFlags::INDEX_MASK_BF |
+                                             IndexTypeFlags::INDEX_MASK_HNSW |
+                                             IndexTypeFlags::INDEX_MASK_TIERED_HNSW;