Skip to content

[core] Open compiled model as file stream when MMAP disabled. #31774

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
praasz merged 7 commits into from
Aug 21, 2025
Merged

[core] Open compiled model as file stream when MMAP disabled. #31774

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c3087a2
Open compiled model as file stream when MMAP disabled.
praasz Aug 18, 2025
8b9dca7
Remove commented code
praasz Aug 18, 2025
5ba2cf5
Fix model importer
praasz Aug 19, 2025
c1ad896
Fix build issue
praasz Aug 19, 2025
3410019
Fix TEMPLATE test
praasz Aug 19, 2025
eeda3aa
Fix CPU caching when mmap not supported by CPU plugin
praasz Aug 20, 2025
5a81bfd
Merge branch 'master' into bugfix/peak-memory-use-if-mmap-disabled
mlukasze Aug 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions src/inference/src/cache_manager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@
*/
#pragma once

#include <filesystem>
#include <fstream>
#include <functional>
#include <memory>
#include <string>
#include <variant>

#include "openvino/runtime/shared_buffer.hpp"
#include "openvino/runtime/tensor.hpp"
Expand Down Expand Up @@ -67,10 +69,15 @@ class ICacheManager {
*/
virtual void write_cache_entry(const std::string& id, StreamWriter writer) = 0;

/**
* @brief Variant type for compiled blob representation
*/
using CompiledBlobVariant = std::variant<const ov::Tensor, std::reference_wrapper<std::istream>>;

/**
* @brief Function passing created input stream
*/
using StreamReader = std::function<void(ov::Tensor&)>;
using StreamReader = std::function<void(CompiledBlobVariant&)>;

/**
* @brief Callback when OpenVINO intends to read model from cache
Expand Down Expand Up @@ -137,9 +144,14 @@ class FileStorageCacheManager final : public ICacheManager {
ScopedLocale plocal_C(LC_ALL, "C");
const auto blob_file_name = getBlobFile(id);
if (std::filesystem::exists(blob_file_name)) {
auto compiled_blob =
read_tensor_data(blob_file_name, element::u8, PartialShape::dynamic(1), 0, enable_mmap);
reader(compiled_blob);
if (enable_mmap) {
CompiledBlobVariant compiled_blob{std::in_place_index<0>, ov::read_tensor_data(blob_file_name)};
reader(compiled_blob);
} else {
std::ifstream stream(blob_file_name, std::ios_base::binary);
CompiledBlobVariant compiled_blob{std::in_place_index<1>, std::ref(stream)};
reader(compiled_blob);
}
}
}

Expand Down
35 changes: 25 additions & 10 deletions src/inference/src/dev/core_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1531,16 +1531,24 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::load_model_from_cache(
cacheContent.blobId,
cacheContent.mmap_enabled && ov::util::contains(plugin.get_property(ov::internal::supported_properties),
ov::internal::caching_with_mmap),
[&](ov::Tensor& compiled_blob) {
[&](ICacheManager::CompiledBlobVariant& compiled_blob) {
OV_ITT_SCOPE(FIRST_INFERENCE,
ov::itt::domains::LoadTime,
"Core::load_model_from_cache::ReadStreamAndImport");
ov::CompiledBlobHeader header;
size_t compiled_blob_offset = 0;
try {
header.read_from_buffer(static_cast<const char*>(compiled_blob.data()),
compiled_blob.get_byte_size(),
compiled_blob_offset);
ov::util::VariantVisitor header_reader{[&](const ov::Tensor& tensor) {
header.read_from_buffer(
static_cast<const char*>(tensor.data()),
tensor.get_byte_size(),
compiled_blob_offset);
},
[&](std::reference_wrapper<std::istream> stream) {
stream >> header;
}};
std::visit(header_reader, compiled_blob);

if (header.get_file_info() != ov::ModelCache::calculate_file_info(cacheContent.modelPath)) {
// Original file is changed, don't use cache
OPENVINO_THROW("Original model file is changed");
Expand Down Expand Up @@ -1594,12 +1602,19 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::load_model_from_cache(
}
}

ov::Tensor compiled_blob_without_header{compiled_blob,
{compiled_blob_offset},
{compiled_blob.get_size()}};

compiled_model = context ? plugin.import_model(compiled_blob_without_header, context, update_config)
: plugin.import_model(compiled_blob_without_header, update_config);
ov::util::VariantVisitor model_importer{
[&](const ov::Tensor& compiled_blob) -> ov::SoPtr<ov::ICompiledModel> {
const ov::Tensor compiled_blob_without_header{compiled_blob,
{compiled_blob_offset},
{compiled_blob.get_size()}};
return context ? plugin.import_model(compiled_blob_without_header, context, update_config)
: plugin.import_model(compiled_blob_without_header, update_config);
},
[&](std::reference_wrapper<std::istream> stream) -> ov::SoPtr<ov::ICompiledModel> {
return context ? plugin.import_model(stream, context, update_config)
: plugin.import_model(stream, update_config);
}};
compiled_model = std::visit(model_importer, compiled_blob);
});
} catch (const HeaderException&) {
// For these exceptions just remove old cache and set that import didn't work
Expand Down
Loading
Loading