From b14bd5916c21a68a9dfec1d41f8aa69f8a3862af Mon Sep 17 00:00:00 2001 From: Ricardo Subtil Date: Sun, 22 May 2022 16:37:37 +0100 Subject: [PATCH] Implemented unload of PCK files --- core/config/project_settings.cpp | 23 ++++++ core/config/project_settings.h | 2 + core/io/file_access_pack.cpp | 122 +++++++++++++++++++++++++++++-- core/io/file_access_pack.h | 58 ++++++++++----- core/io/file_access_zip.cpp | 2 + core/io/resource.cpp | 6 ++ core/io/resource.h | 1 + doc/classes/ProjectSettings.xml | 17 +++++ 8 files changed, 207 insertions(+), 24 deletions(-) diff --git a/core/config/project_settings.cpp b/core/config/project_settings.cpp index a9673cf79b33..b058467cb1f9 100644 --- a/core/config/project_settings.cpp +++ b/core/config/project_settings.cpp @@ -581,6 +581,27 @@ bool ProjectSettings::_load_resource_pack(const String &p_pack, bool p_replace_f return true; } +bool ProjectSettings::_unload_resource_pack(const String &p_pack) { + if (PackedData::get_singleton()->is_disabled()) { + return false; + } + + bool ok = PackedData::get_singleton()->remove_pack(p_pack) == OK; + if (!ok) { + return false; + } + + return true; +} + +bool ProjectSettings::_is_pack_loaded(const String &p_pack) { + if (PackedData::get_singleton()->is_disabled()) { + return false; + } + + return PackedData::get_singleton()->is_pack_loaded(p_pack); +} + void ProjectSettings::_convert_to_last_version(int p_from_version) { #ifndef DISABLE_DEPRECATED if (p_from_version <= 3) { @@ -1506,6 +1527,8 @@ void ProjectSettings::_bind_methods() { ClassDB::bind_method(D_METHOD("globalize_path", "path"), &ProjectSettings::globalize_path); ClassDB::bind_method(D_METHOD("save"), &ProjectSettings::save); ClassDB::bind_method(D_METHOD("load_resource_pack", "pack", "replace_files", "offset"), &ProjectSettings::load_resource_pack, DEFVAL(true), DEFVAL(0)); + ClassDB::bind_method(D_METHOD("unload_resource_pack", "pack"), &ProjectSettings::_unload_resource_pack); + ClassDB::bind_method(D_METHOD("is_pack_loaded", "name"), &ProjectSettings::_is_pack_loaded); ClassDB::bind_method(D_METHOD("save_custom", "file"), &ProjectSettings::_save_custom_bnd); diff --git a/core/config/project_settings.h b/core/config/project_settings.h index cf82d03c610e..0634bdb282bc 100644 --- a/core/config/project_settings.h +++ b/core/config/project_settings.h @@ -141,6 +141,8 @@ class ProjectSettings : public Object { bool load_resource_pack(const String &p_pack, bool p_replace_files, int p_offset); bool _load_resource_pack(const String &p_pack, bool p_replace_files = true, int p_offset = 0, bool p_main_pack = false); + bool _unload_resource_pack(const String &p_pack); + bool _is_pack_loaded(const String &p_pack); void _add_property_info_bind(const Dictionary &p_info); diff --git a/core/io/file_access_pack.cpp b/core/io/file_access_pack.cpp index 244469e9d90c..23ddcb428231 100644 --- a/core/io/file_access_pack.cpp +++ b/core/io/file_access_pack.cpp @@ -31,10 +31,16 @@ #include "file_access_pack.h" #include "core/io/file_access_encrypted.h" +#include "core/io/resource.h" #include "core/object/script_language.h" #include "core/os/os.h" #include "core/version.h" +#include "modules/modules_enabled.gen.h" // For gdscript. +#ifdef MODULE_GDSCRIPT_ENABLED +#include "modules/gdscript/gdscript_cache.h" +#endif + Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t p_offset) { for (int i = 0; i < sources.size(); i++) { if (sources[i]->try_open_pack(p_path, p_replace_files, p_offset)) { @@ -45,22 +51,79 @@ Error PackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t return ERR_FILE_UNRECOGNIZED; } +Error PackedData::remove_pack(const String &p_path) { + if (!is_pack_loaded(p_path)) { + return ERR_FILE_UNRECOGNIZED; + } + + Vector reload_packs; + Vector::Iterator> to_remove; + for (HashMap::Iterator E = files.begin(); E; ++E) { + PackedFile pf = E->value; + if (pf.pack.name != p_path) { + continue; + } + + PathMD5 pmd5 = pf.pack.replaced_pack; + if (pmd5.set) { + if (!reload_packs.has(pmd5)) { + reload_packs.push_back(pmd5); + } + } + + remove_path(pf.filepath); + to_remove.push_back(E); + } + + for (const HashMap::Iterator &E : to_remove) { + files.remove(E); + } + remove_loaded_pack(p_path); + + // For reloading files unloaded by more recent packs, we simply need to reload packs with replace_files disabled. + for (PathMD5 pmd5 : reload_packs) { + if (!loaded_packs.has(pmd5)) { + continue; + } + LoadedPackInfo pack_info = loaded_packs[pmd5]; + Error err = add_pack(pack_info.name, false, pack_info.offset); + if (err) { + return err; + } + } + + return OK; +} + void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted, bool p_bundle) { String simplified_path = p_path.simplify_path().trim_prefix("res://"); PathMD5 pmd5(simplified_path.md5_buffer()); + // When running on the editor, the base files are not loaded from a main pack file. + // This extra check prevents packs from overriding those base files. +#ifdef TOOLS_ENABLED + bool exists = files.has(pmd5) || FileAccess::exists(simplified_path); +#else bool exists = files.has(pmd5); +#endif + + PackInfo pi; + pi.name = p_pkg_path; + if (p_replace_files && exists) { + pi.replaced_pack = PathMD5(files[pmd5].pack.name.md5_buffer()); + } PackedFile pf; pf.encrypted = p_encrypted; pf.bundle = p_bundle; - pf.pack = p_pkg_path; + pf.pack = pi; pf.offset = p_ofs; pf.size = p_size; for (int i = 0; i < 16; i++) { pf.md5[i] = p_md5[i]; } pf.src = p_src; + pf.filepath = simplified_path.replace_first("res://", ""); if (!exists || p_replace_files) { files[pmd5] = pf; @@ -68,6 +131,7 @@ void PackedData::add_path(const String &p_pkg_path, const String &p_path, uint64 if (!exists) { // Search for directory. + String p = pf.filepath; PackedDir *cd = root; if (simplified_path.contains_char('/')) { // In a subdirectory. @@ -114,10 +178,54 @@ void PackedData::remove_path(const String &p_path) { } } } + String filename = simplified_path.get_file(); + cd->files.erase(filename); - cd->files.erase(simplified_path.get_file()); + // Clear empty folders. + while (cd && cd->files.is_empty() && cd->subdirs.is_empty()) { + String name = cd->name; + cd = cd->parent; + if (cd) { + cd->subdirs.erase(name); + } + } files.erase(pmd5); + + String res_path = "res://" + p_path; + + // Remove paths from cache. + if (ResourceCache::has(res_path)) { + ResourceCache::remove_cached_resource(res_path); + } + + // GDScript also caches scripts internally, so they too must be removed. +#ifdef MODULE_GDSCRIPT_ENABLED + if (GDScriptCache::get_cached_script(res_path).is_valid()) { + GDScriptCache::remove_script(res_path); + } +#endif +} + +void PackedData::add_loaded_pack(const String &p_path, const uint64_t &p_offset) { + if (!is_pack_loaded(p_path)) { + LoadedPackInfo pack_info; + pack_info.name = p_path; + pack_info.offset = p_offset; + + PathMD5 pmd5(p_path.md5_buffer()); + loaded_packs.insert(pmd5, pack_info); + } +} + +void PackedData::remove_loaded_pack(const String &p_path) { + PathMD5 pmd5(p_path.md5_buffer()); + loaded_packs.erase(pmd5); +} + +bool PackedData::is_pack_loaded(const String &p_pack_path) const { + PathMD5 pmd5(p_pack_path.md5_buffer()); + return loaded_packs.has(pmd5); } void PackedData::add_pack_source(PackSource *p_source) { @@ -305,6 +413,8 @@ bool PackedSourcePCK::try_open_pack(const String &p_path, bool p_replace_files, f = fae; } + PackedData::get_singleton()->add_loaded_pack(p_path, p_offset); + for (int i = 0; i < file_count; i++) { uint32_t sl = f->get_32(); CharString cs; @@ -476,17 +586,17 @@ FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFil f = FileAccess::open(simplified_path, FileAccess::READ | FileAccess::SKIP_PACK); off = 0; // For the sparse pack offset is always zero. } else { - f = FileAccess::open(pf.pack, FileAccess::READ); + f = FileAccess::open(pf.pack.name, FileAccess::READ); f->seek(pf.offset); off = pf.offset; } - ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack))); + ERR_FAIL_COND_MSG(f.is_null(), vformat("Can't open pack-referenced file '%s'.", String(pf.pack.name))); if (pf.encrypted) { Ref fae; fae.instantiate(); - ERR_FAIL_COND_MSG(fae.is_null(), vformat("Can't open encrypted pack-referenced file '%s'.", String(pf.pack))); + ERR_FAIL_COND_MSG(fae.is_null(), vformat("Can't open encrypted pack-referenced file '%s'.", String(pf.pack.name))); Vector key; key.resize(32); @@ -495,7 +605,7 @@ FileAccessPack::FileAccessPack(const String &p_path, const PackedData::PackedFil } Error err = fae->open_and_parse(f, key, FileAccessEncrypted::MODE_READ, false); - ERR_FAIL_COND_MSG(err, vformat("Can't open encrypted pack-referenced file '%s'.", String(pf.pack))); + ERR_FAIL_COND_MSG(err, vformat("Can't open encrypted pack-referenced file '%s'.", String(pf.pack.name))); f = fae; off = 0; } diff --git a/core/io/file_access_pack.h b/core/io/file_access_pack.h index c2c7575542ce..a22a3f161459 100644 --- a/core/io/file_access_pack.h +++ b/core/io/file_access_pack.h @@ -64,31 +64,17 @@ class PackedData { friend class PackSource; public: - struct PackedFile { - String pack; - uint64_t offset; //if offset is ZERO, the file was ERASED - uint64_t size; - uint8_t md5[16]; - PackSource *src = nullptr; - bool encrypted; - bool bundle; - }; - -private: - struct PackedDir { - PackedDir *parent = nullptr; - String name; - HashMap subdirs; - HashSet files; - }; - struct PathMD5 { uint64_t a = 0; uint64_t b = 0; + bool set = false; bool operator==(const PathMD5 &p_val) const { return (a == p_val.a) && (b == p_val.b); } + bool operator!=(const PathMD5 &p_val) const { + return (a != p_val.a) || (b != p_val.b); + } static uint32_t hash(const PathMD5 &p_val) { uint32_t h = hash_murmur3_one_32(p_val.a); return hash_fmix32(hash_murmur3_one_32(p_val.b, h)); @@ -99,10 +85,41 @@ class PackedData { explicit PathMD5(const Vector &p_buf) { a = *((uint64_t *)&p_buf[0]); b = *((uint64_t *)&p_buf[8]); + set = true; } }; + struct PackInfo { + String name; + PathMD5 replaced_pack; + }; + + struct PackedFile { + PackInfo pack; + String filepath; + uint64_t offset; //if offset is ZERO, the file was ERASED + uint64_t size; + uint8_t md5[16]; + PackSource *src = nullptr; + bool encrypted; + bool bundle; + }; + +private: + struct PackedDir { + PackedDir *parent = nullptr; + String name; + HashMap subdirs; + HashSet files; + }; + + struct LoadedPackInfo { + String name; + uint64_t offset; + }; + HashMap files; + HashMap loaded_packs; Vector sources; @@ -121,11 +138,16 @@ class PackedData { uint8_t *get_file_hash(const String &p_path); HashSet get_file_paths() const; + void add_loaded_pack(const String &p_path, const uint64_t &p_offset); + void remove_loaded_pack(const String &p_path); + bool is_pack_loaded(const String &p_pack_path) const; + void set_disabled(bool p_disabled) { disabled = p_disabled; } _FORCE_INLINE_ bool is_disabled() const { return disabled; } static PackedData *get_singleton() { return singleton; } Error add_pack(const String &p_path, bool p_replace_files, uint64_t p_offset); + Error remove_pack(const String &p_path); void clear(); diff --git a/core/io/file_access_zip.cpp b/core/io/file_access_zip.cpp index 33bceee52d99..e947e561b689 100644 --- a/core/io/file_access_zip.cpp +++ b/core/io/file_access_zip.cpp @@ -177,6 +177,8 @@ bool ZipArchive::try_open_pack(const String &p_path, bool p_replace_files, uint6 packages.push_back(pkg); int pkg_num = packages.size() - 1; + PackedData::get_singleton()->add_loaded_pack(p_path, 0); + for (uint64_t i = 0; i < gi.number_entry; i++) { char filename_inzip[256]; diff --git a/core/io/resource.cpp b/core/io/resource.cpp index f2ecdba930b0..e0b72c6b777f 100644 --- a/core/io/resource.cpp +++ b/core/io/resource.cpp @@ -866,6 +866,12 @@ void ResourceCache::get_cached_resources(List> *p_resources) { } } +void ResourceCache::remove_cached_resource(const String &p_path) { + lock.lock(); + resources.erase(p_path); + lock.unlock(); +} + int ResourceCache::get_cached_resource_count() { MutexLock mutex_lock(lock); return resources.size(); diff --git a/core/io/resource.h b/core/io/resource.h index 090d2248e2c8..d08f1ee275a8 100644 --- a/core/io/resource.h +++ b/core/io/resource.h @@ -208,5 +208,6 @@ class ResourceCache { static bool has(const String &p_path); static Ref get_ref(const String &p_path); static void get_cached_resources(List> *p_resources); + static void remove_cached_resource(const String &p_path); static int get_cached_resource_count(); }; diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index 39e364810563..00965ade9b8e 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -153,6 +153,14 @@ [b]Note:[/b] In order to be be detected, custom settings have to be either defined with [method set_setting], or exist in the [code]project.godot[/code] file. This is especially relevant when using [method set_initial_value]. + + + + + Returns [code]true[/code] if a given .pck or .zip file is currently loaded. + [b]Note:[/b] A pack is considered loaded even if nothing is currently using its resources, or if all of its files have been replaced by other packs. + + @@ -257,6 +265,15 @@ This can also be used to erase custom project settings. To do this change the setting value to [code]null[/code]. + + + + + Unloads a previously loaded [param pack] from the resource filesystem. Returns [code]true[/code] on success. + [b]Note:[/b] If this pack replaced files when it was loaded, and if the affected pack is still loaded, then all the replaced files will be restored. + [b]Warning:[/b] Unloading a pack can cause issues and instability if any of it's resources are still being used. Ensure nothing is still using the pack's resources when unloading it. + +