From 15162a734d7c9a5a3aecb7fbda41041c394f78a5 Mon Sep 17 00:00:00 2001 From: AnyOldName3 Date: Thu, 10 Apr 2025 16:16:19 +0100 Subject: [PATCH] Avoid IO in resolveParentFileIndices In the olden days, we passed it a vector of open ESMReader instances, as they knew the filenames and sizes, so were a convenient source of this knowledge. When the ReadersCache was introduced as a pool of readers to limit the maximum number of simultaneously open file handles (to avoid going over the OS' limit) it was a poor substitute. * We iterate over all the earlier readers in order in a double loop, which is the worst case scenario for an LRU pool as once we're past the size limit, we're guaranteed maximum thrashing - the least recently used item is the most likely to be used next, so the worst to evict. * We didn't want to read any ESM files, just know whether they'd been read and what their sizes were, so didn't want to open a file handle, which the ReadersCache forced us to do. Obviously, opening lots of file handles isn't fast, and as this was an operation done for each content file which iterated over the file's masters and within that loop iterated over every loaded file, that's O(n^3) complexity in the worst case, and for things like delta plugin merged plugins, they hit the worst case in long load orders. This resolves the freeze reported as https://gitlab.com/OpenMW/openmw/-/issues/8425, but there may be other freezes on launch. --- components/esm3/esmreader.cpp | 6 ++-- components/esm3/readerscache.cpp | 54 ++++++++++++++++++++++++++++++++ components/esm3/readerscache.hpp | 5 +++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/components/esm3/esmreader.cpp b/components/esm3/esmreader.cpp index 4f69b8edef..adb0d1c103 100644 --- a/components/esm3/esmreader.cpp +++ b/components/esm3/esmreader.cpp @@ -73,10 +73,10 @@ namespace ESM int index = getIndex(); for (int i = 0; i < getIndex(); i++) { - const ESM::ReadersCache::BusyItem reader = readers.get(static_cast(i)); - if (reader->getFileSize() == 0) + if (readers.getFileSize(static_cast(i)) == 0) continue; // Content file in non-ESM format - const auto fnamecandidate = Files::pathToUnicodeString(reader->getName().filename()); + const auto fnamecandidate + = Files::pathToUnicodeString(readers.getName(static_cast(i)).filename()); if (Misc::StringUtils::ciEqual(fname, fnamecandidate)) { index = i; diff --git a/components/esm3/readerscache.cpp b/components/esm3/readerscache.cpp index 732a2a22ba..a80236240e 100644 --- a/components/esm3/readerscache.cpp +++ b/components/esm3/readerscache.cpp @@ -47,6 +47,7 @@ namespace ESM { it->mReader.open(*it->mName); it->mName.reset(); + it->mFileSize.reset(); } mBusyItems.splice(mBusyItems.end(), mClosedItems, it); break; @@ -57,6 +58,58 @@ namespace ESM return BusyItem(*this, it); } + const std::filesystem::path& ReadersCache::getName(std::size_t index) const + { + const auto indexIt = mIndex.find(index); + if (indexIt == mIndex.end()) + throw std::logic_error("ESMReader at index " + std::to_string(index) + " has not been created yet"); + else + { + switch (indexIt->second->mState) + { + case State::Busy: + case State::Free: + return indexIt->second->mReader.getName(); + case State::Closed: + if (indexIt->second->mName) + return *indexIt->second->mName; + else + throw std::logic_error( + "ESMReader at index " + std::to_string(index) + " has forgotten its filename"); + default: + throw std::logic_error("ESMReader at index " + std::to_string(index) + " in unknown state"); + } + } + } + + std::size_t ReadersCache::getFileSize(std::size_t index) + { + const auto indexIt = mIndex.find(index); + if (indexIt == mIndex.end()) + return 0; + else + { + switch (indexIt->second->mState) + { + case State::Busy: + case State::Free: + if (!indexIt->second->mReader.getName().empty()) + return indexIt->second->mReader.getFileSize(); + else + throw std::logic_error( + "ESMReader at index " + std::to_string(index) + " has not been opened yet"); + case State::Closed: + if (indexIt->second->mFileSize) + return *indexIt->second->mFileSize; + else + throw std::logic_error( + "ESMReader at index " + std::to_string(index) + " has forgotten its file size"); + default: + throw std::logic_error("ESMReader at index " + std::to_string(index) + " in unknown state"); + } + } + } + void ReadersCache::closeExtraReaders() { while (!mFreeItems.empty() && mBusyItems.size() + mFreeItems.size() + 1 > mCapacity) @@ -65,6 +118,7 @@ namespace ESM if (it->mReader.isOpen()) { it->mName = it->mReader.getName(); + it->mFileSize = it->mReader.getFileSize(); it->mReader.close(); } mClosedItems.splice(mClosedItems.end(), mFreeItems, it); diff --git a/components/esm3/readerscache.hpp b/components/esm3/readerscache.hpp index 80ca5d1ef3..3f6ac01874 100644 --- a/components/esm3/readerscache.hpp +++ b/components/esm3/readerscache.hpp @@ -26,6 +26,7 @@ namespace ESM State mState = State::Busy; ESMReader mReader; std::optional mName; + std::optional mFileSize; Item() = default; }; @@ -55,6 +56,10 @@ namespace ESM BusyItem get(std::size_t index); + const std::filesystem::path& getName(std::size_t index) const; + + std::size_t getFileSize(std::size_t index); + void clear(); private: