From 39feb47bc846215d87b97c48501874db2d1bf0bd Mon Sep 17 00:00:00 2001 From: Tillmann Karras Date: Sun, 27 Apr 2025 00:17:29 +0100 Subject: [PATCH] Load system game settings from a single compressed file On Windows this should make extracting/updating Dolphin significantly faster. --- Source/Core/Common/IniFile.cpp | 56 ++++++++++++++++++- Source/Core/Common/IniFile.h | 19 +++++++ .../Core/ConfigLoaders/GameConfigLoader.cpp | 10 +++- .../Core/ConfigLoaders/GameConfigLoader.h | 7 +++ Source/Core/Core/ConfigManager.cpp | 6 +- Source/Core/Core/NetPlayServer.cpp | 3 +- .../Core/DolphinQt/Config/GameConfigEdit.cpp | 16 ++++-- .../DolphinQt/Config/GameConfigWidget.cpp | 30 +++++++--- Tools/build-default-settings.py | 18 ++++++ 9 files changed, 148 insertions(+), 17 deletions(-) create mode 100755 Tools/build-default-settings.py diff --git a/Source/Core/Common/IniFile.cpp b/Source/Core/Common/IniFile.cpp index 6e6749cb0d..d6c38ac249 100644 --- a/Source/Core/Common/IniFile.cpp +++ b/Source/Core/Common/IniFile.cpp @@ -7,11 +7,14 @@ #include #include #include +#include #include #include #include #include +#include + #include "Common/FileUtil.h" #include "Common/StringUtil.h" @@ -246,6 +249,25 @@ bool IniFile::Load(const std::string& filename, bool keep_current_data) std::ifstream in; File::OpenFStream(in, filename, std::ios::in); + return Load(in); +} + +bool IniFile::Load(const IniDirectory& dir, const std::string& filename, bool keep_current_data) +{ + if (!keep_current_data) + sections.clear(); + + auto content = dir.Get(filename); + if (!content) + return false; + + // TODO: avoid string copy + std::stringstream in{std::string(*content), std::ios::in}; + return Load(in); +} + +bool IniFile::Load(std::istream& in) +{ if (in.fail()) return false; @@ -311,8 +333,6 @@ bool IniFile::Load(const std::string& filename, bool keep_current_data) } } } - - in.close(); return true; } @@ -352,6 +372,38 @@ bool IniFile::Save(const std::string& filename) return File::RenameSync(temp, filename); } +IniDirectory::IniDirectory(const std::string& filename) +{ + std::string src; + if (!File::ReadFileToString(filename, src)) + return; + unsigned long long want_size = ZSTD_getFrameContentSize(src.data(), src.size()); + if (want_size == ZSTD_CONTENTSIZE_UNKNOWN || want_size == ZSTD_CONTENTSIZE_ERROR) + return; + m_data.reset(want_size); + size_t got_size = ZSTD_decompress(m_data.data(), want_size, src.data(), src.size()); + if (got_size != want_size) + return; + + for (size_t i = 0; i < m_data.size();) + { + auto name = std::string_view(&m_data[i]); + i += name.size() + 1; + auto content = std::string_view(&m_data[i]); + i += content.size() + 1; + m_files.emplace(name, content); + } +} + +std::optional IniDirectory::Get(std::string_view filename) const +{ + const auto it = m_files.find(filename); + if (it == m_files.end()) + return {}; + + return it->second; +} + // Unit test. TODO: Move to the real unit test framework. /* int main() diff --git a/Source/Core/Common/IniFile.h b/Source/Core/Common/IniFile.h index 31a7ee0e6b..b4823328dc 100644 --- a/Source/Core/Common/IniFile.h +++ b/Source/Core/Common/IniFile.h @@ -6,15 +6,31 @@ #include #include #include +#include +#include #include #include #include +#include "Common/Buffer.h" #include "Common/CommonTypes.h" #include "Common/StringUtil.h" namespace Common { + +class IniDirectory +{ +public: + IniDirectory(const std::string& filename); + static const IniDirectory& GetInstance(); + std::optional Get(std::string_view filename) const; + +private: + Common::UniqueBuffer m_data; + std::map m_files; +}; + class IniFile { public: @@ -92,6 +108,8 @@ public: * user-specified) and should eventually be replaced with a less stupid system. */ bool Load(const std::string& filename, bool keep_current_data = false); + bool Load(const IniDirectory& dir, const std::string& filename, bool keep_current_data = false); + bool Load(std::istream& in); bool Save(const std::string& filename); @@ -145,4 +163,5 @@ private: static const std::string& NULL_STRING; }; + } // namespace Common diff --git a/Source/Core/Core/ConfigLoaders/GameConfigLoader.cpp b/Source/Core/Core/ConfigLoaders/GameConfigLoader.cpp index a922a9ec38..d61a8a1972 100644 --- a/Source/Core/Core/ConfigLoaders/GameConfigLoader.cpp +++ b/Source/Core/Core/ConfigLoaders/GameConfigLoader.cpp @@ -174,6 +174,13 @@ static SectionKey GetINILocationFromConfig(const Location& location) return {Config::GetSystemName(location.system) + "." + location.section, location.key}; } +const Common::IniDirectory& GetDefaultGameSettings() +{ + static Common::IniDirectory s_sys_inis(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + "default.bin.zstd"); + return s_sys_inis; +} + // INI Game layer configuration loader class INIGameConfigLayerLoader final : public Config::ConfigLayerLoader { @@ -189,8 +196,9 @@ public: Common::IniFile ini; if (layer->GetLayer() == Config::LayerType::GlobalGame) { + auto& sys_inis = GetDefaultGameSettings(); for (const std::string& filename : GetGameIniFilenames(m_id, m_revision)) - ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true); + ini.Load(sys_inis, filename, true); } else { diff --git a/Source/Core/Core/ConfigLoaders/GameConfigLoader.h b/Source/Core/Core/ConfigLoaders/GameConfigLoader.h index 78f849b4fe..537ff0f19b 100644 --- a/Source/Core/Core/ConfigLoaders/GameConfigLoader.h +++ b/Source/Core/Core/ConfigLoaders/GameConfigLoader.h @@ -11,6 +11,11 @@ #include "Common/CommonTypes.h" +namespace Common +{ +class IniDirectory; +} + namespace Config { class ConfigLayerLoader; @@ -18,6 +23,8 @@ class ConfigLayerLoader; namespace ConfigLoaders { +const Common::IniDirectory& GetDefaultGameSettings(); + std::vector GetGameIniFilenames(const std::string& id, std::optional revision); std::unique_ptr GenerateGlobalGameConfigLoader(const std::string& id, diff --git a/Source/Core/Core/ConfigManager.cpp b/Source/Core/Core/ConfigManager.cpp index e005e78341..39004136d2 100644 --- a/Source/Core/Core/ConfigManager.cpp +++ b/Source/Core/Core/ConfigManager.cpp @@ -510,8 +510,9 @@ Common::IniFile SConfig::LoadGameIni() const Common::IniFile SConfig::LoadDefaultGameIni(const std::string& id, std::optional revision) { Common::IniFile game_ini; + auto& sys_inis = ConfigLoaders::GetDefaultGameSettings(); for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision)) - game_ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true); + game_ini.Load(sys_inis, filename, true); return game_ini; } @@ -526,8 +527,9 @@ Common::IniFile SConfig::LoadLocalGameIni(const std::string& id, std::optional revision) { Common::IniFile game_ini; + auto& sys_inis = ConfigLoaders::GetDefaultGameSettings(); for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision)) - game_ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true); + game_ini.Load(sys_inis, filename, true); for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision)) game_ini.Load(File::GetUserPath(D_GAMESETTINGS_IDX) + filename, true); return game_ini; diff --git a/Source/Core/Core/NetPlayServer.cpp b/Source/Core/Core/NetPlayServer.cpp index 4cff258e9b..b11958d2e5 100644 --- a/Source/Core/Core/NetPlayServer.cpp +++ b/Source/Core/Core/NetPlayServer.cpp @@ -2055,8 +2055,9 @@ bool NetPlayServer::SyncCodes() const auto game_id = game->GetGameID(); const auto revision = game->GetRevision(); Common::IniFile globalIni; + auto& sys_ini = ConfigLoaders::GetDefaultGameSettings(); for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision)) - globalIni.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true); + globalIni.Load(sys_ini, filename, true); Common::IniFile localIni; for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision)) localIni.Load(File::GetUserPath(D_GAMESETTINGS_IDX) + filename, true); diff --git a/Source/Core/DolphinQt/Config/GameConfigEdit.cpp b/Source/Core/DolphinQt/Config/GameConfigEdit.cpp index d114704f6a..191ad12c0a 100644 --- a/Source/Core/DolphinQt/Config/GameConfigEdit.cpp +++ b/Source/Core/DolphinQt/Config/GameConfigEdit.cpp @@ -105,11 +105,19 @@ void GameConfigEdit::AddDescription(const QString& keyword, const QString& descr void GameConfigEdit::LoadFile() { - QFile file(m_path); - if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) - return; + if (m_read_only) + { + // HACK + m_edit->setPlainText(m_path); + } + else + { + QFile file(m_path); + if (!file.open(QIODevice::ReadOnly | QIODevice::Text)) + return; - m_edit->setPlainText(QString::fromStdString(file.readAll().toStdString())); + m_edit->setPlainText(QString::fromStdString(file.readAll().toStdString())); + } } void GameConfigEdit::SaveFile() diff --git a/Source/Core/DolphinQt/Config/GameConfigWidget.cpp b/Source/Core/DolphinQt/Config/GameConfigWidget.cpp index 99d7b45cce..baaf54ccf5 100644 --- a/Source/Core/DolphinQt/Config/GameConfigWidget.cpp +++ b/Source/Core/DolphinQt/Config/GameConfigWidget.cpp @@ -17,6 +17,7 @@ #include "Common/Config/Config.h" #include "Common/Config/Layer.h" #include "Common/FileUtil.h" +#include "Common/IniFile.h" #include "Core/Config/GraphicsSettings.h" #include "Core/Config/MainSettings.h" @@ -39,13 +40,29 @@ static void PopulateTab(QTabWidget* tab, const std::string& path, std::string& game_id, u16 revision, bool read_only) { - for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision)) + if (read_only) { - const std::string ini_path = path + filename; - if (File::Exists(ini_path)) + auto& sys_inis = ConfigLoaders::GetDefaultGameSettings(); + for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision)) { - auto* edit = new GameConfigEdit(nullptr, QString::fromStdString(ini_path), read_only); - tab->addTab(edit, QString::fromStdString(filename)); + if (auto content = sys_inis.Get(filename)) + { + auto* edit = + new GameConfigEdit(nullptr, QString::fromStdString(std::string(*content)), read_only); + tab->addTab(edit, QString::fromStdString(filename)); + } + } + } + else + { + for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision)) + { + const std::string ini_path = path + filename; + if (File::Exists(ini_path)) + { + auto* edit = new GameConfigEdit(nullptr, QString::fromStdString(ini_path), read_only); + tab->addTab(edit, QString::fromStdString(filename)); + } } } } @@ -65,8 +82,7 @@ GameConfigWidget::GameConfigWidget(const UICommon::GameFile& game) : m_game(game CreateWidgets(); connect(&Settings::Instance(), &Settings::ConfigChanged, this, &GameConfigWidget::LoadSettings); - PopulateTab(m_default_tab, File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP, m_game_id, - m_game.GetRevision(), true); + PopulateTab(m_default_tab, "", m_game_id, m_game.GetRevision(), true); PopulateTab(m_local_tab, File::GetUserPath(D_GAMESETTINGS_IDX), m_game_id, m_game.GetRevision(), false); diff --git a/Tools/build-default-settings.py b/Tools/build-default-settings.py new file mode 100755 index 0000000000..85348e72d5 --- /dev/null +++ b/Tools/build-default-settings.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 + +# We have over 1700 ini files now. +# Extracting/updating them all is very slow on Windows. +# This script compresses them all into a single file during the build. + +from glob import glob +from zstandard import ZstdCompressor + +raw = b'' +for i in glob('*.ini'): + raw += i.encode('ascii') + b'\0' + ini = open(i, 'rb').read() + assert b'\0' not in ini + raw += ini + b'\0' + +cooked = ZstdCompressor(19).compress(raw) +open('default.bin.zstd', 'wb+').write(cooked)