Load system game settings from a single compressed file

On Windows this should make extracting/updating Dolphin significantly
faster.

Source/Core/Common/IniFile.cpp

@@ -7,11 +7,14 @@
 #include <cstddef>
 #include <fstream>
 #include <map>
+#include <sstream>
 #include <string>
 #include <string_view>
 #include <utility>
 #include <vector>
 
+#include <zstd.h>
+
 #include "Common/FileUtil.h"
 #include "Common/StringUtil.h"
 
@@ -246,6 +249,25 @@ bool IniFile::Load(const std::string& filename, bool keep_current_data)
   std::ifstream in;
   File::OpenFStream(in, filename, std::ios::in);
 
+  return Load(in);
+}
+
+bool IniFile::Load(const IniDirectory& dir, const std::string& filename, bool keep_current_data)
+{
+  if (!keep_current_data)
+    sections.clear();
+
+  auto content = dir.Get(filename);
+  if (!content)
+    return false;
+
+  // TODO: avoid string copy
+  std::stringstream in{std::string(*content), std::ios::in};
+  return Load(in);
+}
+
+bool IniFile::Load(std::istream& in)
+{
   if (in.fail())
     return false;
 
@@ -311,8 +333,6 @@ bool IniFile::Load(const std::string& filename, bool keep_current_data)
       }
     }
   }
-
-  in.close();
   return true;
 }
 
@@ -352,6 +372,38 @@ bool IniFile::Save(const std::string& filename)
   return File::RenameSync(temp, filename);
 }
 
+IniDirectory::IniDirectory(const std::string& filename)
+{
+  std::string src;
+  if (!File::ReadFileToString(filename, src))
+    return;
+  unsigned long long want_size = ZSTD_getFrameContentSize(src.data(), src.size());
+  if (want_size == ZSTD_CONTENTSIZE_UNKNOWN || want_size == ZSTD_CONTENTSIZE_ERROR)
+    return;
+  m_data.reset(want_size);
+  size_t got_size = ZSTD_decompress(m_data.data(), want_size, src.data(), src.size());
+  if (got_size != want_size)
+    return;
+
+  for (size_t i = 0; i < m_data.size();)
+  {
+    auto name = std::string_view(&m_data[i]);
+    i += name.size() + 1;
+    auto content = std::string_view(&m_data[i]);
+    i += content.size() + 1;
+    m_files.emplace(name, content);
+  }
+}
+
+std::optional<std::string_view> IniDirectory::Get(std::string_view filename) const
+{
+  const auto it = m_files.find(filename);
+  if (it == m_files.end())
+    return {};
+
+  return it->second;
+}
+
 // Unit test. TODO: Move to the real unit test framework.
 /*
    int main()

Source/Core/Common/IniFile.h

@@ -6,15 +6,31 @@
 #include <algorithm>
 #include <list>
 #include <map>
+#include <optional>
+#include <span>
 #include <string>
 #include <string_view>
 #include <vector>
 
+#include "Common/Buffer.h"
 #include "Common/CommonTypes.h"
 #include "Common/StringUtil.h"
 
 namespace Common
 {
+
+class IniDirectory
+{
+public:
+  IniDirectory(const std::string& filename);
+  static const IniDirectory& GetInstance();
+  std::optional<std::string_view> Get(std::string_view filename) const;
+
+private:
+  Common::UniqueBuffer<char> m_data;
+  std::map<std::string_view, std::string_view> m_files;
+};
+
 class IniFile
 {
 public:
@@ -92,6 +108,8 @@ public:
    * user-specified) and should eventually be replaced with a less stupid system.
    */
   bool Load(const std::string& filename, bool keep_current_data = false);
+  bool Load(const IniDirectory& dir, const std::string& filename, bool keep_current_data = false);
+  bool Load(std::istream& in);
 
   bool Save(const std::string& filename);
 
@@ -145,4 +163,5 @@ private:
 
   static const std::string& NULL_STRING;
 };
+
 }  // namespace Common

Source/Core/Core/ConfigLoaders/GameConfigLoader.cpp

@@ -174,6 +174,13 @@ static SectionKey GetINILocationFromConfig(const Location& location)
   return {Config::GetSystemName(location.system) + "." + location.section, location.key};
 }
 
+const Common::IniDirectory& GetDefaultGameSettings()
+{
+  static Common::IniDirectory s_sys_inis(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP
+                                         "default.bin.zstd");
+  return s_sys_inis;
+}
+
 // INI Game layer configuration loader
 class INIGameConfigLayerLoader final : public Config::ConfigLayerLoader
 {
@@ -189,8 +196,9 @@ public:
     Common::IniFile ini;
     if (layer->GetLayer() == Config::LayerType::GlobalGame)
     {
+      auto& sys_inis = GetDefaultGameSettings();
       for (const std::string& filename : GetGameIniFilenames(m_id, m_revision))
-        ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true);
+        ini.Load(sys_inis, filename, true);
     }
     else
     {

Source/Core/Core/ConfigLoaders/GameConfigLoader.h

@@ -11,6 +11,11 @@
 
 #include "Common/CommonTypes.h"
 
+namespace Common
+{
+class IniDirectory;
+}
+
 namespace Config
 {
 class ConfigLayerLoader;
@@ -18,6 +23,8 @@ class ConfigLayerLoader;
 
 namespace ConfigLoaders
 {
+const Common::IniDirectory& GetDefaultGameSettings();
+
 std::vector<std::string> GetGameIniFilenames(const std::string& id, std::optional<u16> revision);
 
 std::unique_ptr<Config::ConfigLayerLoader> GenerateGlobalGameConfigLoader(const std::string& id,

Source/Core/Core/ConfigManager.cpp

@@ -510,8 +510,9 @@ Common::IniFile SConfig::LoadGameIni() const
 Common::IniFile SConfig::LoadDefaultGameIni(const std::string& id, std::optional<u16> revision)
 {
   Common::IniFile game_ini;
+  auto& sys_inis = ConfigLoaders::GetDefaultGameSettings();
   for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision))
-    game_ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true);
+    game_ini.Load(sys_inis, filename, true);
   return game_ini;
 }
 
@@ -526,8 +527,9 @@ Common::IniFile SConfig::LoadLocalGameIni(const std::string& id, std::optional<u
 Common::IniFile SConfig::LoadGameIni(const std::string& id, std::optional<u16> revision)
 {
   Common::IniFile game_ini;
+  auto& sys_inis = ConfigLoaders::GetDefaultGameSettings();
   for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision))
-    game_ini.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true);
+    game_ini.Load(sys_inis, filename, true);
   for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(id, revision))
     game_ini.Load(File::GetUserPath(D_GAMESETTINGS_IDX) + filename, true);
   return game_ini;

Source/Core/Core/NetPlayServer.cpp

@@ -2055,8 +2055,9 @@ bool NetPlayServer::SyncCodes()
   const auto game_id = game->GetGameID();
   const auto revision = game->GetRevision();
   Common::IniFile globalIni;
+  auto& sys_ini = ConfigLoaders::GetDefaultGameSettings();
   for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision))
-    globalIni.Load(File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP + filename, true);
+    globalIni.Load(sys_ini, filename, true);
   Common::IniFile localIni;
   for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision))
     localIni.Load(File::GetUserPath(D_GAMESETTINGS_IDX) + filename, true);

Source/Core/DolphinQt/Config/GameConfigEdit.cpp

@@ -105,11 +105,19 @@ void GameConfigEdit::AddDescription(const QString& keyword, const QString& descr
 
 void GameConfigEdit::LoadFile()
 {
-  QFile file(m_path);
+  if (m_read_only)
-  if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
+  {
-    return;
+    // HACK
+    m_edit->setPlainText(m_path);
+  }
+  else
+  {
+    QFile file(m_path);
+    if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
+      return;
 
-  m_edit->setPlainText(QString::fromStdString(file.readAll().toStdString()));
+    m_edit->setPlainText(QString::fromStdString(file.readAll().toStdString()));
+  }
 }
 
 void GameConfigEdit::SaveFile()

Source/Core/DolphinQt/Config/GameConfigWidget.cpp

@@ -17,6 +17,7 @@
 #include "Common/Config/Config.h"
 #include "Common/Config/Layer.h"
 #include "Common/FileUtil.h"
+#include "Common/IniFile.h"
 
 #include "Core/Config/GraphicsSettings.h"
 #include "Core/Config/MainSettings.h"
@@ -39,13 +40,29 @@
 static void PopulateTab(QTabWidget* tab, const std::string& path, std::string& game_id,
                         u16 revision, bool read_only)
 {
-  for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision))
+  if (read_only)
   {
-    const std::string ini_path = path + filename;
+    auto& sys_inis = ConfigLoaders::GetDefaultGameSettings();
-    if (File::Exists(ini_path))
+    for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision))
     {
-      auto* edit = new GameConfigEdit(nullptr, QString::fromStdString(ini_path), read_only);
+      if (auto content = sys_inis.Get(filename))
-      tab->addTab(edit, QString::fromStdString(filename));
+      {
+        auto* edit =
+            new GameConfigEdit(nullptr, QString::fromStdString(std::string(*content)), read_only);
+        tab->addTab(edit, QString::fromStdString(filename));
+      }
+    }
+  }
+  else
+  {
+    for (const std::string& filename : ConfigLoaders::GetGameIniFilenames(game_id, revision))
+    {
+      const std::string ini_path = path + filename;
+      if (File::Exists(ini_path))
+      {
+        auto* edit = new GameConfigEdit(nullptr, QString::fromStdString(ini_path), read_only);
+        tab->addTab(edit, QString::fromStdString(filename));
+      }
     }
   }
 }
@@ -65,8 +82,7 @@ GameConfigWidget::GameConfigWidget(const UICommon::GameFile& game) : m_game(game
   CreateWidgets();
   connect(&Settings::Instance(), &Settings::ConfigChanged, this, &GameConfigWidget::LoadSettings);
 
-  PopulateTab(m_default_tab, File::GetSysDirectory() + GAMESETTINGS_DIR DIR_SEP, m_game_id,
+  PopulateTab(m_default_tab, "", m_game_id, m_game.GetRevision(), true);
-              m_game.GetRevision(), true);
   PopulateTab(m_local_tab, File::GetUserPath(D_GAMESETTINGS_IDX), m_game_id, m_game.GetRevision(),
               false);
 

Tools/build-default-settings.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# We have over 1700 ini files now.
+# Extracting/updating them all is very slow on Windows.
+# This script compresses them all into a single file during the build.
+
+from glob import glob
+from zstandard import ZstdCompressor
+
+raw = b''
+for i in glob('*.ini'):
+  raw += i.encode('ascii') + b'\0'
+  ini = open(i, 'rb').read()
+  assert b'\0' not in ini
+  raw += ini + b'\0'
+
+cooked = ZstdCompressor(19).compress(raw)
+open('default.bin.zstd', 'wb+').write(cooked)