Merge branch 'navmeshtool_binary_log' into 'master'

Send status of navmeshtool to launcher using type binary messages

See merge request OpenMW/openmw!1630

apps/launcher/datafilespage.cpp

@@ -20,11 +20,73 @@
 #include <components/config/gamesettings.hpp>
 #include <components/config/launchersettings.hpp>
 
+#include <components/navmeshtool/protocol.hpp>
+
 #include "utils/textinputdialog.hpp"
 
-
 const char *Launcher::DataFilesPage::mDefaultContentListName = "Default";
 
+namespace Launcher
+{
+    namespace
+    {
+        struct HandleNavMeshToolMessage
+        {
+            int mCellsCount;
+            int mExpectedMaxProgress;
+            int mMaxProgress;
+            int mProgress;
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ExpectedCells&& message) const
+            {
+                return HandleNavMeshToolMessage {
+                    static_cast<int>(message.mCount),
+                    mExpectedMaxProgress,
+                    static_cast<int>(message.mCount) * 100,
+                    mProgress
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ProcessedCells&& message) const
+            {
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    mExpectedMaxProgress,
+                    mMaxProgress,
+                    std::max(mProgress, static_cast<int>(message.mCount))
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::ExpectedTiles&& message) const
+            {
+                const int expectedMaxProgress = mCellsCount + static_cast<int>(message.mCount);
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    expectedMaxProgress,
+                    std::max(mMaxProgress, expectedMaxProgress),
+                    mProgress
+                };
+            }
+
+            HandleNavMeshToolMessage operator()(NavMeshTool::GeneratedTiles&& message) const
+            {
+                int progress = mCellsCount + static_cast<int>(message.mCount);
+                if (mExpectedMaxProgress < mMaxProgress)
+                    progress += static_cast<int>(std::round(
+                        (mMaxProgress - mExpectedMaxProgress)
+                        * (static_cast<float>(progress) / static_cast<float>(mExpectedMaxProgress))
+                    ));
+                return HandleNavMeshToolMessage {
+                    mCellsCount,
+                    mExpectedMaxProgress,
+                    mMaxProgress,
+                    std::max(mProgress, progress)
+                };
+            }
+        };
+    }
+}
+
 Launcher::DataFilesPage::DataFilesPage(Files::ConfigurationManager &cfg, Config::GameSettings &gameSettings,
                                        Config::LauncherSettings &launcherSettings, MainDialog *parent)
     : QWidget(parent)
@@ -95,8 +157,8 @@ void Launcher::DataFilesPage::buildView()
     connect(ui.updateNavMeshButton, SIGNAL(clicked()), this, SLOT(startNavMeshTool()));
     connect(ui.cancelNavMeshButton, SIGNAL(clicked()), this, SLOT(killNavMeshTool()));
 
-    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardOutput()), this, SLOT(updateNavMeshProgress()));
-    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardError()), this, SLOT(updateNavMeshProgress()));
+    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardOutput()), this, SLOT(readNavMeshToolStdout()));
+    connect(mNavMeshToolInvoker->getProcess(), SIGNAL(readyReadStandardError()), this, SLOT(readNavMeshToolStderr()));
     connect(mNavMeshToolInvoker->getProcess(), SIGNAL(finished(int, QProcess::ExitStatus)), this, SLOT(navMeshToolFinished(int, QProcess::ExitStatus)));
 }
 
@@ -429,7 +491,9 @@ void Launcher::DataFilesPage::startNavMeshTool()
     ui.navMeshProgressBar->setValue(0);
     ui.navMeshProgressBar->setMaximum(1);
 
-    if (!mNavMeshToolInvoker->startProcess(QLatin1String("openmw-navmeshtool")))
+    mNavMeshToolProgress = NavMeshToolProgress {};
+
+    if (!mNavMeshToolInvoker->startProcess(QLatin1String("openmw-navmeshtool"), QStringList({"--write-binary-log"})))
         return;
 
     ui.cancelNavMeshButton->setEnabled(true);
@@ -441,39 +505,60 @@ void Launcher::DataFilesPage::killNavMeshTool()
     mNavMeshToolInvoker->killProcess();
 }
 
-void Launcher::DataFilesPage::updateNavMeshProgress()
+void Launcher::DataFilesPage::readNavMeshToolStderr()
+{
+    updateNavMeshProgress(4096);
+}
+
+void Launcher::DataFilesPage::updateNavMeshProgress(int minDataSize)
 {
     QProcess& process = *mNavMeshToolInvoker->getProcess();
-    QString text;
-    while (process.canReadLine())
-    {
-        const QByteArray line = process.readLine();
-        const auto end = std::find_if(line.rbegin(), line.rend(), [] (auto v) { return v != '\n' && v != '\r'; });
-        text = QString::fromUtf8(line.mid(0, line.size() - (end - line.rbegin())));
-        ui.navMeshLogPlainTextEdit->appendPlainText(text);
-    }
-    const QRegularExpression pattern(R"([\( ](\d+)/(\d+)[\) ])");
-    QRegularExpressionMatch match = pattern.match(text);
-    if (!match.hasMatch())
+    mNavMeshToolProgress.mMessagesData.append(process.readAllStandardError());
+    if (mNavMeshToolProgress.mMessagesData.size() < minDataSize)
         return;
-    int value = match.captured(1).toInt();
-    const int maximum = match.captured(2).toInt();
-    if (text.contains("cell"))
-        ui.navMeshProgressBar->setMaximum(maximum * 100);
-    else if (maximum > ui.navMeshProgressBar->maximum())
-        ui.navMeshProgressBar->setMaximum(maximum);
-    else
-        value += static_cast<int>(std::round(
-            (ui.navMeshProgressBar->maximum() - maximum)
-            * (static_cast<float>(value) / static_cast<float>(maximum))
-        ));
-    ui.navMeshProgressBar->setValue(value);
+    const std::byte* const begin = reinterpret_cast<const std::byte*>(mNavMeshToolProgress.mMessagesData.constData());
+    const std::byte* const end = begin + mNavMeshToolProgress.mMessagesData.size();
+    const std::byte* position = begin;
+    HandleNavMeshToolMessage handle {
+        mNavMeshToolProgress.mCellsCount,
+        mNavMeshToolProgress.mExpectedMaxProgress,
+        ui.navMeshProgressBar->maximum(),
+        ui.navMeshProgressBar->value(),
+    };
+    while (true)
+    {
+        NavMeshTool::Message message;
+        const std::byte* const nextPosition = NavMeshTool::deserialize(position, end, message);
+        if (nextPosition == position)
+            break;
+        position = nextPosition;
+        handle = std::visit(handle, NavMeshTool::decode(message));
+    }
+    if (position != begin)
+        mNavMeshToolProgress.mMessagesData = mNavMeshToolProgress.mMessagesData.mid(position - begin);
+    mNavMeshToolProgress.mCellsCount = handle.mCellsCount;
+    mNavMeshToolProgress.mExpectedMaxProgress = handle.mExpectedMaxProgress;
+    ui.navMeshProgressBar->setMaximum(handle.mMaxProgress);
+    ui.navMeshProgressBar->setValue(handle.mProgress);
+}
+
+void Launcher::DataFilesPage::readNavMeshToolStdout()
+{
+    QProcess& process = *mNavMeshToolInvoker->getProcess();
+    QByteArray& logData = mNavMeshToolProgress.mLogData;
+    logData.append(process.readAllStandardOutput());
+    const int lineEnd = logData.lastIndexOf('\n');
+    if (lineEnd == -1)
+        return;
+    const int size = logData.size() >= lineEnd && logData[lineEnd - 1] == '\r' ? lineEnd - 1 : lineEnd;
+    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(logData.data(), size));
+    logData = logData.mid(lineEnd + 1);
 }
 
 void Launcher::DataFilesPage::navMeshToolFinished(int exitCode, QProcess::ExitStatus exitStatus)
 {
-    updateNavMeshProgress();
-    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(mNavMeshToolInvoker->getProcess()->readAll()));
+    updateNavMeshProgress(0);
+    ui.navMeshLogPlainTextEdit->appendPlainText(QString::fromUtf8(mNavMeshToolInvoker->getProcess()->readAllStandardOutput()));
     if (exitCode == 0 && exitStatus == QProcess::ExitStatus::NormalExit)
         ui.navMeshProgressBar->setValue(ui.navMeshProgressBar->maximum());
     ui.cancelNavMeshButton->setEnabled(false);

apps/launcher/datafilespage.hpp

@@ -73,7 +73,8 @@ namespace Launcher
 
         void startNavMeshTool();
         void killNavMeshTool();
-        void updateNavMeshProgress();
+        void readNavMeshToolStdout();
+        void readNavMeshToolStderr();
         void navMeshToolFinished(int exitCode, QProcess::ExitStatus exitStatus);
 
     public:
@@ -81,6 +82,14 @@ namespace Launcher
         const static char *mDefaultContentListName;
 
     private:
+        struct NavMeshToolProgress
+        {
+            QByteArray mLogData;
+            QByteArray mMessagesData;
+            std::map<std::uint64_t, std::string> mWorldspaces;
+            int mCellsCount = 0;
+            int mExpectedMaxProgress = 0;
+        };
 
         MainDialog *mMainDialog;
         TextInputDialog *mNewProfileDialog;
@@ -96,6 +105,7 @@ namespace Launcher
         QString mDataLocal;
 
         Process::ProcessInvoker* mNavMeshToolInvoker;
+        NavMeshToolProgress mNavMeshToolProgress;
 
         void buildView();
         void setProfile (int index, bool savePrevious);
@@ -107,6 +117,7 @@ namespace Launcher
         void populateFileViews(const QString& contentModelName);
         void reloadCells(QStringList selectedFiles);
         void refreshDataFilesView ();
+        void updateNavMeshProgress(int minDataSize);
 
         class PathIterator
         {

apps/launcher/main.cpp

@@ -4,6 +4,8 @@
 #include <QTextCodec>
 #include <QDir>
 
+#include <components/debug/debugging.hpp>
+
 #ifdef MAC_OS_X_VERSION_MIN_REQUIRED
 #undef MAC_OS_X_VERSION_MIN_REQUIRED
 // We need to do this because of Qt: https://bugreports.qt-project.org/browse/QTBUG-22154
@@ -12,7 +14,7 @@
 
 #include "maindialog.hpp"
 
-int main(int argc, char *argv[])
+int runLauncher(int argc, char *argv[])
 {
     try
     {
@@ -49,3 +51,8 @@ int main(int argc, char *argv[])
         return 0;
     }
 }
+
+int main(int argc, char *argv[])
+{
+    return wrapApplication(runLauncher, argc, argv, "Launcher");
+}

apps/navmeshtool/main.cpp

@@ -31,6 +31,11 @@
 #include <thread>
 #include <vector>
 
+#ifdef WIN32
+#include <fcntl.h>
+#include <io.h>
+#endif
+
 namespace NavMeshTool
 {
     namespace
@@ -86,6 +91,9 @@ namespace NavMeshTool
 
                 ("remove-unused-tiles", bpo::value<bool>()->implicit_value(true)
                     ->default_value(false), "remove tiles from cache that will not be used with current content profile")
+
+                ("write-binary-log", bpo::value<bool>()->implicit_value(true)
+                    ->default_value(false), "write progress in binary messages to be consumed by the launcher")
             ;
             Files::ConfigurationManager::addCommonOptions(result);
 
@@ -145,6 +153,12 @@ namespace NavMeshTool
 
             const bool processInteriorCells = variables["process-interior-cells"].as<bool>();
             const bool removeUnusedTiles = variables["remove-unused-tiles"].as<bool>();
+            const bool writeBinaryLog = variables["write-binary-log"].as<bool>();
+
+#ifdef WIN32
+            if (writeBinaryLog)
+                _setmode(_fileno(stderr), _O_BINARY);
+#endif
 
             Fallback::Map::init(variables["fallback"].as<Fallback::FallbackMap>().mMap);
 
@@ -180,10 +194,10 @@ namespace NavMeshTool
             navigatorSettings.mRecast.mSwimHeightScale = EsmLoader::getGameSetting(esmData.mGameSettings, "fSwimHeightScale").getFloat();
 
             WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager,
-                                                            esmData, processInteriorCells);
+                                                            esmData, processInteriorCells, writeBinaryLog);
 
             generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, removeUnusedTiles,
-                                    cellsData, std::move(db));
+                                    writeBinaryLog, cellsData, std::move(db));
 
             Log(Debug::Info) << "Done";
 

apps/navmeshtool/navmesh.cpp

@@ -15,6 +15,8 @@
 #include <components/misc/progressreporter.hpp>
 #include <components/sceneutil/workqueue.hpp>
 #include <components/sqlite3/transaction.hpp>
+#include <components/debug/debugging.hpp>
+#include <components/navmeshtool/protocol.hpp>
 
 #include <osg/Vec3f>
 
@@ -51,6 +53,18 @@ namespace NavMeshTool
                 << "%) navmesh tiles are generated";
         }
 
+        template <class T>
+        void serializeToStderr(const T& value)
+        {
+            const std::vector<std::byte> data = serialize(value);
+            getLockedRawStderr()->write(reinterpret_cast<const char*>(data.data()), static_cast<std::streamsize>(data.size()));
+        }
+
+        void logGeneratedTilesMessage(std::size_t number)
+        {
+            serializeToStderr(GeneratedTiles {static_cast<std::uint64_t>(number)});
+        }
+
         struct LogGeneratedTiles
         {
             void operator()(std::size_t provided, std::size_t expected) const
@@ -64,9 +78,10 @@ namespace NavMeshTool
         public:
             std::atomic_size_t mExpected {0};
 
-            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles)
+            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles, bool writeBinaryLog)
                 : mDb(std::move(db))
                 , mRemoveUnusedTiles(removeUnusedTiles)
+                , mWriteBinaryLog(writeBinaryLog)
                 , mTransaction(mDb.startTransaction(Sqlite3::TransactionMode::Immediate))
                 , mNextTileId(mDb.getMaxTileId() + 1)
                 , mNextShapeId(mDb.getMaxShapeId() + 1)
@@ -178,6 +193,8 @@ namespace NavMeshTool
                     }
                 }
                 logGeneratedTiles(mProvided, mExpected);
+                if (mWriteBinaryLog)
+                    logGeneratedTilesMessage(mProvided);
                 return !mCancelled;
             }
 
@@ -211,6 +228,7 @@ namespace NavMeshTool
             mutable std::mutex mMutex;
             NavMeshDb mDb;
             const bool mRemoveUnusedTiles;
+            const bool mWriteBinaryLog;
             Transaction mTransaction;
             TileId mNextTileId;
             std::condition_variable mHasTile;
@@ -223,18 +241,20 @@ namespace NavMeshTool
                 const std::size_t provided = mProvided.fetch_add(1, std::memory_order_relaxed) + 1;
                 mReporter(provided, mExpected);
                 mHasTile.notify_one();
+                if (mWriteBinaryLog)
+                    logGeneratedTilesMessage(provided);
             }
         };
     }
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings,
-        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data,
+        std::size_t threadsNumber, bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& data,
         NavMeshDb&& db)
     {
         Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
 
         SceneUtil::WorkQueue workQueue(threadsNumber);
-        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles);
+        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles, writeBinaryLog);
         std::size_t tiles = 0;
         std::mt19937_64 random;
 
@@ -256,6 +276,9 @@ namespace NavMeshTool
 
             tiles += worldspaceTiles.size();
 
+            if (writeBinaryLog)
+                serializeToStderr(ExpectedTiles {static_cast<std::uint64_t>(tiles)});
+
             navMeshTileConsumer->mExpected = tiles;
 
             std::shuffle(worldspaceTiles.begin(), worldspaceTiles.end(), random);

apps/navmeshtool/navmesh.hpp

@@ -16,7 +16,7 @@ namespace NavMeshTool
     struct WorldspaceData;
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const DetourNavigator::Settings& settings,
-        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& cellsData,
+        std::size_t threadsNumber, bool removeUnusedTiles, bool writeBinaryLog, WorldspaceData& cellsData,
         DetourNavigator::NavMeshDb&& db);
 }
 

apps/navmeshtool/worldspacedata.cpp

@@ -18,6 +18,8 @@
 #include <components/resource/bulletshapemanager.hpp>
 #include <components/settings/settings.hpp>
 #include <components/vfs/manager.hpp>
+#include <components/debug/debugging.hpp>
+#include <components/navmeshtool/protocol.hpp>
 
 #include <LinearMath/btVector3.h>
 
@@ -214,6 +216,13 @@ namespace NavMeshTool
             surface.mSize = static_cast<std::size_t>(ESM::Land::LAND_SIZE);
             return {surface, landData.mMinHeight, landData.mMaxHeight};
         }
+
+        template <class T>
+        void serializeToStderr(const T& value)
+        {
+            const std::vector<std::byte> data = serialize(value);
+            getRawStderr().write(reinterpret_cast<const char*>(data.data()), static_cast<std::streamsize>(data.size()));
+        }
     }
 
     WorldspaceNavMeshInput::WorldspaceNavMeshInput(std::string worldspace, const DetourNavigator::RecastSettings& settings)
@@ -226,7 +235,7 @@ namespace NavMeshTool
 
     WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
-        bool processInteriorCells)
+        bool processInteriorCells, bool writeBinaryLog)
     {
         Log(Debug::Info) << "Processing " << esmData.mCells.size() << " cells...";
 
@@ -235,6 +244,9 @@ namespace NavMeshTool
 
         std::size_t objectsCounter = 0;
 
+        if (writeBinaryLog)
+            serializeToStderr(ExpectedCells {static_cast<std::uint64_t>(esmData.mCells.size())});
+
         for (std::size_t i = 0; i < esmData.mCells.size(); ++i)
         {
             const ESM::Cell& cell = esmData.mCells[i];
@@ -242,6 +254,8 @@ namespace NavMeshTool
 
             if (!exterior && !processInteriorCells)
             {
+                if (writeBinaryLog)
+                    serializeToStderr(ProcessedCells {static_cast<std::uint64_t>(i + 1)});
                 Log(Debug::Info) << "Skipped interior"
                     << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") \"" << cell.getDescription() << "\"";
                 continue;
@@ -311,8 +325,13 @@ namespace NavMeshTool
                     data.mObjects.emplace_back(std::move(object));
                 });
 
+            const auto cellDescription = cell.getDescription();
+
+            if (writeBinaryLog)
+                serializeToStderr(ProcessedCells {static_cast<std::uint64_t>(i + 1)});
+
             Log(Debug::Info) << "Processed " << (exterior ? "exterior" : "interior")
-                << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") " << cell.getDescription()
+                << " cell (" << (i + 1) << "/" << esmData.mCells.size() << ") " << cellDescription
                 << " with " << (data.mObjects.size() - cellObjectsBegin) << " objects";
         }
 

apps/navmeshtool/worldspacedata.hpp

@@ -91,7 +91,7 @@ namespace NavMeshTool
 
     WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
         const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
-        bool processInteriorCells);
+        bool processInteriorCells, bool writeBinaryLog);
 }
 
 #endif

apps/openmw_test_suite/serialization/binaryreader.cpp

@@ -64,4 +64,27 @@ namespace
         const TestFormat<Mode::Read> format;
         EXPECT_THROW(binaryReader(format, values.data(), values.size()), std::runtime_error);
     }
+
+    TEST(DetourNavigatorSerializationBinaryReaderTest, shouldSetPointerToCurrentBufferPosition)
+    {
+        std::vector<std::byte> data(8);
+        BinaryReader binaryReader(data.data(), data.data() + data.size());
+        const std::byte* ptr = nullptr;
+        const TestFormat<Mode::Read> format;
+        binaryReader(format, ptr);
+        EXPECT_EQ(ptr, data.data());
+    }
+
+    TEST(DetourNavigatorSerializationBinaryReaderTest, shouldNotAdvanceAfterPointer)
+    {
+        std::vector<std::byte> data(8);
+        BinaryReader binaryReader(data.data(), data.data() + data.size());
+        const std::byte* ptr1 = nullptr;
+        const std::byte* ptr2 = nullptr;
+        const TestFormat<Mode::Read> format;
+        binaryReader(format, ptr1);
+        binaryReader(format, ptr2);
+        EXPECT_EQ(ptr1, data.data());
+        EXPECT_EQ(ptr2, data.data());
+    }
 }