Run Lua integration tests starting with menu script

This allows writing tests for menu scripts.

Keep global script as entry point to morrowind tests.

Fix menu.newGame and menu.loadGame to hide main menu.

scripts/data/integration_tests/test_lua_api/global.lua

@@ -7,7 +7,7 @@ local vfs = require('openmw.vfs')
 local world = require('openmw.world')
 local I = require('openmw.interfaces')
 
-testing.registerGlobalTest('testTimers', function()
+testing.registerGlobalTest('timers', function()
     testing.expectAlmostEqual(core.getGameTimeScale(), 30, 'incorrect getGameTimeScale() result')
     testing.expectAlmostEqual(core.getSimulationTimeScale(), 1, 'incorrect getSimulationTimeScale result')
 
@@ -41,7 +41,7 @@ testing.registerGlobalTest('testTimers', function()
     testing.expectGreaterOrEqual(ts2, 1, 'async:newUnsavableSimulationTimer failed')
 end)
 
-testing.registerGlobalTest('testTeleport', function()
+testing.registerGlobalTest('teleport', function()
     local player = world.players[1]
     player:teleport('', util.vector3(100, 50, 500), util.transform.rotateZ(math.rad(90)))
     coroutine.yield()
@@ -74,14 +74,14 @@ testing.registerGlobalTest('testTeleport', function()
     testing.expectEqualWithDelta(player.rotation:getYaw(), math.rad(-90), 0.05, 'teleporting changes rotation')
 end)
 
-testing.registerGlobalTest('testGetGMST', function()
+testing.registerGlobalTest('getGMST', function()
     testing.expectEqual(core.getGMST('non-existed gmst'), nil)
     testing.expectEqual(core.getGMST('Water_RippleFrameCount'), 4)
     testing.expectEqual(core.getGMST('Inventory_DirectionalDiffuseR'), 0.5)
     testing.expectEqual(core.getGMST('Level_Up_Level2'), 'something')
 end)
 
-testing.registerGlobalTest('testMWScript', function()
+testing.registerGlobalTest('MWScript', function()
     local variableStoreCount = 18
     local variableStore = world.mwscript.getGlobalVariables(player)
     testing.expectEqual(variableStoreCount, #variableStore)
@@ -122,7 +122,7 @@ local function testRecordStore(store, storeName, skipPairs)
     testing.expectEqual(status, true, storeName)
 end
 
-testing.registerGlobalTest('testRecordStores', function()
+testing.registerGlobalTest('record stores', function()
     for key, type in pairs(types) do
         if type.records then
             testRecordStore(type, key)
@@ -143,7 +143,7 @@ testing.registerGlobalTest('testRecordStores', function()
     testRecordStore(types.Player.birthSigns, "birthSigns")
 end)
 
-testing.registerGlobalTest('testRecordCreation', function()
+testing.registerGlobalTest('record creation', function()
     local newLight = {
         isCarriable = true,
         isDynamic = true,
@@ -168,7 +168,7 @@ testing.registerGlobalTest('testRecordCreation', function()
     end
 end)
 
-testing.registerGlobalTest('testUTF8Chars', function()
+testing.registerGlobalTest('UTF-8 characters', function()
     testing.expectEqual(utf8.codepoint("😀"), 0x1F600)
 
     local chars = {}
@@ -195,7 +195,7 @@ testing.registerGlobalTest('testUTF8Chars', function()
     end
 end)
 
-testing.registerGlobalTest('testUTF8Strings', function()
+testing.registerGlobalTest('UTF-8 strings', function()
     local utf8str = "Hello, 你好, 🌎!"
 
     local str = ""
@@ -208,7 +208,7 @@ testing.registerGlobalTest('testUTF8Strings', function()
     testing.expectEqual(utf8.offset(utf8str, 9), 11)
 end)
 
-testing.registerGlobalTest('testMemoryLimit', function()
+testing.registerGlobalTest('memory limit', function()
     local ok, err = pcall(function()
         local t = {}
         local n = 1
@@ -228,7 +228,7 @@ local function initPlayer()
     return player
 end
 
-testing.registerGlobalTest('testVFS', function()
+testing.registerGlobalTest('vfs', function()
     local file = 'test_vfs_dir/lines.txt'
     local nosuchfile = 'test_vfs_dir/nosuchfile'
     testing.expectEqual(vfs.fileExists(file), true, 'lines.txt should exist')
@@ -274,9 +274,9 @@ testing.registerGlobalTest('testVFS', function()
     end
 end)
 
-testing.registerGlobalTest('testCommitCrime', function()
+testing.registerGlobalTest('commit crime', function()
     local player = initPlayer()
-    testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `testCommitCrime`')
+    testing.expectEqual(player == nil, false, 'A viable player reference should exist to run `commit crime`')
     testing.expectEqual(I.Crimes == nil, false, 'Crimes interface should be available in global contexts')
 
     -- Reset crime level to have a clean slate
@@ -296,8 +296,8 @@ testing.registerGlobalTest('testCommitCrime', function()
     testing.expectEqual(types.Player.getCrimeLevel(player), 0, "Crime level should not change if the victim's alarm value is low and there's no other witnesses")
 end)
 
-testing.registerGlobalTest('testRecordModelProperty', function()
+testing.registerGlobalTest('record model property', function()
-    local player = initPlayer()
+    local player = world.players[1]
     testing.expectEqual(types.NPC.record(player).model, 'meshes/basicplayer.dae')
 end)
 
@@ -308,27 +308,27 @@ local function registerPlayerTest(name)
     end)
 end
 
-registerPlayerTest('playerYawRotation')
+registerPlayerTest('player yaw rotation')
-registerPlayerTest('playerPitchRotation')
+registerPlayerTest('player pitch rotation')
-registerPlayerTest('playerPitchAndYawRotation')
+registerPlayerTest('player pitch and yaw rotation')
-registerPlayerTest('playerRotation')
+registerPlayerTest('player rotation')
-registerPlayerTest('playerForwardRunning')
+registerPlayerTest('player forward running')
-registerPlayerTest('playerDiagonalWalking')
+registerPlayerTest('player diagonal walking')
 registerPlayerTest('findPath')
 registerPlayerTest('findRandomPointAroundCircle')
 registerPlayerTest('castNavigationRay')
 registerPlayerTest('findNearestNavMeshPosition')
-registerPlayerTest('playerMemoryLimit')
+registerPlayerTest('player memory limit')
 
-testing.registerGlobalTest('playerWeaponAttack', function()
+testing.registerGlobalTest('player weapon attack', function()
     local player = initPlayer()
     world.createObject('basic_dagger1h', 1):moveInto(player)
-    testing.runLocalTest(player, 'playerWeaponAttack')
+    testing.runLocalTest(player, 'player weapon attack')
 end)
 
 return {
     engineHandlers = {
-        onUpdate = testing.makeUpdateGlobal(),
+        onUpdate = testing.updateGlobal,
     },
     eventHandlers = testing.globalEventHandlers,
 }

scripts/data/integration_tests/test_lua_api/menu.lua

@@ -0,0 +1,43 @@
+local testing = require('testing_util')
+local menu = require('openmw.menu')
+
+local function registerGlobalTest(name, description)
+    testing.registerMenuTest(description or name, function()
+        menu.newGame()
+        coroutine.yield()
+        testing.runGlobalTest(name)
+    end)
+end
+
+registerGlobalTest('timers')
+registerGlobalTest('teleport')
+registerGlobalTest('getGMST')
+registerGlobalTest('MWScript')
+registerGlobalTest('record stores')
+registerGlobalTest('record creation')
+registerGlobalTest('UTF-8 characters')
+registerGlobalTest('UTF-8 strings')
+registerGlobalTest('memory limit')
+registerGlobalTest('vfs')
+registerGlobalTest('commit crime')
+registerGlobalTest('record model property')
+
+registerGlobalTest('player yaw rotation', 'rotating player with controls.yawChange should change rotation')
+registerGlobalTest('player pitch rotation', 'rotating player with controls.pitchChange should change rotation')
+registerGlobalTest('player pitch and yaw rotation', 'rotating player with controls.pitchChange and controls.yawChange should change rotation')
+registerGlobalTest('player rotation', 'rotating player should not lead to nan rotation')
+registerGlobalTest('player forward running')
+registerGlobalTest('player diagonal walking')
+registerGlobalTest('findPath')
+registerGlobalTest('findRandomPointAroundCircle')
+registerGlobalTest('castNavigationRay')
+registerGlobalTest('findNearestNavMeshPosition')
+registerGlobalTest('player memory limit')
+registerGlobalTest('player weapon attack', 'player with equipped weapon on attack should damage health of other actors')
+
+return {
+    engineHandlers = {
+        onFrame = testing.makeUpdateMenu(),
+    },
+    eventHandlers = testing.menuEventHandlers,
+}

scripts/data/integration_tests/test_lua_api/openmw.cfg

@@ -1,4 +1,4 @@
-content=test.omwscripts
+content=test_lua_api.omwscripts
 
 # Needed to test `core.getGMST`
 fallback=Water_RippleFrameCount,4

scripts/data/integration_tests/test_lua_api/player.lua

@@ -40,7 +40,7 @@ local function rotateByPitch(object, target)
     rotate(object, target, nil)
 end
 
-testing.registerLocalTest('playerYawRotation',
+testing.registerLocalTest('player yaw rotation',
     function()
         local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ()
         local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX()
@@ -60,7 +60,7 @@ testing.registerLocalTest('playerYawRotation',
         testing.expectEqualWithDelta(gamma2, initialGammaZYX, 0.05, 'Gamma rotation in ZYX convention should not change')
     end)
 
-testing.registerLocalTest('playerPitchRotation',
+testing.registerLocalTest('player pitch rotation',
     function()
         local initialAlphaXZ, initialGammaXZ = self.rotation:getAnglesXZ()
         local initialAlphaZYX, initialBetaZYX, initialGammaZYX = self.rotation:getAnglesZYX()
@@ -80,7 +80,7 @@ testing.registerLocalTest('playerPitchRotation',
         testing.expectEqualWithDelta(gamma2, targetPitch, 0.05, 'Incorrect gamma rotation in ZYX convention')
     end)
 
-testing.registerLocalTest('playerPitchAndYawRotation',
+testing.registerLocalTest('player pitch and yaw rotation',
     function()
         local targetPitch = math.rad(-30)
         local targetYaw = math.rad(-60)
@@ -99,7 +99,7 @@ testing.registerLocalTest('playerPitchAndYawRotation',
         testing.expectEqualWithDelta(gamma2, math.rad(-16), 0.05, 'Incorrect gamma rotation in ZYX convention')
     end)
 
-testing.registerLocalTest('playerRotation',
+testing.registerLocalTest('player rotation',
     function()
         local rotation = math.sqrt(2)
         local endTime = core.getSimulationTime() + 3
@@ -123,7 +123,7 @@ testing.registerLocalTest('playerRotation',
         end
     end)
 
-testing.registerLocalTest('playerForwardRunning',
+testing.registerLocalTest('player forward running',
     function()
         local startPos = self.position
         local endTime = core.getSimulationTime() + 1
@@ -141,7 +141,7 @@ testing.registerLocalTest('playerForwardRunning',
         testing.expectEqualWithDelta(direction.y, 1, 0.1, 'Run forward, Y coord')
     end)
 
-testing.registerLocalTest('playerDiagonalWalking',
+testing.registerLocalTest('player diagonal walking',
     function()
         local startPos = self.position
         local endTime = core.getSimulationTime() + 1
@@ -220,7 +220,7 @@ testing.registerLocalTest('findNearestNavMeshPosition',
             'Navigation mesh position ' .. testing.formatActualExpected(result, expected))
     end)
 
-testing.registerLocalTest('playerMemoryLimit',
+testing.registerLocalTest('player memory limit',
     function()
         local ok, err = pcall(function()
             local str = 'a'
@@ -232,7 +232,7 @@ testing.registerLocalTest('playerMemoryLimit',
         testing.expectEqual(err, 'not enough memory')
     end)
 
-testing.registerLocalTest('playerWeaponAttack',
+testing.registerLocalTest('player weapon attack',
     function()
         camera.setMode(camera.MODE.ThirdPerson)
 

scripts/data/integration_tests/test_lua_api/test.omwscripts

@@ -1,2 +0,0 @@
-GLOBAL: test.lua
-PLAYER: player.lua

scripts/data/integration_tests/test_lua_api/test_lua_api.omwscripts

@@ -0,0 +1,3 @@
+MENU: menu.lua
+GLOBAL: global.lua
+PLAYER: player.lua

scripts/data/integration_tests/testing_util/testing_util.lua

@@ -3,29 +3,21 @@ local util = require('openmw.util')
 
 local M = {}
 
+local menuTestsOrder = {}
+local menuTests = {}
+
 local globalTestsOrder = {}
 local globalTests = {}
 local globalTestRunner = nil
+local currentGlobalTest = nil
+local currentGlobalTestError = nil
 
 local localTests = {}
 local localTestRunner = nil
 local currentLocalTest = nil
 local currentLocalTestError = nil
 
-function M.makeUpdateGlobal()
+local function makeTestCoroutine(fn)
-    local fn = function()
-        for i, test in ipairs(globalTestsOrder) do
-            local name, fn = unpack(test)
-            print('TEST_START', i, name)
-            local status, err = pcall(fn)
-            if status then
-                print('TEST_OK', i, name)
-            else
-                print('TEST_FAILED', i, name, err)
-            end
-        end
-        core.quit()
-    end
     local co = coroutine.create(fn)
     return function()
         if coroutine.status(co) ~= 'dead' then
@@ -34,11 +26,64 @@ function M.makeUpdateGlobal()
     end
 end
 
+local function runTests(tests)
+    for i, test in ipairs(tests) do
+        local name, fn = unpack(test)
+        print('TEST_START', i, name)
+        local status, err = pcall(fn)
+        if status then
+            print('TEST_OK', i, name)
+        else
+            print('TEST_FAILED', i, name, err)
+        end
+    end
+    core.quit()
+end
+
+function M.makeUpdateMenu()
+    return makeTestCoroutine(function()
+        print('Running menu tests...')
+        runTests(menuTestsOrder)
+    end)
+end
+
+function M.makeUpdateGlobal()
+    return makeTestCoroutine(function()
+        print('Running global tests...')
+        runTests(globalTestsOrder)
+    end)
+end
+
+function M.registerMenuTest(name, fn)
+    menuTests[name] = fn
+    table.insert(menuTestsOrder, {name, fn})
+end
+
+function M.runGlobalTest(name)
+    currentGlobalTest = name
+    currentGlobalTestError = nil
+    core.sendGlobalEvent('runGlobalTest', name)
+    while currentGlobalTest do
+        coroutine.yield()
+    end
+    if currentGlobalTestError then
+        error(currentGlobalTestError, 2)
+    end
+end
+
 function M.registerGlobalTest(name, fn)
     globalTests[name] = fn
     table.insert(globalTestsOrder, {name, fn})
 end
 
+function M.updateGlobal()
+    if globalTestRunner and coroutine.status(globalTestRunner) ~= 'dead' then
+        coroutine.resume(globalTestRunner)
+    else
+        globalTestRunner = nil
+    end
+end
+
 function M.runLocalTest(obj, name)
     currentLocalTest = name
     currentLocalTestError = nil
@@ -208,11 +253,38 @@ function M.formatActualExpected(actual, expected)
     return string.format('actual: %s, expected: %s', actual, expected)
 end
 
+-- used only in menu scripts
+M.menuEventHandlers = {
+    globalTestFinished = function(data)
+        if data.name ~= currentGlobalTest then
+            error(string.format('globalTestFinished with incorrect name %s, expected %s', data.name, currentGlobalTest), 2)
+        end
+        currentGlobalTest = nil
+        currentGlobalTestError = data.errMsg
+    end,
+}
+
 -- used only in global scripts
 M.globalEventHandlers = {
+    runGlobalTest = function(name)
+        fn = globalTests[name]
+        local types = require('openmw.types')
+        local world = require('openmw.world')
+        if not fn then
+            types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg='Global test is not found'})
+            return
+        end
+        globalTestRunner = coroutine.create(function()
+            local status, err = pcall(fn)
+            if status then
+                err = nil
+            end
+            types.Player.sendMenuEvent(world.players[1], 'globalTestFinished', {name=name, errMsg=err})
+        end)
+    end,
     localTestFinished = function(data)
         if data.name ~= currentLocalTest then
-            error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest))
+            error(string.format('localTestFinished with incorrect name %s, expected %s', data.name, currentLocalTest), 2)
         end
         currentLocalTest = nil
         currentLocalTestError = data.errMsg

scripts/integration_tests.py

@@ -83,7 +83,7 @@ def run_test(test_name):
     test_success = True
     fatal_errors = list()
     with subprocess.Popen(
-        [openmw_binary, "--replace=config", "--config", config_dir, "--skip-menu", "--no-grab", "--no-sound"],
+        [openmw_binary, "--replace=config", "--config", config_dir, "--no-grab"],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         encoding="utf-8",