Merge 305ea4a00e into fbbfc07ff1

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "OTRExporter"]
 	path = OTRExporter
 	url = https://github.com/harbourmasters/OTRExporter
+[submodule "miniaudio"]
+	path = soh/include/miniaudio
+	url = https://github.com/mackron/miniaudio

run-clang-format.ps1

@@ -36,7 +36,8 @@ if (-not (Test-Path $clangFormatFilePath) -or ($currentVersion -ne $requiredVers
 
 $basePath = (Resolve-Path .).Path
 $files = Get-ChildItem -Path $basePath\soh -Recurse -File `
-    | Where-Object { ($_.Extension -eq '.c' -or $_.Extension -eq '.cpp' -or `
+    | Where-Object { (-not ($_.FullName -like "*\soh\include\miniaudio*")) -and `
+                     ($_.Extension -eq '.c' -or $_.Extension -eq '.cpp' -or `
                       (($_.Extension -eq '.h' -or $_.Extension -eq '.hpp') -and `
                        (-not ($_.FullName -like "*\soh\src\*" -or $_.FullName -like "*\soh\include\*")))) -and `
                      (-not ($_.FullName -like "*\soh\assets\*")) }

run-clang-format.sh

@@ -26,4 +26,4 @@
 # and pass it as an argument to clang-format
 # verbose to print files being formatted and X out of Y status
 
-find soh -type f \( -name "*.c" -o -name "*.cpp" -o \( \( -name "*.h" -o -name "*.hpp" \) ! -path "soh/src/*" ! -path "soh/include/*" \) \) ! -path "soh/assets/*" -print0 | xargs -0 clang-format-14 -i --verbose
+find soh -type f \( ! -path "soh/include/miniaudio/*" -a \( -name "*.c" -o -name "*.cpp" -o \( \( -name "*.h" -o -name "*.hpp" \) ! -path "soh/src/*" ! -path "soh/include/*" \) \) \) ! -path "soh/assets/*" -print0 | xargs -0 clang-format-14 -i --verbose

soh/CMakeLists.txt

@@ -153,6 +153,11 @@ else()
     list(FILTER soh__ EXCLUDE REGEX "soh/Enhancements/speechsynthesizer/(Darwin|SAPI).*")
 endif()
 
+# handle accessible audio engine removals
+if (CMAKE_SYSTEM_NAME MATCHES "NintendoSwitch|CafeOS")
+    list(FILTER soh__Enhancements EXCLUDE REGEX "soh/Enhancements/accessible-actors/")
+endif()
+
 # soh/Extractor {{{
 list(FILTER ship__ EXCLUDE REGEX "soh/Extractor/*")
 

soh/include/miniaudio

@@ -0,0 +1 @@
+Subproject commit 350784a9467a79d0fa65802132668e5afbcf3777

soh/soh/Enhancements/accessible-actors/AccessibleAudioEngine.cpp

@@ -0,0 +1,599 @@
+#define AAE_CHANNELS 2
+#define AAE_SAMPLE_RATE 44100
+#define AAE_MAX_BUFFER_SIZE AAE_SAMPLE_RATE / 10
+#define AAE_PREP_CHUNK_SIZE 64
+#define AAE_MIX_CHUNK_SIZE 64
+#define AAE_GC_INTERVAL 20 * 60 // How often, in frames, do we clean up sound handles that are no longer active.
+#define AAE_MAX_DB_REDUCTION -20
+#define AAE_LPF_ORDER 4
+
+#define NOMINMAX // because Windows is a joke.
+#define MINIAUDIO_IMPLEMENTATION
+#include "AccessibleAudioEngine.h"
+
+extern "C" {
+int AudioPlayer_GetDesiredBuffered();
+}
+#include <math.h>
+#include <algorithm>
+#include <stdexcept>
+#include <spdlog/spdlog.h>
+
+enum AAE_COMMANDS {
+    AAE_START = 0,
+    AAE_STOP,
+    AAE_STOP_ALL,
+    AAE_PITCH,
+    AAE_PITCH_BEHIND, // Specify how much to change the pitch when the sound is behind the listener.
+    AAE_VOLUME,
+    AAE_PAN,
+    AAE_FILTER,
+    AAE_SEEK,
+    AAE_POS,
+    AAE_PREPARE,
+    AAE_TERMINATE,
+};
+typedef int8_t s8;
+typedef uint8_t u8;
+// Processing for our custom audio positioning.
+static float lerp_aae(float x, float y, float z) {
+    return (1.0 - z) * x + z * y;
+}
+
+static float computeGain(SoundExtras* extras) {
+    if (extras->maxDistance == 0)
+        return 0;
+    float leftover = ma_volume_db_to_linear(AAE_MAX_DB_REDUCTION);
+    float normDist = fabs(extras->distToPlayer) / extras->maxDistance;
+    float db = lerp_aae(0, AAE_MAX_DB_REDUCTION, normDist);
+    float gain = ma_volume_db_to_linear(db);
+    gain -= lerp_aae(0, leftover, normDist);
+    return gain;
+}
+// Borrow the pan calculation from the game itself. Todo: this is technical debt, so copy/ revise it or something at
+// some point.
+extern "C" int8_t Audio_ComputeSoundPanSigned(float x, float z, uint8_t token);
+static void positioner_process_pcm_frames(ma_node* pNode, const float** ppFramesIn, ma_uint32* pFrameCountIn,
+                                          float** ppFramesOut, ma_uint32* pFrameCountOut) {
+
+    const float* framesIn = ppFramesIn[0];
+    float* framesOut = ppFramesOut[0];
+    ma_copy_pcm_frames(framesOut, framesIn, *pFrameCountIn, ma_format_f32, 2);
+    *pFrameCountOut = *pFrameCountIn;
+    SoundExtras* extras = (SoundExtras*)pNode;
+    // Pan the sound based on its projected position.
+    float pan;
+    // Use the game's panning mechanism, which returns a signed 8-bit integer between 0 (far-left) and 127 (far-right).
+    // It would appear that the correct thing to do is interpret this value as a gain factor in decibels. In practice,
+    // values below 38 or above 90 are never seen, so a sound that's panned far to one side or the other amounts to
+    // about -25DB worth of attenuation. Also: lie about the value of Z and give it a constant value to prevent weird
+    // behaviour when Z is far away.
+    s8 panSigned = Audio_ComputeSoundPanSigned(extras->x, extras->z, 4);
+    int db;
+    if (panSigned < 64)
+        db = 64 - panSigned;
+    else
+        db = panSigned - 64;
+    pan = 1.0 - fabs(ma_volume_db_to_linear(-db / 2));
+    if (panSigned < 64)
+        pan = -pan;
+
+    ma_panner_set_pan(&extras->panner, pan);
+    ma_panner_process_pcm_frames(&extras->panner, framesOut, framesOut, *pFrameCountIn);
+    // Next we'll apply the gain based on the object's distance relationship to the player. The strategy here is to use
+    // a combination of decibel-based and linear attenuation, so that the gain reaches 0 at the exact point when the
+    // object is at exactly the maximum distance from the player.
+
+    float gain = computeGain(extras);
+    ma_gainer_set_gain(&extras->gainer, gain);
+    ma_gainer_process_pcm_frames(&extras->gainer, framesOut, framesOut, *pFrameCountIn);
+    // Run LPF only when necessary because we can't afford to run a 4th-order lowpass on every single sound. This
+    // probably causes minor glitches when the filter switches on and off. Todo: cross that bridge.
+    if (extras->cutoff != 1.0)
+        ma_lpf_process_pcm_frames(&extras->filter, framesOut, framesOut, *pFrameCountIn);
+}
+
+static ma_node_vtable positioner_vtable = { positioner_process_pcm_frames, NULL, 1, 1, 0 };
+static ma_uint32 positioner_channels[1] = { 2 };
+
+void AccessibleAudioEngine::destroy() {
+    switch (initialized) {
+        case 3:
+            ma_engine_uninit(&engine);
+        case 2:
+            ma_pcm_rb_uninit(&preparedOutput);
+        case 1:
+            ma_resource_manager_uninit(&resourceManager);
+    }
+}
+
+void AccessibleAudioEngine::destroyAndThrow(const char* exceptionText) {
+    destroy();
+    throw std::runtime_error(exceptionText);
+}
+
+uint32_t AccessibleAudioEngine::retrieve(float* buffer, uint32_t nFrames) {
+    uint32_t framesAvailable = ma_pcm_rb_available_read(&preparedOutput);
+    if (nFrames > framesAvailable)
+        nFrames = framesAvailable;
+    if (nFrames == 0)
+        return 0;
+    uint32_t ogNFrames = nFrames;
+    while (nFrames > 0) {
+        void* readBuffer;
+        uint32_t framesObtained = nFrames;
+        ma_pcm_rb_acquire_read(&preparedOutput, &framesObtained, (void**)&readBuffer);
+        if (framesObtained > nFrames)
+            framesObtained = nFrames;
+
+        memcpy(buffer, readBuffer, sizeof(float) * framesObtained * AAE_CHANNELS);
+        buffer += framesObtained * AAE_CHANNELS;
+        nFrames -= framesObtained;
+        ma_pcm_rb_commit_read(&preparedOutput, framesObtained);
+    }
+
+    return ogNFrames;
+}
+
+void AccessibleAudioEngine::doPrepare(SoundAction& action) {
+    framesUntilGC--;
+    int nFrames = ma_pcm_rb_available_write(&preparedOutput);
+    if (nFrames <= 0)
+        return;
+
+    float* chunk;
+    while (nFrames > 0) {
+        // This should not loop more than twice.
+        uint32_t nextChunk = nFrames;
+        ma_pcm_rb_acquire_write(&preparedOutput, &nextChunk,
+                                (void**)&chunk); // Might reduce nextChunk if there isn't enough buffer space available
+                                                 // to accommodate the request.
+        ma_uint64 framesRead = 0;
+        ma_engine_read_pcm_frames(&engine, chunk, nextChunk, &framesRead);
+        // Even if we get fewer frames than expected, we should still submit a full buffer of silence.
+        if (framesRead < nextChunk)
+            ma_silence_pcm_frames(chunk + (framesRead * 2), (nextChunk - framesRead), ma_format_f32, 2);
+        ma_pcm_rb_commit_write(&preparedOutput, (uint32_t)nextChunk);
+        nFrames -= nextChunk;
+    }
+}
+int AccessibleAudioEngine::getSoundActions(SoundAction* dest, int limit) {
+    std::unique_lock<std::mutex> lock(mtx);
+    while (soundActions.empty())
+        cv.wait(lock);
+    int actionsOut = 0;
+    while (!soundActions.empty() && limit > 0) {
+        dest[actionsOut] = soundActions.front();
+        soundActions.pop_front();
+        actionsOut++;
+        limit--;
+    }
+    return actionsOut;
+}
+void AccessibleAudioEngine::postSoundActions() {
+    {
+        std::scoped_lock<std::mutex> lock(mtx);
+        for (int i = 0; i < nextOutgoingSoundAction; i++)
+            soundActions.push_back(outgoingSoundActions[i]);
+    }
+    cv.notify_one();
+    nextOutgoingSoundAction = 0;
+}
+void AccessibleAudioEngine::postHighPrioritySoundAction(SoundAction& action) {
+    std::scoped_lock<std::mutex> lock(mtx);
+    soundActions.push_front(action);
+    cv.notify_one();
+}
+SoundAction& AccessibleAudioEngine::getNextOutgoingSoundAction() {
+    if (nextOutgoingSoundAction >= AAE_SOUND_ACTION_BATCH_SIZE)
+        postSoundActions();
+    nextOutgoingSoundAction++;
+    return outgoingSoundActions[nextOutgoingSoundAction - 1];
+}
+void AccessibleAudioEngine::runThread() {
+    bool shouldTerminate = false;
+    SoundAction incomingSoundActions[AAE_SOUND_ACTION_BATCH_SIZE];
+    while (true) {
+        processAudioJobs();
+        if (framesUntilGC <= 0)
+            garbageCollect();
+
+        int batchSize = getSoundActions(incomingSoundActions, AAE_SOUND_ACTION_BATCH_SIZE);
+        for (int i = 0; i < batchSize; i++) {
+            SoundAction& action = incomingSoundActions[i];
+            switch (action.command) {
+                case AAE_TERMINATE:
+                    return;
+                case AAE_START:
+                    doPlaySound(action);
+                    break;
+                case AAE_STOP:
+                    doStopSound(action);
+                    break;
+                case AAE_STOP_ALL:
+                    doStopAllSounds(action);
+                    break;
+                case AAE_PITCH:
+                    doSetPitch(action);
+                    break;
+                case AAE_PITCH_BEHIND:
+                    doSetPitchBehindModifier(action);
+                    break;
+                case AAE_VOLUME:
+                    doSetVolume(action);
+                    break;
+                case AAE_PAN:
+                    doSetPan(action);
+                    break;
+                case AAE_FILTER:
+                    doSetFilter(action);
+                    break;
+                case AAE_SEEK:
+                    doSeekSound(action);
+                    break;
+                case AAE_POS:
+                    doSetSoundPos(action);
+                    break;
+                case AAE_PREPARE:
+                    doPrepare(action);
+                    break;
+            }
+        }
+    }
+}
+
+SoundSlot* AccessibleAudioEngine::findSound(SoundAction& action) {
+    if (action.slot < 0 || action.slot >= AAE_SLOTS_PER_HANDLE)
+        return NULL;
+    auto i = sounds.find(action.handle);
+    if (i == sounds.end())
+        return NULL;
+    SoundSlot& target = i->second[action.slot];
+    if (!target.active)
+        return NULL;
+    return &target;
+}
+
+void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
+    SoundSlot* sound;
+    if (sounds.contains(action.handle)) {
+        sound = &sounds[action.handle][action.slot];
+        if (sound->active) {
+            ma_sound_stop(&sound->sound);
+            destroySound(sound);
+        }
+    } else {
+        SoundSlots temp;
+        for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++)
+            temp[i].active = false;
+
+        sounds[action.handle] = temp;
+        sound = &sounds[action.handle][action.slot];
+    }
+
+    ma_result result = ma_sound_init_from_file(&engine, action.path.c_str(),
+                                               MA_SOUND_FLAG_NO_SPATIALIZATION | MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT,
+                                               NULL, NULL, &sound->sound);
+    if (result != MA_SUCCESS) {
+        SPDLOG_ERROR("failed to play sound: {}", ma_result_description(result));
+        return;
+    }
+
+    initSoundExtras(sound);
+    ma_sound_set_looping(&sound->sound, action.looping);
+    // We actually attach the extras to the engine, not the sound itself.
+    ma_node_attach_output_bus(&sound->extras, 0, ma_node_graph_get_endpoint(&engine.nodeGraph), 0);
+
+    ma_sound_start(&sound->sound);
+
+    sound->active = true;
+}
+
+void AccessibleAudioEngine::doStopSound(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    destroySound(slot);
+}
+void AccessibleAudioEngine::doStopAllSounds(SoundAction& action) {
+    auto it = sounds.find(action.handle);
+    if (it == sounds.end())
+        return;
+    SoundSlots& slots = it->second;
+    for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++) {
+        if (slots[i].active)
+            destroySound(&slots[i]);
+    }
+    sounds.erase(it);
+}
+void AccessibleAudioEngine::doSetPitch(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.pitch = action.pitch;
+    float pitch = action.pitch;
+    if (slot->extras.z < 0)
+        pitch *= (1.0 - slot->extras.pitchBehindModifier);
+    ma_sound_set_pitch(&slot->sound, pitch);
+}
+
+void AccessibleAudioEngine::doSetPitchBehindModifier(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.pitchBehindModifier = action.pitch;
+}
+void AccessibleAudioEngine::doSetVolume(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_set_volume(&slot->sound, action.pitch);
+}
+void AccessibleAudioEngine::doSetPan(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_set_pan(&slot->sound, action.pan);
+}
+void AccessibleAudioEngine::doSetFilter(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.cutoff = action.cutoff;
+    ma_lpf_config config = ma_lpf_config_init(ma_format_f32, AAE_CHANNELS, AAE_SAMPLE_RATE,
+                                              lerp_aae(0.0, AAE_SAMPLE_RATE / 2, action.cutoff), AAE_LPF_ORDER);
+    ma_lpf_reinit(&config, &slot->extras.filter);
+}
+void AccessibleAudioEngine::doSeekSound(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    ma_sound_seek_to_pcm_frame(&slot->sound, action.offset);
+}
+void AccessibleAudioEngine::doSetSoundPos(SoundAction& action) {
+    SoundSlot* slot = findSound(action);
+    if (slot == NULL)
+        return;
+    slot->extras.x = action.posX;
+    slot->extras.y = action.posY;
+    slot->extras.z = action.posZ;
+    slot->extras.distToPlayer = action.distToPlayer;
+    slot->extras.maxDistance = action.maxDistance;
+    float pitch = slot->extras.pitch;
+    if (action.posZ < 0)
+        pitch *= (1.0 - slot->extras.pitchBehindModifier);
+    ma_sound_set_pitch(&slot->sound, pitch);
+}
+void AccessibleAudioEngine::garbageCollect() {
+    for (auto i = sounds.begin(); i != sounds.end();) {
+        int deadSlots = 0;
+        for (int x = 0; x < AAE_SLOTS_PER_HANDLE; x++) {
+            if (!i->second[x].active)
+                deadSlots++;
+            else if (!ma_sound_is_playing(&i->second[x].sound)) {
+                destroySound(&i->second[x]);
+                i->second[x].active = false;
+                deadSlots++;
+            }
+        }
+        if (deadSlots == AAE_SLOTS_PER_HANDLE) // Entire batch is garbage.
+            i = sounds.erase(i);
+        else
+            i++;
+    }
+    framesUntilGC = AAE_GC_INTERVAL;
+}
+void AccessibleAudioEngine::processAudioJobs() {
+    ma_job job;
+    while (ma_resource_manager_next_job(&resourceManager, &job) == MA_SUCCESS)
+        ma_job_process(&job);
+}
+bool AccessibleAudioEngine::initSoundExtras(SoundSlot* slot) {
+    ma_node_config config = ma_node_config_init();
+    config.inputBusCount = 1;
+    config.outputBusCount = 1;
+    config.pInputChannels = positioner_channels;
+    config.pOutputChannels = positioner_channels;
+    config.vtable = &positioner_vtable;
+    memset(&slot->extras, 0, sizeof(SoundExtras));
+    if (ma_node_init(&engine.nodeGraph, &config, NULL, &slot->extras) != MA_SUCCESS)
+        return false;
+    ma_panner_config pc = ma_panner_config_init(ma_format_f32, AAE_CHANNELS);
+    pc.mode = ma_pan_mode_balance;
+    ma_panner_init(&pc, &slot->extras.panner);
+    ma_gainer_config gc = ma_gainer_config_init(
+        AAE_CHANNELS,
+        AAE_SAMPLE_RATE / 20); // Allow one in-game frame for the gain to work its way towards the target value.
+    if (ma_gainer_init(&gc, NULL, &slot->extras.gainer) != MA_SUCCESS)
+        return false;
+    ma_lpf_config fc =
+        ma_lpf_config_init(ma_format_f32, AAE_CHANNELS, AAE_SAMPLE_RATE, AAE_SAMPLE_RATE / 2, AAE_LPF_ORDER);
+    ma_lpf_init(&fc, NULL, &slot->extras.filter);
+    slot->extras.cutoff = 1.0f;
+    slot->extras.pitch = 1.0f;
+    slot->extras.pitchBehindModifier = 0.0f;
+    // ma_node_attach_output_bus(&slot->sound, 0, &slot->extras.filter, 0);
+    ma_node_attach_output_bus(&slot->sound, 0, &slot->extras, 0);
+    return true;
+}
+
+void AccessibleAudioEngine::destroySound(SoundSlot* slot) {
+    ma_node_detach_all_output_buses(&slot->extras);
+    ma_sound_uninit(&slot->sound);
+    ma_gainer_uninit(&slot->extras.gainer, NULL);
+
+    slot->active = false;
+}
+
+AccessibleAudioEngine::AccessibleAudioEngine() {
+    initialized = 0;
+    ma_resource_manager_config rmc = ma_resource_manager_config_init();
+    rmc.decodedChannels = AAE_CHANNELS;
+    rmc.decodedFormat = ma_format_f32;
+    rmc.decodedSampleRate = AAE_SAMPLE_RATE;
+    rmc.flags = MA_RESOURCE_MANAGER_FLAG_NON_BLOCKING;
+    rmc.jobThreadCount = 0;
+    if (ma_resource_manager_init(&rmc, &resourceManager) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the resource manager.");
+    initialized = 1;
+    if (ma_pcm_rb_init(ma_format_f32, AAE_CHANNELS, AAE_MAX_BUFFER_SIZE, NULL, NULL, &preparedOutput) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the output buffer.");
+    initialized = 2;
+    ma_engine_config ec = ma_engine_config_init();
+    ec.channels = AAE_CHANNELS;
+    ec.noDevice = true;
+    ec.sampleRate = AAE_SAMPLE_RATE;
+    ec.pResourceManager = &resourceManager;
+    ec.listenerCount = 1;
+
+    if (ma_engine_init(&ec, &engine) != MA_SUCCESS)
+        destroyAndThrow("AccessibleAudioEngine: Unable to initialize the audio engine.");
+    initialized = 3;
+    nextOutgoingSoundAction = 0;
+    framesUntilGC = AAE_GC_INTERVAL;
+    thread = std::thread(&AccessibleAudioEngine::runThread, this);
+}
+AccessibleAudioEngine::~AccessibleAudioEngine() {
+    // Place a terminate command on the top of the pile, then wait for thread to die.
+    SoundAction action;
+    action.command = AAE_TERMINATE;
+    postHighPrioritySoundAction(action);
+    thread.join();
+    destroy();
+}
+void AccessibleAudioEngine::mix(int16_t* ogBuffer, uint32_t nFrames) {
+    float sourceChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
+    float mixedChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
+    while (nFrames > 0) {
+        uint32_t nextChunk = std::min<uint32_t>(AAE_MIX_CHUNK_SIZE, nFrames);
+        // This is so that it doesn't matter if we have less output available than expected.
+        ma_silence_pcm_frames(sourceChunk, nextChunk, ma_format_f32, AAE_CHANNELS);
+        ma_silence_pcm_frames(mixedChunk, nextChunk, ma_format_f32, AAE_CHANNELS);
+        retrieve(sourceChunk, nextChunk);
+        // The game's output is changed to 32-bit floating point samples.
+        ma_pcm_s16_to_f32(mixedChunk, ogBuffer, nextChunk * AAE_CHANNELS, ma_dither_mode_none);
+        ma_mix_pcm_frames_f32(mixedChunk, sourceChunk, nextChunk, AAE_CHANNELS, 1.0);
+        // If we've gone over 1.0, we'll need to scale back before we go back to 16-bit or we'll distort.
+        float scalar = 1.0;
+        for (int i = 0; i < nextChunk * AAE_CHANNELS; i++)
+            scalar = std::max(scalar, mixedChunk[i]);
+        if (scalar > 1.0) {
+            scalar = 1.0 / scalar;
+            for (int i = 0; i < nextChunk * AAE_CHANNELS; i++)
+                mixedChunk[i] *= scalar;
+        }
+        // Chunk is ready to go out via the game's usual channels
+        ma_pcm_f32_to_s16(ogBuffer, mixedChunk, nextChunk * AAE_CHANNELS, ma_dither_mode_triangle);
+        ogBuffer += nextChunk * AAE_CHANNELS;
+        nFrames -= nextChunk;
+    }
+}
+
+void AccessibleAudioEngine::playSound(uintptr_t handle, int slot, const char* path, bool looping) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_START;
+    action.path = path;
+    action.looping = looping;
+}
+
+void AccessibleAudioEngine::stopSound(uintptr_t handle, int slot) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_STOP;
+    action.handle = (uintptr_t)handle;
+    action.slot = slot;
+}
+
+void AccessibleAudioEngine::stopAllSounds(uintptr_t handle) {
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_STOP_ALL;
+    action.handle = handle;
+}
+
+void AccessibleAudioEngine::setPitch(uintptr_t handle, int slot, float pitch) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PITCH;
+    action.handle = handle;
+    action.slot = slot;
+    action.pitch = pitch;
+}
+
+void AccessibleAudioEngine::setPitchBehindModifier(uintptr_t handle, int slot, float mod) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PITCH_BEHIND;
+    action.handle = handle;
+    action.slot = slot;
+    action.pitch = mod;
+}
+
+void AccessibleAudioEngine::setVolume(uintptr_t handle, int slot, float volume) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_VOLUME;
+    action.handle = handle;
+    action.slot = slot;
+    action.volume = volume;
+}
+
+void AccessibleAudioEngine::setPan(uintptr_t handle, int slot, float pan) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PAN;
+    action.handle = handle;
+    action.slot = slot;
+    action.pan = pan;
+}
+void AccessibleAudioEngine::setFilter(uintptr_t handle, int slot, float cutoff) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    if (cutoff < 0.0 || cutoff > 1.0)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_FILTER;
+    action.cutoff = cutoff;
+}
+
+void AccessibleAudioEngine::seekSound(uintptr_t handle, int slot, size_t offset) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.handle = handle;
+    action.slot = slot;
+    action.command = AAE_SEEK;
+    action.offset = offset;
+}
+
+void AccessibleAudioEngine::setSoundPosition(uintptr_t handle, int slot, float posX, float posY, float posZ,
+                                             float distToPlayer, float maxDistance) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_POS;
+    action.handle = handle;
+    action.slot = slot;
+    action.posX = posX;
+    action.posY = posY;
+    action.posZ = posZ;
+    action.distToPlayer = distToPlayer;
+    action.maxDistance = maxDistance;
+}
+
+void AccessibleAudioEngine::prepare() {
+    SoundAction& action = getNextOutgoingSoundAction();
+    action.command = AAE_PREPARE;
+    // This is called once at the end of every frame, so now is the time to post all of the accumulated commands.
+    postSoundActions();
+}

soh/soh/Enhancements/accessible-actors/AccessibleAudioEngine.h

@@ -0,0 +1,152 @@
+#pragma once
+#include <stdint.h>
+#include <thread>
+#include <mutex>
+#include <deque>
+#include <condition_variable>
+#include <string>
+#include <unordered_map>
+#include <array>
+
+#include "soh/Enhancements/audio/miniaudio.h"
+
+#define AAE_SOUND_ACTION_BATCH_SIZE 64
+#define AAE_SLOTS_PER_HANDLE 16
+class IResource;
+struct DecodedSample {
+    void* data; // A wav file.
+    size_t dataSize;
+};
+struct SoundAction {
+    uintptr_t handle; // This handle is user-defined and uniquely identifies a sound source. It can be anything, but the
+                      // address of an object with which the sound is associated is recommended.
+    int slot;         // Allows multiple sounds per handle. The exact number is controlled by AAE_SOUNDS_PER_HANDLE.
+    int command;      // One of the items belonging to AAE_COMMANDS.
+    std::string path; // If command is AAE_START, this is the path to the desired resource.
+    bool looping;     // If command is AAE_START, specifies whether or not the sound should loop.
+    union {
+        float pitch;
+        float volume;
+        float pan;
+        float cutoff;
+        size_t offset; // for seeking.
+        float distance;
+    };
+
+    // Position and rotation vectors for AAE_POS
+    float posX;
+    float posY;
+    float posZ;
+    float distToPlayer;
+    float maxDistance;
+    uint32_t frames; // If command is AAE_PREPARE, this tells the engine how many PCM frames to get ready.
+};
+
+typedef struct {
+    ma_node_base base;
+    ma_panner panner;
+    ma_gainer gainer;
+    ma_lpf filter;
+    float cutoff;
+    float x;
+    float y;
+    float z;
+    float distToPlayer;
+    float maxDistance;
+    float pitch;
+    float pitchBehindModifier;
+} SoundExtras; // Used for attenuation and other effects.
+
+typedef struct {
+    ma_sound sound;
+    SoundExtras extras;
+
+    bool active;
+} SoundSlot;
+typedef std::array<SoundSlot, AAE_SLOTS_PER_HANDLE> SoundSlots;
+
+class AccessibleAudioEngine {
+    int initialized;
+    ma_engine engine;
+    ma_pcm_rb preparedOutput;             // Lock-free single producer single consumer.
+    std::deque<SoundAction> soundActions; // A command cue.
+    std::thread thread;
+    std::condition_variable cv;
+    std::mutex mtx;
+    std::unordered_map<uintptr_t, SoundSlots> sounds;
+    SoundAction
+        outgoingSoundActions[AAE_SOUND_ACTION_BATCH_SIZE]; // Allows batch delivery of SoundActions to the FIFO to
+                                                           // minimize the amount of time spent locking and unlocking.
+    int nextOutgoingSoundAction;
+    int framesUntilGC;
+    void destroy(); // Called by the destructor, or if a throw occurs during construction.
+    // Dismantal a partial initialization and throw an exception.
+    void destroyAndThrow(const char* exceptionText);
+
+    // Retrieve some audio from the output buffer. This is to be performed by the audio thread. May return less data
+    // than requested.
+    uint32_t retrieve(float* buffer, uint32_t nFrames);
+    // Functions dealing with the command queue.
+    int getSoundActions(SoundAction* dest, int limit);
+    void postSoundActions();
+    void postHighPrioritySoundAction(SoundAction& action); // For now this is used only for termination events.
+
+    SoundAction& getNextOutgoingSoundAction();
+    void runThread();
+    // Find a sound by handle and slot, if it exists.
+    SoundSlot* findSound(SoundAction& action);
+
+    // Functions which correspond to SoundAction commands.
+    // Ready a sound for playback.
+    void doPlaySound(SoundAction& action);
+    void doStopSound(SoundAction& action);
+    void doStopAllSounds(SoundAction& action);
+
+    void doSetPitch(SoundAction& action);
+    void doSetPitchBehindModifier(SoundAction& action);
+    void doSetVolume(SoundAction& action);
+    void doSetPan(SoundAction& action);
+    void doSetFilter(SoundAction& action);
+    void doSeekSound(SoundAction& action);
+
+    void doSetListenerPos(SoundAction& action);
+    void doSetSoundPos(SoundAction& action);
+    // Generate some output, and store it in the output buffer for later retrieval. May generate less output than
+    // requested if buffer space is insufficient.
+    void doPrepare(SoundAction& action);
+    // Run every so often to clean up expired sound handles.
+    void garbageCollect();
+    // Run MiniAudio's jobs.
+    void processAudioJobs();
+    // Set up the panner and other effect processing on a sound slot.
+    bool initSoundExtras(SoundSlot* slot);
+    void destroySound(SoundSlot* slot);
+
+  public:
+    AccessibleAudioEngine();
+    ~AccessibleAudioEngine();
+    // Mix the game's audio with this engine's audio to produce the final mix. To be performed exclusively in the audio
+    // thread. Mixing is done in-place (meaning the buffer containing the game's audio is overwritten with the mixed
+    // content).
+    void mix(int16_t* ogBuffer, uint32_t nFrames);
+    // Start playing a sound.
+    void playSound(uintptr_t handle, int slot, const char* path, bool looping);
+    void stopSound(uintptr_t handle, int slot);
+    // Stop all sounds belonging to a handle.
+    void stopAllSounds(uintptr_t handle);
+
+    void setPitch(uintptr_t handle, int slot, float pitch);
+    void setPitchBehindModifier(uintptr_t handle, int slot, float mod);
+    void setVolume(uintptr_t handle, int slot, float volume);
+    void setPan(uintptr_t handle, int slot, float pan);
+    // Set the lowpass filter cutoff. Set to 1.0 for no filtering.
+    void setFilter(uintptr_t handle, int slot, float cutoff);
+    // Seek the sound to a particular PCM frame.
+    void seekSound(uintptr_t handle, int slot, size_t offset);
+    void setSoundPosition(uintptr_t handle, int slot, float posX, float posY, float posZ, float distToPlayer,
+                          float maxDistance);
+    // Schedule the preparation of output for delivery.
+    void prepare();
+
+    ma_resource_manager resourceManager;
+};

soh/soh/Enhancements/accessible-actors/ActorAccessibility.cpp

@@ -0,0 +1,716 @@
+#include "ActorAccessibility.h"
+#include "AccessibleAudioEngine.h"
+#include "soh/OTRGlobals.h"
+#include "resource/type/Blob.h"
+
+#include <map>
+#include <random>
+#include <vector>
+#include <functions.h>
+#include <variables.h>
+#include <macros.h>
+#include "ResourceType.h"
+#include "SfxExtractor.h"
+
+#include <sstream>
+#include "File.h"
+#include <unordered_set>
+#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
+#include "soh/Enhancements/tts/tts.h"
+#include "soh/Enhancements/game-interactor/GameInteractor.h"
+
+extern "C" {
+extern PlayState* gPlayState;
+extern bool freezeGame;
+extern bool freezeActors;
+}
+
+const char* GetLanguageCode();
+
+// This is the amount in DB that a sound will be reduced by when it is at the maximum distance from the player.
+#define MAX_DB_REDUCTION 35
+
+extern "C" {
+// Used to tell where polygons are located.
+void CollisionPoly_GetVertices(CollisionPoly* poly, Vec3s* vtxList, Vec3f* dest);
+}
+
+typedef struct {
+    union {
+        struct {
+            s16 sceneIndex; // Corresponds directly to the game's scene indices.
+            s16 roomIndex;  // Corresponds directly to the game's room indices.
+        } values;
+        s32 raw; // Combination of the two which can be used for dictionary lookups.
+    };
+
+} SceneAndRoom;
+
+// Maps actors to their accessibility policies, which describe how accessibility should treat them.
+typedef std::map<s16, ActorAccessibilityPolicy> SupportedActors_t;
+typedef std::map<Actor*, uint64_t>
+    TrackedActors_t; // Maps real actors to internal IDs specific to accessibility.
+                     // Maps internal IDs to wrapped actor objects. These actors can be real or virtual.
+typedef std::map<uint64_t, AccessibleActor> AccessibleActorList_t;
+typedef std::vector<AccessibleActor> VAList_t; // Denotes a list of virtual actors specific to a single room.
+typedef std::map<s32, VAList_t> VAZones_t; // Maps room/ scene indices to their corresponding virtual actor collections.
+// A list of scenes which have already been visited (since the game was launched). Used to prevent
+// re-creation of terrain VAs every time the player reloads a scene.
+typedef std::unordered_set<s16> SceneList_t;
+
+typedef struct {
+    std::string path;
+    std::shared_ptr<Ship::File> resource;
+} SfxRecord;
+
+class AudioGlossaryData {
+  public:
+    AccessibleActorList_t accessibleActorList;
+    AccessibleActorList_t::iterator current = accessibleActorList.begin();
+    bool GlossaryStarted = false;
+    int cooldown = 0;
+    int frameCount = 0;
+    s16 currentScene = -1;
+    s8 currentRoom = -1;
+};
+
+class ActorAccessibility {
+  public:
+    int isOn = 0;
+    uint64_t nextActorID = 0;
+    SupportedActors_t supportedActors;
+    TrackedActors_t trackedActors;
+    AccessibleActorList_t accessibleActorList;
+    AudioGlossaryData* glossary;
+    VAZones_t vaZones;
+    SceneList_t sceneList;
+    AccessibleAudioEngine* audioEngine;
+    SfxExtractor sfxExtractor;
+    // Maps internal sfx to external (prerendered) resources.
+    std::unordered_map<s16, SfxRecord> sfxMap;
+    int extractSfx = 0;
+    s16 currentScene = -1;
+    s8 currentRoom = -1;
+    VirtualActorList* currentEverywhere = NULL;
+    VirtualActorList* currentSceneGlobal = NULL;
+    VirtualActorList* currentRoomLocal = NULL;
+};
+static ActorAccessibility* aa;
+
+uint64_t ActorAccessibility_GetNextID() {
+    return aa->nextActorID++;
+}
+
+void ActorAccessibility_PrepareNextAudioFrame();
+
+// Hooks for game-interactor.
+void ActorAccessibility_OnActorInit(void* actor) {
+    ActorAccessibility_TrackNewActor((Actor*)actor);
+}
+void ActorAccessibility_OnGameFrameUpdate() {
+    if (gPlayState == NULL)
+        return;
+    if (!GameInteractor::IsSaveLoaded() && !aa->extractSfx)
+        return; // Title screen, skip.
+
+    ActorAccessibility_RunAccessibilityForAllActors(gPlayState);
+}
+void ActorAccessibility_OnActorDestroy(void* actor) {
+    ActorAccessibility_RemoveTrackedActor((Actor*)actor);
+}
+void ActorAccessibility_OnGameStillFrozen() {
+    if (gPlayState == NULL)
+        return;
+    if (aa->extractSfx)
+        ActorAccessibility_HandleSoundExtractionMode(gPlayState);
+}
+
+void ActorAccessibility_Init() {
+    aa = new ActorAccessibility();
+    aa->glossary = new AudioGlossaryData();
+    aa->isOn = CVarGetInteger(CVAR_SETTING("A11yAudioInteraction"), 0);
+    if (!aa->isOn)
+        return;
+    aa->extractSfx = CVarGetInteger("gExtractSfx", 0);
+    if (aa->extractSfx)
+        freezeGame = true;
+    ActorAccessibility_InitAudio();
+    ActorAccessibility_InitActors();
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorInit>(ActorAccessibility_OnActorInit);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnActorDestroy>(ActorAccessibility_OnActorDestroy);
+
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnPlayerUpdate>(ActorAccessibility_OnGameFrameUpdate);
+    GameInteractor::Instance->RegisterGameHook<GameInteractor::OnGameStillFrozen>(ActorAccessibility_OnGameStillFrozen);
+}
+void ActorAccessibility_Shutdown() {
+    ActorAccessibility_ShutdownAudio();
+    delete aa;
+}
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName,
+                                   ActorAccessibilityCallback callback) {
+    policy->callback = callback;
+    policy->distance = 500;
+    policy->ydist = 80;
+    policy->englishName = englishName;
+    policy->n = 20;
+    policy->pitch = 1.5;
+    policy->runsAlways = false;
+    policy->sound = 0;
+    policy->volume = 1.0;
+    policy->initUserData = NULL;
+    policy->cleanupUserData = NULL;
+    policy->pitchModifier = 0.1;
+    policy->aimAssist.isProvider = false;
+    policy->aimAssist.sfx = NA_SE_SY_HITPOINT_ALARM;
+    policy->aimAssist.tolerance = 0.0;
+}
+
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName, s16 sfx) {
+    policy->callback = nullptr;
+    policy->distance = 500;
+    policy->ydist = 80;
+    policy->englishName = englishName;
+    policy->n = 20;
+    policy->pitch = 1.5;
+    policy->runsAlways = false;
+    policy->sound = sfx;
+    policy->volume = 1.0;
+    policy->initUserData = NULL;
+    policy->cleanupUserData = NULL;
+    policy->pitchModifier = 0.1;
+    policy->aimAssist.isProvider = false;
+    policy->aimAssist.sfx = NA_SE_SY_HITPOINT_ALARM;
+    policy->aimAssist.tolerance = 0.0;
+}
+
+void ActorAccessibility_AddSupportedActor(s16 type, ActorAccessibilityPolicy policy) {
+    aa->supportedActors[type] = policy;
+}
+
+ActorAccessibilityPolicy* ActorAccessibility_GetPolicyForActor(s16 type) {
+    SupportedActors_t::iterator i = aa->supportedActors.find(type);
+    if (i == aa->supportedActors.end())
+        return NULL;
+    return &(i->second);
+}
+int ActorAccessibility_GetRandomStartingFrameCount(int min, int max) {
+    static std::mt19937 gen;
+    std::uniform_int_distribution<> dist(min, max);
+    return dist(gen);
+}
+
+void ActorAccessibility_TrackNewActor(Actor* actor) {
+    // Don't track actors for which no accessibility policy has been configured.
+    ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(actor->id);
+    if (policy == NULL)
+        return;
+    AccessibleActor accessibleActor;
+    accessibleActor.instanceID = ActorAccessibility_GetNextID();
+    accessibleActor.actor = actor;
+    accessibleActor.id = actor->id;
+    // Stagger the start times so that all of the sounds don't play at exactly the same time.
+    accessibleActor.frameCount = ActorAccessibility_GetRandomStartingFrameCount(0, policy->n);
+    accessibleActor.basePitch = policy->pitch;
+    accessibleActor.policy = *policy;
+    accessibleActor.currentPitch = accessibleActor.policy.pitch;
+    accessibleActor.baseVolume = accessibleActor.policy.volume;
+    accessibleActor.currentVolume = accessibleActor.policy.volume;
+    accessibleActor.sceneIndex = 0;
+    accessibleActor.managedSoundSlots = 0;
+    accessibleActor.aimAssist.framesSinceAimAssist = 32768;
+    accessibleActor.aimAssist.frequency = 10;
+    accessibleActor.aimAssist.pitch = 1.0;
+
+    aa->trackedActors[actor] = accessibleActor.instanceID;
+    aa->accessibleActorList[accessibleActor.instanceID] = accessibleActor;
+    AccessibleActor& savedActor = aa->accessibleActorList[accessibleActor.instanceID];
+    if (policy->initUserData && !policy->initUserData(&savedActor)) {
+        ActorAccessibility_RemoveTrackedActor(actor);
+
+        return; // Probably a malloc error preventing user data initialization.
+    }
+}
+void ActorAccessibility_RemoveTrackedActor(Actor* actor) {
+    TrackedActors_t::iterator i = aa->trackedActors.find(actor);
+    if (i == aa->trackedActors.end())
+        return;
+    uint64_t id = i->second;
+    aa->trackedActors.erase(i);
+    AccessibleActorList_t::iterator i2 = aa->accessibleActorList.find(id);
+    if (i2 == aa->accessibleActorList.end())
+        return;
+    if (i2->second.policy.cleanupUserData)
+        i2->second.policy.cleanupUserData(&i2->second);
+    ActorAccessibility_StopAllSoundsForActor(&i2->second);
+    aa->accessibleActorList.erase(i2);
+}
+
+f32 ActorAccessibility_DBToLinear(float gain) {
+    return powf(10.0, gain / 20.0f);
+}
+f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer) {
+    if (maxDistance == 0)
+        return 0.0;
+    f32 absDistance = fabs(xzDistToPlayer);
+    f32 db = LERP(0.0 - MAX_DB_REDUCTION, 0.0, (maxDistance - absDistance) / maxDistance);
+
+    return ActorAccessibility_DBToLinear(db);
+}
+const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId);
+void ActorAccessibility_PlaySound(void* handle, int slot, s16 sfxId, bool looping) {
+    const char* path = ActorAccessibility_MapSfxToExternalAudio(sfxId);
+    if (path == NULL)
+        return;
+    aa->audioEngine->playSound((uintptr_t)handle, slot, path, looping);
+}
+
+void ActorAccessibility_StopSound(void* handle, int slot) {
+    aa->audioEngine->stopSound((uintptr_t)handle, slot);
+}
+void ActorAccessibility_StopAllSounds(void* handle) {
+    aa->audioEngine->stopAllSounds((uintptr_t)handle);
+}
+void ActorAccessibility_SetSoundPitch(void* handle, int slot, float pitch) {
+    aa->audioEngine->setPitch((uintptr_t)handle, slot, pitch);
+}
+void ActorAccessibility_SetPitchBehindModifier(void* handle, int slot, float mod) {
+    aa->audioEngine->setPitchBehindModifier((uintptr_t)handle, slot, mod);
+}
+void ActorAccessibility_SetSoundPos(void* handle, int slot, Vec3f* pos, f32 distToPlayer, f32 maxDistance) {
+    aa->audioEngine->setSoundPosition((uintptr_t)handle, slot, pos->x, pos->y, pos->z, distToPlayer, maxDistance);
+}
+void ActorAccessibility_SetSoundVolume(void* handle, int slot, float volume) {
+    aa->audioEngine->setVolume((uintptr_t)handle, slot, volume);
+}
+void ActorAccessibility_SetSoundPan(void* handle, int slot, Vec3f* projectedPos) {
+    float pan = projectedPos->x / 270;
+    if (pan < -1.0)
+        pan = -1.0;
+    if (pan > 1.0)
+        pan = 1.0;
+    aa->audioEngine->setPan((uintptr_t)handle, slot, pan);
+}
+void ActorAccessibility_SetSoundFilter(void* handle, int slot, float cutoff) {
+    aa->audioEngine->setFilter((uintptr_t)handle, slot, cutoff);
+}
+void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset) {
+    aa->audioEngine->seekSound((uintptr_t)handle, slot, offset);
+}
+void ActorAccessibility_ConfigureSoundForActor(AccessibleActor* actor, int slot) {
+    ActorAccessibility_SetSoundPitch(actor, slot, actor->policy.pitch);
+    ActorAccessibility_SetPitchBehindModifier(actor, slot, actor->policy.pitchModifier);
+    ActorAccessibility_SetSoundPos(actor, slot, &actor->projectedPos, actor->xyzDistToPlayer, actor->policy.distance);
+    ActorAccessibility_SetSoundVolume(actor, slot, actor->policy.volume);
+    actor->managedSoundSlots |= 1 << slot;
+}
+void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping) {
+    if (slot < 0 || slot > AAE_SLOTS_PER_HANDLE)
+        return;
+    ActorAccessibility_PlaySound(actor, slot, sfxId, looping);
+    ActorAccessibility_ConfigureSoundForActor(actor, slot);
+}
+void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot) {
+    if (slot < 0 || slot >= AAE_SLOTS_PER_HANDLE)
+        return;
+    ActorAccessibility_StopSound(actor, slot);
+    actor->managedSoundSlots &= ~(1 << slot);
+}
+
+void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor) {
+    ActorAccessibility_StopAllSounds(actor);
+    actor->managedSoundSlots = 0;
+}
+
+bool ActorAccessibility_IsRealActor(AccessibleActor* actor) {
+    return actor->actor != NULL;
+}
+void ActorAccessibility_CopyParamsFromRealActor(AccessibleActor* actor) {
+    Player* player = GET_PLAYER(actor->play);
+    if (actor->actor == NULL)
+        return;
+    actor->projectedPos = actor->actor->projectedPos;
+    actor->xzDistToPlayer = actor->actor->xzDistToPlayer;
+    actor->isDrawn = actor->actor->isDrawn;
+    actor->xyzDistToPlayer = Math_Vec3f_DistXYZ(&actor->actor->world.pos, &player->actor.world.pos);
+}
+
+void ActorAccessibility_StopAllVirtualActors(VirtualActorList* list) {
+    if (list == NULL)
+        return;
+
+    VAList_t* val = (VAList_t*)list;
+    for (auto i = val->begin(); i != val->end(); i++)
+        ActorAccessibility_StopAllSounds((void*)&(*i));
+}
+
+void ActorAccessibility_RunAccessibilityForActor(PlayState* play, AccessibleActor* actor) {
+    actor->play = play;
+    if (ActorAccessibility_IsRealActor(actor)) {
+        ActorAccessibility_CopyParamsFromRealActor(actor);
+    } else {
+        Player* player = GET_PLAYER(play);
+        f32 w = 0.0f;
+        // Set actor->projectedPos.
+        SkinMatrix_Vec3fMtxFMultXYZW(&play->viewProjectionMtxF, &actor->world.pos, &actor->projectedPos, &w);
+        actor->xzDistToPlayer = Math_Vec3f_DistXZ(&actor->world.pos, &player->actor.world.pos);
+        actor->xyzDistToPlayer = Math_Vec3f_DistXYZ(&actor->world.pos, &player->actor.world.pos);
+        actor->yDistToPlayer = fabs((actor->world.pos.y) - (player->actor.world.pos.y));
+    }
+
+    if (actor->actor != NULL && fabs(actor->actor->yDistToPlayer) > actor->policy.ydist) {
+        return;
+    }
+    // Send sound parameters to the new audio engine. Eventually remove the old stuff once all actors are carried over.
+    for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++) {
+        if (actor->managedSoundSlots & (1 << i)) {
+            ActorAccessibility_SetSoundPos(actor, i, &actor->projectedPos, actor->xyzDistToPlayer,
+                                           actor->policy.distance);
+            // Judgement call: pitch changes are rare enough that it doesn't make sense to pay the cost of updating it
+            // every frame. If you want a pitch change, call the function as needed.
+        }
+    }
+    actor->frameCount++;
+    if (aa->glossary->GlossaryStarted) {
+        aa->glossary->frameCount++;
+    }
+    if (actor->frameCount % actor->policy.n)
+        return;
+    if (!actor->policy.runsAlways && actor->xyzDistToPlayer > actor->policy.distance) {
+        return;
+    }
+    if (actor->isDrawn == 0 && actor->actor->id != ACTOR_EN_IT && actor->actor->id != ACTOR_EN_OKARINA_TAG &&
+        !aa->glossary->GlossaryStarted)
+        return;
+    if (actor->policy.aimAssist.isProvider) {
+        Player* player = GET_PLAYER(play);
+        if (player->stateFlags1 & PLAYER_STATE1_FIRST_PERSON &&
+            (player->stateFlags1 & PLAYER_STATE1_USING_BOOMERANG || player->stateFlags1 & PLAYER_STATE1_ITEM_IN_HAND)) {
+            ActorAccessibility_SetSoundPitch(actor, 9, actor->aimAssist.pitch);
+            actor->aimAssist.framesSinceAimAssist++;
+            ActorAccessibility_ProvideAimAssistForActor(actor);
+            // The above will have taken care of setting the appropriate frequency and pitch, so we'll take care of the
+            // audio here based on those results.
+            if (actor->aimAssist.framesSinceAimAssist >= actor->aimAssist.frequency) {
+
+                actor->aimAssist.framesSinceAimAssist = 0;
+                ActorAccessibility_PlaySoundForActor(actor, 9, actor->policy.aimAssist.sfx, false);
+            }
+        } else {
+            // Make sure there's no delay the next time you draw your bow or whatever.
+            actor->aimAssist.framesSinceAimAssist = 32768;
+        }
+    }
+
+    if (actor->policy.callback != nullptr) {
+        actor->policy.callback(actor);
+    } else {
+        ActorAccessibility_PlaySoundForActor(actor, 0, actor->policy.sound, false);
+    }
+}
+void ActorAccessibility_RunAccessibilityForAllActors(PlayState* play) {
+    Player* player = GET_PLAYER(play);
+    if (play->sceneNum != aa->currentScene) {
+        ActorAccessibility_StopAllVirtualActors(aa->currentEverywhere);
+        ActorAccessibility_StopAllVirtualActors(aa->currentSceneGlobal);
+        ActorAccessibility_StopAllVirtualActors(aa->currentRoomLocal);
+        aa->currentEverywhere = ActorAccessibility_GetVirtualActorList(EVERYWHERE, 0);
+        aa->currentSceneGlobal = ActorAccessibility_GetVirtualActorList(play->sceneNum, -1);
+        aa->currentScene = play->sceneNum;
+        aa->currentRoomLocal = NULL;
+        aa->currentRoom = -1;
+    }
+    if (aa->currentRoom != play->roomCtx.curRoom.num) {
+        ActorAccessibility_StopAllVirtualActors(aa->currentRoomLocal);
+        aa->currentRoomLocal = ActorAccessibility_GetVirtualActorList(play->sceneNum, play->roomCtx.curRoom.num);
+        aa->currentRoom = play->roomCtx.curRoom.num;
+    }
+    if (aa->glossary->currentScene != play->sceneNum || aa->glossary->currentRoom != play->roomCtx.curRoom.num) {
+        if (aa->glossary->GlossaryStarted) {
+            aa->glossary->cooldown = 0;
+            aa->glossary->GlossaryStarted = false;
+            freezeActors = false;
+        }
+    }
+    if (player->stateFlags1 & PLAYER_STATE1_IN_CUTSCENE) {
+        return;
+    }
+    ActorAccessibility_AudioGlossary(play);
+    if (aa->glossary->GlossaryStarted) {
+
+        return;
+    }
+    // Real actors.
+    for (AccessibleActorList_t::iterator i = aa->accessibleActorList.begin(); i != aa->accessibleActorList.end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &i->second);
+    // Virtual actors in the "everywhere" group.
+    VAList_t* list = (VAList_t*)aa->currentEverywhere;
+
+    for (VAList_t::iterator i = list->begin(); i != list->end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &(*i));
+    // Virtual actors for the current room and scene.
+    list = (VAList_t*)aa->currentRoomLocal;
+    for (VAList_t::iterator i = list->begin(); i != list->end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &(*i));
+    // Scene-global virtual actors. Most of these are automatically generated VAs from polygons, because there's no way
+    // to sort these into rooms.
+    list = (VAList_t*)aa->currentSceneGlobal;
+    for (VAList_t::iterator i = list->begin(); i != list->end(); i++)
+        ActorAccessibility_RunAccessibilityForActor(play, &(*i));
+
+    // Processes external audio engine.
+    ActorAccessibility_PrepareNextAudioFrame();
+}
+
+void ActorAccessibility_AudioGlossary(PlayState* play) {
+    if (aa->glossary->GlossaryStarted) {
+        freezeActors = true;
+        AccessibleActor glossaryActor = (*aa->glossary->current).second;
+        ActorAccessibility_CopyParamsFromRealActor(&glossaryActor);
+        glossaryActor.policy.distance = glossaryActor.xzDistToPlayer * 3;
+        glossaryActor.policy.ydist = 1000;
+        glossaryActor.frameCount = aa->glossary->frameCount;
+        ActorAccessibility_RunAccessibilityForActor(play, &glossaryActor);
+    }
+    if (aa->glossary->cooldown != 0) {
+        aa->glossary->cooldown--;
+        return;
+    }
+
+    OSContPad* trackerButtonsPressed =
+        std::dynamic_pointer_cast<LUS::ControlDeck>(Ship::Context::GetInstance()->GetControlDeck())->GetPads();
+    bool comboStartGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DUP &&
+                              trackerButtonsPressed[0].button & BTN_L;
+    if (comboStartGlossary) {
+        aa->glossary->GlossaryStarted = true;
+        aa->glossary->current = aa->accessibleActorList.begin();
+        aa->glossary->currentScene = play->sceneNum;
+        aa->glossary->currentRoom = play->roomCtx.curRoom.num;
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+        return;
+    }
+    bool comboNextGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DRIGHT &&
+                             trackerButtonsPressed[0].button & BTN_L;
+    if (comboNextGlossary && aa->glossary->GlossaryStarted) {
+        aa->glossary->current++;
+        if (aa->glossary->current == aa->accessibleActorList.end()) {
+            aa->glossary->current = aa->accessibleActorList.begin();
+        };
+        aa->glossary->cooldown = 5;
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+    }
+    bool comboPrevGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DLEFT &&
+                             trackerButtonsPressed[0].button & BTN_L;
+    if (comboPrevGlossary && aa->glossary->GlossaryStarted) {
+        if (aa->glossary->current != aa->accessibleActorList.begin()) {
+            aa->glossary->current--;
+        }
+        aa->glossary->cooldown = 5;
+
+        SpeechSynthesizer::Instance->Speak((*aa->glossary->current).second.policy.englishName, GetLanguageCode());
+    }
+    bool comboDisableGlossary = trackerButtonsPressed != nullptr && trackerButtonsPressed[0].button & BTN_DDOWN &&
+                                trackerButtonsPressed[0].button & BTN_L;
+    if (comboDisableGlossary) {
+        aa->glossary->cooldown = 0;
+        aa->glossary->GlossaryStarted = false;
+        freezeActors = false;
+    }
+    // Processes external audio engine.
+    ActorAccessibility_PrepareNextAudioFrame();
+}
+
+// Virtual actor config.
+VirtualActorList* ActorAccessibility_GetVirtualActorList(s16 sceneNum, s8 roomNum) {
+    SceneAndRoom sr;
+    sr.values.sceneIndex = sceneNum;
+    sr.values.roomIndex = roomNum;
+    if (sceneNum == EVERYWHERE)
+        sr.values.sceneIndex = EVERYWHERE;
+
+    VAList_t* l = &aa->vaZones[sr.raw];
+    return (VirtualActorList*)l;
+}
+AccessibleActor* ActorAccessibility_AddVirtualActor(VirtualActorList* list, VIRTUAL_ACTOR_TABLE type, PosRot where) {
+    ActorAccessibilityPolicy* policy = ActorAccessibility_GetPolicyForActor(type);
+    if (policy == NULL)
+        return NULL;
+
+    AccessibleActor actor;
+    actor.actor = NULL;
+    actor.basePitch = 1.0;
+    actor.baseVolume = 1.0;
+    actor.currentPitch = 1.0;
+    actor.currentVolume = 1.0;
+    actor.frameCount = 0;
+    actor.id = (s16)type;
+    actor.instanceID = ActorAccessibility_GetNextID();
+    actor.isDrawn = 1;
+    actor.play = NULL;
+    actor.world = where;
+    actor.sceneIndex = 0;
+    actor.managedSoundSlots = 0;
+    actor.aimAssist.framesSinceAimAssist = 0;
+    actor.aimAssist.frequency = 10;
+    actor.aimAssist.pitch = 1.0;
+
+    actor.policy = *policy;
+    VAList_t* l = (VAList_t*)list;
+    l->push_back(actor);
+    size_t index = l->size() - 1;
+    AccessibleActor* savedActor = &(*l)[l->size() - 1];
+    if (policy->initUserData && !policy->initUserData(savedActor)) {
+        l->pop_back();
+
+        return NULL; // Probably a malloc error preventing user data initialization.
+    }
+    return savedActor;
+}
+void ActorAccessibility_InterpretCurrentScene(PlayState* play) {
+    if (aa->sceneList.contains(play->sceneNum))
+        return; // Scene interpretation already complete for this scene.
+    aa->sceneList.insert(play->sceneNum);
+    VirtualActorList* list = ActorAccessibility_GetVirtualActorList(play->sceneNum, -1); // Scene-global VAs.
+    if (list == NULL)
+        return;
+    for (int i = 0; i < play->colCtx.colHeader->numPolygons; i++) {
+        CollisionPoly* poly = &play->colCtx.colHeader->polyList[i];
+        // checks if climable
+        if ((func_80041DB8(&play->colCtx, poly, BGCHECK_SCENE) == 8 ||
+             func_80041DB8(&play->colCtx, poly, BGCHECK_SCENE) == 3)) {
+            ActorAccessibility_PolyToVirtualActor(play, poly, VA_CLIMB, list);
+        }
+        if (SurfaceType_IsWallDamage(&play->colCtx, poly, BGCHECK_SCENE)) {
+            ActorAccessibility_PolyToVirtualActor(play, poly, VA_SPIKE, list);
+        }
+        if (SurfaceType_GetSceneExitIndex(&play->colCtx, poly, BGCHECK_SCENE) != 0)
+            ActorAccessibility_PolyToVirtualActor(play, poly, VA_AREA_CHANGE, list);
+        /*s8 floorparam = func_80041D4C(&play->colCtx, poly, BGCHECK_SCENE);
+        if (floorparam == 2) {
+        ActorAccessibility_PolyToVirtualActor(play, poly, VA_SPIKE, list);
+        }*/
+    }
+}
+// Convert poly to VA.
+void ActorAccessibility_PolyToVirtualActor(PlayState* play, CollisionPoly* poly, VIRTUAL_ACTOR_TABLE va,
+                                           VirtualActorList* destination) {
+    Vec3f polyVerts[3];
+    CollisionPoly_GetVertices(poly, play->colCtx.colHeader->vtxList, polyVerts);
+    PosRot where;
+    where.pos.y = std::min(polyVerts[0].y, std::min(polyVerts[1].y, polyVerts[2].y));
+    f32 minX = std::min(polyVerts[0].x, std::min(polyVerts[1].x, polyVerts[2].x));
+    f32 maxX = std::max(polyVerts[0].x, std::max(polyVerts[1].x, polyVerts[2].x));
+    f32 minZ = std::min(polyVerts[0].z, std::min(polyVerts[1].z, polyVerts[2].z));
+    f32 maxZ = std::max(polyVerts[0].z, std::max(polyVerts[1].z, polyVerts[2].z));
+    where.pos.x = maxX - ((maxX - minX) / 2);
+    where.pos.z = maxZ - ((maxZ - minZ) / 2);
+    where.rot = { 0, 0, 0 };
+    AccessibleActor* actor = ActorAccessibility_AddVirtualActor(destination, va, where);
+    if (actor == NULL)
+        return;
+    if (va == VA_AREA_CHANGE) {
+        actor->sceneIndex = SurfaceType_GetSceneExitIndex(&play->colCtx, poly, BGCHECK_SCENE);
+        s16 nextEntranceIndex = play->setupExitList[actor->sceneIndex - 1];
+        actor->sceneIndex = gEntranceTable[nextEntranceIndex].scene;
+    }
+}
+void ActorAccessibility_AnnounceRoomNumber(PlayState* play) {
+    std::stringstream ss;
+    ss << "Room" << (int)play->roomCtx.curRoom.num;
+    if (Flags_GetClear(play, play->roomCtx.curRoom.num))
+        ss << ", completed." << std::endl;
+    else
+        ss << "." << std::endl;
+    SpeechSynthesizer::Instance->Speak(ss.str().c_str(), GetLanguageCode());
+}
+// Aim cue support.
+void ActorAccessibility_ProvideAimAssistForActor(AccessibleActor* actor) {
+    Player* player = GET_PLAYER(actor->play);
+    s32 angle = player->actor.focus.rot.x;
+    angle = angle / -14000.0 * 16384;
+    f32 slope = Math_SinS(angle) / Math_CosS(angle) * 1.0;
+    s32 yIntercept = (slope * (actor->xzDistToPlayer)) + player->actor.focus.pos.y;
+    s32 yHight = actor->actor->world.pos.y + 25;
+    if (slope < 1) {
+        slope = 1;
+    }
+    s32 correction = (1 - 1 / slope) * 100;
+    if ((yIntercept) > yHight + 25) {
+        actor->aimAssist.pitch = 1.5;
+    } else if ((yIntercept) < yHight - 25) {
+        actor->aimAssist.pitch = 0.5;
+    }
+    s32 yDiff = fabs(yIntercept - yHight);
+    if (yIntercept - yHight > 0) {
+        yDiff -= correction;
+        if (yDiff < 0) {
+            yDiff = 0;
+        }
+    }
+    if (yDiff > 300) {
+        actor->aimAssist.frequency = 30;
+    } else {
+        actor->aimAssist.frequency = 1 + (uint8_t)(yDiff / 5);
+    }
+}
+
+// External audio engine stuff.
+bool ActorAccessibility_InitAudio() {
+    try {
+        aa->audioEngine = new AccessibleAudioEngine();
+    } catch (...) {
+        aa->audioEngine = NULL;
+        return false;
+    }
+    return true;
+}
+void ActorAccessibility_ShutdownAudio() {
+    if (aa->isOn) {
+        delete aa->audioEngine;
+        aa->isOn = 0;
+    }
+}
+void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames) {
+    if (aa->isOn) {
+        aa->audioEngine->mix(ogBuffer, nFrames);
+    }
+}
+// Map one of the game's sfx to a path which as understood by the external audio engine. The returned token is a
+// short hex string that can be passed directly to the audio engine.
+const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId) {
+    SfxRecord* record;
+    auto it = aa->sfxMap.find(sfxId);
+    if (it == aa->sfxMap.end()) {
+        SfxRecord tempRecord;
+        std::string fullPath = SfxExtractor::getExternalFileName(sfxId);
+        auto res = Ship::Context::GetInstance()->GetResourceManager()->GetArchiveManager()->LoadFile(fullPath);
+
+        if (res == nullptr)
+            return NULL; // Resource doesn't exist, user's gotta run the extractor.
+        tempRecord.resource = res;
+        std::stringstream ss;
+        ss << std::setw(4) << std::setfill('0') << std::hex << sfxId;
+        tempRecord.path = ss.str();
+        auto pair = aa->sfxMap.insert({ sfxId, tempRecord });
+        record = &pair.first->second;
+        ma_resource_manager_register_decoded_data(&aa->audioEngine->resourceManager, record->path.c_str(),
+                                                  record->resource->Buffer->data(),
+                                                  record->resource->Buffer->size() / 2, ma_format_s16, 1, 44100);
+    } else {
+        record = &it->second;
+    }
+
+    return record->path.c_str();
+}
+
+// Call once per frame to tell the audio engine to start working on the latest batch of queued instructions.
+void ActorAccessibility_PrepareNextAudioFrame() {
+    aa->audioEngine->prepare();
+}
+
+void ActorAccessibility_HandleSoundExtractionMode(PlayState* play) {
+    aa->sfxExtractor.frameCallback();
+}
+
+void ActorAccessibility_DoSoundExtractionStep() {
+    aa->sfxExtractor.captureCallback();
+}

soh/soh/Enhancements/accessible-actors/ActorAccessibility.h

@@ -0,0 +1,180 @@
+#pragma once
+#include <z64.h>
+
+struct AccessibleActor;
+// A callback that is run regularely as the game progresses in order to provide accessibility services for an actor.
+
+typedef void (*ActorAccessibilityCallback)(AccessibleActor*);
+// A callback which allows AccessibleActor instances to initialize custom user data (called once per instantiation).
+typedef bool (*ActorAccessibilityUserDataInit)(AccessibleActor*);
+// A callback that can be used to clean up user data when an actor is destroyed.
+typedef void (*ActorAccessibilityUserDataCleanup)(AccessibleActor*);
+
+struct VirtualActorList;
+
+struct ActorAccessibilityPolicy {
+    const char* englishName;
+
+    ActorAccessibilityCallback callback; // If set, it will be called once every n frames. If null, then sfx will be
+                                         // played once every n frames.
+    s16 sound;                           // The ID of a sound to play. Ignored if the callback is set.
+
+    int n;        // How often to run the callback in frames.
+    f32 distance; // Maximum xz distance from player before the actor should be considered out of range.
+    f32 ydist;    // Maximum y distance from player before the actor should be considered out of range.
+    f32 pitch;
+    f32 volume;
+    f32 pitchModifier;
+    bool runsAlways; // If set, then the distance policy is ignored.
+    ActorAccessibilityUserDataInit initUserData;
+    ActorAccessibilityUserDataCleanup cleanupUserData;
+    // Aim assist settings.
+    struct {
+        bool isProvider; // determines whether or not this actor supports aim assist.
+        s16 sfx;         // The sound to play when this actor provides aim assist. Uses sound slot 9.
+        f32 tolerance;   // How close to the center of the actor does Link have to aim for aim assist to consider
+                         // it lined up.
+    } aimAssist;
+};
+
+// Accessible actor object. This can be a "real" actor (one that corresponds to an actual actor in the game) or a
+// "virtual" actor (which does not actually exist in the game, but is used to create extra sounds for the player).
+// One potential use of virtual actors is to place sounds at static platforms or other things that aren't represented by
+// actors.
+
+struct AccessibleActor {
+    uint64_t instanceID;
+
+    Actor* actor; // This can be null for a virtual actor.
+    s16 id; // For real actors, we copy the ID of the actor. For virtual actors we have our own table of values which
+            // are out of range for real actors.
+    f32 yDistToPlayer;
+    f32 xzDistToPlayer;
+    f32 xyzDistToPlayer;
+    PosRot world;
+    Vec3f projectedPos;
+    PlayState* play;
+    u8 isDrawn; // Do we just never play accessibility sounds for actors that aren't drawn?
+
+    int frameCount; // Incremented every time the callback is called. The callback is free to modify this. Can be used
+                    // to implement playback of sounds at regular intervals.
+    f32 baseVolume;
+    f32 currentVolume;
+    f32 basePitch;
+
+    f32 currentPitch;
+    s16 sceneIndex; // If this actor represents a scene transition, then this will contain the destination scene index.
+                    // Zero otherwise.
+    u16 managedSoundSlots; // These have their attenuation and panning parameters updated every frame automatically.
+    struct {
+        u16 framesSinceAimAssist; // Allows rate-based vertical aim assist. Incremented every frame for aim assist
+                                  // actors. Manually reset by aim assist provider.
+        f32 pitch;                // Used to report whether Link is aiming higher or lower than the actor.
+        u8 frequency; // How often the sound will be played. Lower frequencies indicate that Link's vertical aim is
+                      // closer to the actor.
+    } aimAssist;
+
+    // Add more state as needed.
+    ActorAccessibilityPolicy policy; // A copy, so it can be customized on a per-actor basis if needed.
+    void* userData;                  // Set by the policy. Can be anything.
+};
+
+// Initialize accessibility.
+void ActorAccessibility_Init();
+void ActorAccessibility_InitActors();
+void ActorAccessibility_Shutdown();
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName,
+                                   ActorAccessibilityCallback callback);
+void ActorAccessibility_InitPolicy(ActorAccessibilityPolicy* policy, const char* englishName, s16 sfx);
+
+void ActorAccessibility_TrackNewActor(Actor* actor);
+void ActorAccessibility_RemoveTrackedActor(Actor* actor);
+void ActorAccessibility_AddSupportedActor(s16 type, ActorAccessibilityPolicy policy);
+
+void ActorAccessibility_RunAccessibilityForActor(PlayState* play, AccessibleActor* actor);
+void ActorAccessibility_RunAccessibilityForAllActors(PlayState* play);
+/*
+ *Play sounds (usually from the game) using the external sound engine. This is probably not the function you want to
+ *call most of the time (see below). handle: pointer to an arbitrary object. This object can be anything as it's only
+ *used as a classifier, but it's recommended that you use an AccessibleActor* as your handle whenever possible. Using
+ *AccessibleActor* as the handle gives you automatic cleanup when the actor is killed. slot: Allows multiple sounds to
+ *be assigned to a single handle. The maximum number of slots per actor is 10 by default (but can be controlled by
+ *modifying AAE_SLOTS_PER_HANDLE). sfxId: one of the game's sfx IDs. Note that this plays prerendered sounds which you
+ *must have previously prepared. looping: whether to play the sound just once or on a continuous loop.
+ */
+void ActorAccessibility_PlaySound(void* actor, int slot, s16 sfxId, bool looping);
+// Stop a sound. Todo: consider making this a short fade instead of just cutting it off.
+void ActorAccessibility_StopSound(void* handle, int slot);
+void ActorAccessibility_StopAllSounds(void* handle);
+
+void ActorAccessibility_SetSoundPitch(void* handle, int slot, float pitch);
+// When we don't have access to something super fancy (such as HRTF), blind-accessible games generally use a change in
+// pitch to tell the player that an object is behind the player.
+void ActorAccessibility_SetPitchBehindModifier(void* handle, int slot, float mod);
+
+void ActorAccessibility_SetListenerPos(Vec3f* pos, Vec3f* rot);
+void ActorAccessibility_SetSoundPos(void* handle, int slot, Vec3f* pos, f32 distToPlayer, f32 maxDistance);
+
+void ActorAccessibility_SetSoundVolume(void* handle, int slot, float volume);
+void ActorAccessibility_SetSoundPan(void* handle, int slot, Vec3f* projectedPos);
+void ActorAccessibility_SetSoundFilter(void* handle, int slot, float cutoff);
+void ActorAccessibility_SeekSound(void* handle, int slot, size_t offset);
+
+/*
+ * Play a sound on behalf of an AccessibleActor.
+ * This version includes automatic sound management: pitch, panning and attenuation parameters will be updated
+ * automatically based on the actor's position.
+ *
+ */
+void ActorAccessibility_PlaySoundForActor(AccessibleActor* actor, int slot, s16 sfxId, bool looping);
+
+void ActorAccessibility_StopSoundForActor(AccessibleActor* actor, int slot);
+void ActorAccessibility_StopAllSoundsForActor(AccessibleActor* actor);
+f32 ActorAccessibility_ComputeCurrentVolume(f32 maxDistance, f32 xzDistToPlayer);
+// Computes a relative angle based on Link's (or some other actor's) current angle.
+Vec3s ActorAccessibility_ComputeRelativeAngle(Vec3s* origin, Vec3s* offset);
+void ActorAccessibility_InitCues();
+// Stuff related to lists of virtual actors.
+typedef enum {
+    // Similar to the game's actual actor table. Values here start at 10000 just to be extra safe.
+    VA_INITIAL = 1000,
+    VA_PROTOTYPE, // Remove this one once this thing is working.
+    VA_CRAWLSPACE,
+    VA_TERRAIN_CUE,
+    VA_WALL_CUE,
+    VA_CLIMB,
+    VA_DOOR,
+    VA_AREA_CHANGE,
+    VA_MARKER,
+    VA_SPIKE,
+    VA_GENERAL_HELPER, // Room announcements, action icon and other misc help.
+    VA_AUDIO_COMPASS,  // Points north.
+    VA_STICK_WARNING,  // beep when stick is about to burn out.
+    VA_MAX,
+} VIRTUAL_ACTOR_TABLE;
+
+#define EVERYWHERE -32768 // Denotes a virtual actor that is global
+
+// Get the list of virtual actors for a given scene and room index.
+VirtualActorList* ActorAccessibility_GetVirtualActorList(s16 sceneNum, s8 roomNum);
+AccessibleActor* ActorAccessibility_AddVirtualActor(VirtualActorList* list, VIRTUAL_ACTOR_TABLE type, PosRot where);
+// Parses the loaded seen and converts select polygons (like ladders, spikes and scene exits) into virtual actors.
+void ActorAccessibility_InterpretCurrentScene(PlayState* play);
+// Convert a collision polygon into a virtual actor.
+void ActorAccessibility_PolyToVirtualActor(PlayState* play, CollisionPoly* poly, VIRTUAL_ACTOR_TABLE va,
+                                           VirtualActorList* destination);
+// Report which room of a dungeon the player is in.
+void ActorAccessibility_AnnounceRoomNumber(PlayState* play);
+// Aim cue support.
+void ActorAccessibility_ProvideAimAssistForActor(AccessibleActor* actor);
+// External audio engine stuff.
+//  Initialize the accessible audio engine.
+bool ActorAccessibility_InitAudio();
+void ActorAccessibility_ShutdownAudio();
+// Combine the games' audio with the output from AccessibleAudioEngine. To be called exclusively from the audio thread.
+void ActorAccessibility_MixAccessibleAudioWithGameAudio(int16_t* ogBuffer, uint32_t nFrames);
+void ActorAccessibility_HandleSoundExtractionMode(PlayState* play);
+// This is called by the audio thread when it's ready to try to pull sfx from the game.
+void ActorAccessibility_DoSoundExtractionStep();
+
+void ActorAccessibility_AudioGlossary(PlayState* play);

soh/soh/Enhancements/accessible-actors/SfxExtractor.cpp

@@ -0,0 +1,220 @@
+#include "SfxExtractor.h"
+#include "soh/Enhancements/audio/miniaudio.h"
+#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
+#include "soh/Enhancements/tts/tts.h"
+#include "soh/OTRGlobals.h"
+#include "SfxTable.h"
+#include <sstream>
+extern "C" {
+#include "z64.h"
+#include "functions.h"
+#include "variables.h"
+void AudioMgr_CreateNextAudioBuffer(s16* samples, u32 num_samples);
+extern bool freezeGame;
+}
+
+bool SfxExtractor::isAllZero(int16_t* buffer, size_t count) {
+    for (size_t i = 0; i < count; i++) {
+        if (buffer[i] != 0)
+            return false;
+    }
+    return true;
+}
+
+// Find the beginning of a captured signal.
+size_t SfxExtractor::adjustedStartOfInput() {
+    size_t startOfInput = 0;
+    while (startOfInput + 2 < SFX_EXTRACTION_BUFFER_SIZE * 2 &&
+           (tempBuffer[startOfInput] == 0 || tempBuffer[startOfInput + 1] == 0)) {
+        startOfInput += 2;
+    }
+    return startOfInput;
+}
+
+size_t SfxExtractor::adjustedEndOfInput(size_t endOfInput) {
+    while (endOfInput > 0 && (tempBuffer[endOfInput] == 0 || tempBuffer[endOfInput - 1] == 0)) {
+        endOfInput -= 2;
+    }
+    return endOfInput;
+}
+
+bool SfxExtractor::renderOutput(size_t endOfInput) {
+    size_t startOfInput = adjustedStartOfInput();
+    endOfInput = adjustedEndOfInput(endOfInput);
+    if (endOfInput <= startOfInput) {
+        return true;
+    }
+
+    ma_channel_converter_config config =
+        ma_channel_converter_config_init(ma_format_s16, 2, NULL, 1, NULL, ma_channel_mix_mode_default);
+    ma_channel_converter converter;
+    if (ma_channel_converter_init(&config, NULL, &converter) != MA_SUCCESS) {
+        return false;
+    }
+    std::vector<uint8_t> fileData;
+    std::string fileName = getExternalFileName(currentSfx);
+    int16_t chunk[64];
+    int16_t* mark = tempBuffer + startOfInput;
+    while (mark < tempBuffer + endOfInput) {
+        size_t chunkSize = std::min<size_t>(64, ((tempBuffer + endOfInput) - mark) / 2);
+        ma_result converter_result = ma_channel_converter_process_pcm_frames(&converter, chunk, mark, chunkSize);
+        if (converter_result != MA_SUCCESS) {
+            return false;
+        }
+        fileData.insert(fileData.end(), (uint8_t*)chunk, (uint8_t*)(chunk + chunkSize));
+        mark += chunkSize * 2;
+    }
+    return archive->WriteFile(fileName.c_str(), fileData);
+}
+
+void SfxExtractor::setup() {
+    try {
+        SpeechSynthesizer::Instance->Speak(
+            "Sfx extraction speedrun initiated. Please wait. This will take a few minutes.", "en-US");
+        // Kill the audio thread so we can take control.
+        captureThreadState = CT_WAITING;
+        OTRAudio_InstallSfxCaptureThread();
+        // Make sure we're starting from a clean slate.
+        std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
+        if (std::filesystem::exists(sohAccessibilityPath)) {
+            currentStep = STEP_ERROR_FILE_EXISTS;
+            return;
+        }
+
+        sfxToRip = 0;
+        currentStep = STEP_MAIN;
+        archive = std::make_shared<Ship::O2rArchive>(sohAccessibilityPath);
+        archive->Open();
+    } catch (...) { currentStep = STEP_ERROR; }
+}
+
+void SfxExtractor::ripNextSfx() {
+    {
+        auto lock = OTRAudio_Lock();
+        if (captureThreadState == CT_READY || captureThreadState == CT_PRIMING)
+            return; // Keep going.
+    }
+    // Was the last sfx a loop? If so then we need to stop it, and then we need to run audio out to nowhere for as long
+    // as it takes to get back to a blank slate.
+    if (currentSfx != -1) {
+        Audio_StopSfxByPos(&gSfxDefaultPos);
+        captureThreadState = CT_PRIMING;
+        currentSfx = -1;
+
+        return;
+    }
+    if (sfxToRip == sfxCount) {
+        currentStep = STEP_FINISHED; // Caught 'em all!
+        return;
+    }
+
+    currentSfx = sfxTable[sfxToRip++];
+    Audio_PlaySoundGeneral(currentSfx, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale, &gSfxDefaultFreqAndVolScale,
+                           &gSfxDefaultReverb);
+
+    {
+        auto lock = OTRAudio_Lock();
+        captureThreadState = CT_READY;
+    }
+    maybeGiveProgressReport();
+}
+void SfxExtractor::finished() {
+    OTRAudio_UninstallSfxCaptureThread(); // Returns to normal audio opperation.
+    CVarClear("gExtractSfx");
+    CVarSave();
+    archive->Close();
+    archive = nullptr;
+    freezeGame = false;
+
+    Audio_QueueSeqCmd(NA_BGM_TITLE);
+
+    if (currentStep >= STEP_ERROR) {
+        Audio_PlaySoundGeneral(NA_SE_SY_ERROR, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
+                               &gSfxDefaultFreqAndVolScale, &gSfxDefaultReverb);
+        Audio_PlaySoundGeneral(NA_SE_EN_GANON_LAUGH, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
+                               &gSfxDefaultFreqAndVolScale, &gSfxDefaultReverb);
+        std::stringstream ss;
+        ss << "Sorry, we tried to extract the sound effects, but Ganondorf overruled us with an iron fist."
+           << std::endl;
+        if (currentStep == STEP_ERROR_FILE_EXISTS)
+            ss << "In all seriousness, please delete accessibility.o2r and try again.";
+        SpeechSynthesizer::Instance->Speak(ss.str().c_str(), "en-US");
+    } else
+        Audio_PlayFanfare(NA_BGM_ITEM_GET);
+}
+void SfxExtractor::maybeGiveProgressReport() {
+    for (int i = 0; i < 9; i++) {
+        if (sfxToRip == sfxCount * (i + 1) / 10) {
+            std::stringstream ss;
+            ss << (i + 1) * 10 << " percent complete.";
+            SpeechSynthesizer::Instance->Speak(ss.str().c_str(), "en-US");
+        }
+    }
+}
+
+SfxExtractor::SfxExtractor() {
+    currentStep = STEP_SETUP;
+}
+
+void SfxExtractor::frameCallback() {
+    switch (currentStep) {
+        case STEP_SETUP:
+            setup();
+            break;
+        case STEP_MAIN:
+            ripNextSfx();
+            break;
+        default: // Handles finished as well as a number of error conditions.
+            finished();
+    }
+}
+
+void SfxExtractor::prime() {
+    while (true) {
+        AudioMgr_CreateNextAudioBuffer(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME);
+        if (isAllZero(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME * 2))
+            break;
+    }
+    captureThreadState = CT_FINISHED;
+}
+
+void SfxExtractor::captureCallback() {
+    if (captureThreadState == CT_PRIMING)
+        prime();
+    if (captureThreadState != CT_READY)
+        return; // No work to do at the moment.
+    memset(tempBuffer, 0, SFX_EXTRACTION_BUFFER_SIZE * 4);
+    int16_t* mark = tempBuffer + 0;
+    size_t samplesLeft = SFX_EXTRACTION_BUFFER_SIZE;
+    bool outputStarted = false;
+    int waitTime = 0;
+    while (samplesLeft > 0) {
+        AudioMgr_CreateNextAudioBuffer(mark, SFX_EXTRACTION_ONE_FRAME);
+
+        if (isAllZero(mark, SFX_EXTRACTION_ONE_FRAME * 2)) {
+            if (outputStarted) {
+                break;
+            } else if (waitTime++ < 300) {
+                continue; // Output is silent, allow more time for audio to begin.
+            }
+            captureThreadState = CT_FINISHED; // Sound is unavailable, so skip over it and move on.
+            return;
+        }
+
+        outputStarted = true;
+        size_t samples = std::min<size_t>(SFX_EXTRACTION_ONE_FRAME, samplesLeft);
+        mark += samples * 2;
+        samplesLeft -= samples;
+    }
+    if (renderOutput(mark - tempBuffer)) {
+        captureThreadState = CT_FINISHED;
+    } else {
+        SPDLOG_ERROR("failed to write file to archive, trying again");
+    }
+}
+
+std::string SfxExtractor::getExternalFileName(int16_t sfxId) {
+    std::stringstream ss;
+    ss << "accessibility/audio/" << std::hex << std::setw(4) << std::setfill('0') << sfxId << ".wav";
+    return ss.str();
+}

soh/soh/Enhancements/accessible-actors/SfxExtractor.h

@@ -0,0 +1,49 @@
+#pragma once
+#include "libultraship/libultraship.h"
+
+#define SFX_EXTRACTION_BUFFER_SIZE 44100 * 15
+#define SFX_EXTRACTION_ONE_FRAME 736
+
+enum CaptureThreadStates {
+    CT_WAITING, // for a sound to start ripping.
+    CT_PRIMING,
+    CT_READY,    // to start ripping a sound.
+    CT_FINISHED, // ripping the current sound.
+    CT_SHUTDOWN,
+};
+
+enum SfxExtractionSteps {
+    STEP_SETUP = 0,
+    STEP_MAIN,
+    STEP_FINISHED,
+    STEP_ERROR,
+    STEP_ERROR_FILE_EXISTS,
+};
+
+class SfxExtractor {
+    std::shared_ptr<Ship::Archive> archive;
+    SfxExtractionSteps currentStep;
+    CaptureThreadStates captureThreadState;
+    int sfxToRip;
+    s16 currentSfx;
+    // Stores raw audio data for the sfx currently being ripped.
+    int16_t tempBuffer[(SFX_EXTRACTION_BUFFER_SIZE + SFX_EXTRACTION_ONE_FRAME * 3) * 2];
+    // Check if a buffer contains meaningful audio output.
+    bool isAllZero(int16_t* buffer, size_t count);
+    size_t adjustedStartOfInput();
+    size_t adjustedEndOfInput(size_t endOfInput);
+    bool renderOutput(size_t endOfInput);
+    void setup();
+    void ripNextSfx();
+    void finished(); // Also handles failure.
+    void maybeGiveProgressReport();
+
+  public:
+    SfxExtractor();
+
+    void frameCallback();
+    void prime();
+    // The below is called by the (hijacked) audio thread.
+    void captureCallback();
+    static std::string getExternalFileName(int16_t sfxId);
+};

soh/soh/Enhancements/audio/miniaudio.h

@@ -0,0 +1,9 @@
+#pragma once
+#define MA_NO_FLAC
+#define MA_NO_MP3
+#define MA_NO_THREADING
+#define MA_NO_DEVICE_IO
+#define MA_NO_GENERATION
+#define MA_NO_STDIO
+#define MA_ENABLE_ONLY_SPECIFIC_BACKENDS
+#include "miniaudio/miniaudio.h"

soh/soh/Enhancements/game-interactor/GameInteractor_HookTable.h

@@ -27,6 +27,7 @@ DEFINE_HOOK(OnCuccoOrChickenHatch, ());
 DEFINE_HOOK(OnShopSlotChange, (uint8_t cursorIndex, int16_t price));
 DEFINE_HOOK(OnActorInit, (void* actor));
 DEFINE_HOOK(OnActorUpdate, (void* actor));
+DEFINE_HOOK(OnActorDestroy, (void* actor));
 DEFINE_HOOK(OnActorKill, (void* actor));
 DEFINE_HOOK(OnEnemyDefeat, (void* actor));
 DEFINE_HOOK(OnBossDefeat, (void* actor));
@@ -65,6 +66,7 @@ DEFINE_HOOK(OnUpdateFileBossRushOptionSelection, (uint8_t optionIndex, uint8_t o
 DEFINE_HOOK(OnUpdateFileNameSelection, (int16_t charCode));
 
 DEFINE_HOOK(OnSetGameLanguage, ());
+DEFINE_HOOK(OnGameStillFrozen, ());
 DEFINE_HOOK(OnFileDropped, (std::string filePath));
 DEFINE_HOOK(OnAssetAltChange, ());
 DEFINE_HOOK(OnKaleidoUpdate, ());

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.cpp

@@ -110,7 +110,9 @@ void GameInteractor_ExecuteOnActorUpdate(void* actor) {
     GameInteractor::Instance->ExecuteHooksForPtr<GameInteractor::OnActorUpdate>((uintptr_t)actor, actor);
     GameInteractor::Instance->ExecuteHooksForFilter<GameInteractor::OnActorUpdate>(actor);
 }
-
+void GameInteractor_ExecuteOnActorDestroy(void* actor) {
+    GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorDestroy>(actor);
+}
 void GameInteractor_ExecuteOnActorKill(void* actor) {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnActorKill>(actor);
     GameInteractor::Instance->ExecuteHooksForID<GameInteractor::OnActorKill>(((Actor*)actor)->id, actor);
@@ -286,6 +288,10 @@ void GameInteractor_ExecuteOnSetGameLanguage() {
     GameInteractor::Instance->ExecuteHooks<GameInteractor::OnSetGameLanguage>();
 }
 
+void GameInteractor_ExecuteOnGameStillFrozen() {
+    GameInteractor::Instance->ExecuteHooks<GameInteractor::OnGameStillFrozen>();
+}
+
 // MARK: - System
 
 void GameInteractor_RegisterOnAssetAltChange(void (*fn)(void)) {

soh/soh/Enhancements/game-interactor/GameInteractor_Hooks.h

@@ -29,6 +29,7 @@ void GameInteractor_ExecuteOnOcarinaSongAction();
 void GameInteractor_ExecuteOnCuccoOrChickenHatch();
 void GameInteractor_ExecuteOnActorInit(void* actor);
 void GameInteractor_ExecuteOnActorUpdate(void* actor);
+void GameInteractor_ExecuteOnActorDestroy(void* actor);
 void GameInteractor_ExecuteOnActorKill(void* actor);
 void GameInteractor_ExecuteOnEnemyDefeat(void* actor);
 void GameInteractor_ExecuteOnBossDefeat(void* actor);
@@ -74,6 +75,8 @@ void GameInteractor_ExecuteOnUpdateFileNameSelection(int16_t charCode);
 // MARK: - Game
 void GameInteractor_ExecuteOnSetGameLanguage();
 
+void GameInteractor_ExecuteOnGameStillFrozen();
+
 // MARK: - System
 void GameInteractor_RegisterOnAssetAltChange(void (*fn)(void));
 

soh/soh/OTRGlobals.cpp

@@ -128,6 +128,9 @@ Sail* Sail::Instance;
 
 #include "soh/config/ConfigUpdaters.h"
 #include "soh/ShipInit.hpp"
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+#include "Enhancements/accessible-actors/ActorAccessibility.h"
+#endif
 
 extern "C" {
 #include "src/overlays/actors/ovl_En_Dns/z_en_dns.h"
@@ -290,6 +293,12 @@ OTRGlobals::OTRGlobals() {
             }
         }
     }
+
+    std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
+    if (std::filesystem::exists(sohAccessibilityPath)) {
+        OTRFiles.push_back(sohAccessibilityPath);
+    }
+
     std::sort(patchOTRs.begin(), patchOTRs.end(), [](const std::string& a, const std::string& b) {
         return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end(),
                                             [](char c1, char c2) { return std::tolower(c1) < std::tolower(c2); });
@@ -580,6 +589,11 @@ void OTRAudio_Thread() {
         for (int i = 0; i < AUDIO_FRAMES_PER_UPDATE; i++) {
             AudioMgr_CreateNextAudioBuffer(audio_buffer + i * (num_audio_samples * NUM_AUDIO_CHANNELS),
                                            num_audio_samples);
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+            // Give accessibility a chance to merge its own audio in.
+            ActorAccessibility_MixAccessibleAudioWithGameAudio(
+                audio_buffer + i * (num_audio_samples * NUM_AUDIO_CHANNELS), num_audio_samples);
+#endif
         }
 
         AudioPlayer_Play((u8*)audio_buffer,
@@ -1239,6 +1253,9 @@ extern "C" void InitOTR() {
 #endif
 
     OTRMessage_Init();
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+    ActorAccessibility_Init();
+#endif
     OTRAudio_Init();
     OTRExtScanner();
     VanillaItemTable_Init();
@@ -1290,6 +1307,9 @@ extern "C" void DeinitOTR() {
     }
     SDLNet_Quit();
 #endif
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+    ActorAccessibility_Shutdown();
+#endif
 
     // Destroying gui here because we have shared ptrs to LUS objects which output to SPDLOG which is destroyed before
     // these shared ptrs.
@@ -2513,6 +2533,41 @@ extern "C" void Gfx_RegisterBlendedTexture(const char* name, u8* mask, u8* repla
     gfx_register_blended_texture(name, mask, replacement);
 }
 
+void OTRAudio_SfxCaptureThread() {
+    while (audio.running) {
+        {
+            std::unique_lock<std::mutex> Lock(audio.mutex);
+            while (!audio.processing && audio.running) {
+                audio.cv_to_thread.wait(Lock);
+            }
+
+            if (!audio.running) {
+                break;
+            }
+        }
+        std::unique_lock<std::mutex> Lock(audio.mutex);
+#if !defined(__SWITCH__) && !defined(__WIIU__)
+        ActorAccessibility_DoSoundExtractionStep();
+#endif
+        audio.processing = false;
+        audio.cv_from_thread.notify_one();
+    }
+}
+
+extern "C" void OTRAudio_InstallSfxCaptureThread() {
+    OTRAudio_Exit();
+    audio.running = true;
+    audio.thread = std::thread(OTRAudio_SfxCaptureThread);
+}
+extern "C" void OTRAudio_UninstallSfxCaptureThread() {
+    OTRAudio_Exit();
+    audio.running = true;
+    audio.thread = std::thread(OTRAudio_Thread);
+}
+std::unique_lock<std::mutex> OTRAudio_Lock() {
+    return std::unique_lock<std::mutex>(audio.mutex);
+}
+
 extern "C" void Gfx_UnregisterBlendedTexture(const char* name) {
     gfx_unregister_blended_texture(name);
 }

soh/soh/OTRGlobals.h

@@ -164,6 +164,7 @@ void Gfx_TextureCacheDelete(const uint8_t* addr);
 void SaveManager_ThreadPoolWait();
 void CheckTracker_OnMessageClose();
 
+int32_t GetGIID(uint32_t itemID);
 GetItemID RetrieveGetItemIDFromItemID(ItemID itemID);
 RandomizerGet RetrieveRandomizerGetFromItemID(ItemID itemID);
 #endif
@@ -172,8 +173,12 @@ RandomizerGet RetrieveRandomizerGetFromItemID(ItemID itemID);
 extern "C" {
 #endif
 uint64_t GetUnixTimestamp();
+void OTRAudio_InstallSfxCaptureThread();
+void OTRAudio_UninstallSfxCaptureThread();
 #ifdef __cplusplus
 };
+std::unique_lock<std::mutex> OTRAudio_Lock();
+
 #endif
 
 #endif

soh/soh/SohGui/SohMenuSettings.cpp

@@ -195,6 +195,18 @@ void SohMenu::AddMenuSettings() {
         .CVar(CVAR_SETTING("A11yDisableIdleCam"))
         .RaceDisable(false)
         .Options(CheckboxOptions().Tooltip("Disables the automatic re-centering of the camera when idle."));
+
+    AddWidget(path, "Accessible Audio Cues", WIDGET_CVAR_CHECKBOX)
+        .CVar(CVAR_SETTING("A11yAudioInteraction"))
+        .RaceDisable(false)
+        .Options(CheckboxOptions().Tooltip("Enables accessibility audio cues"));
+
+    AddWidget(path, "Extract Sfx", WIDGET_CVAR_CHECKBOX)
+        .CVar("gExtractSfx")
+        .RaceDisable(false)
+        .Options(CheckboxOptions().Tooltip("Extracts the sfx to be used in accessible audio cues, must be run once for "
+                                           "the audio cues to play then restart game"));
+
     AddWidget(path, "EXPERIMENTAL", WIDGET_SEPARATOR_TEXT).Options(TextOptions().Color(Colors::Orange));
     AddWidget(path, "ImGui Menu Scaling", WIDGET_CVAR_COMBOBOX)
         .CVar(CVAR_SETTING("ImGuiScale"))

soh/src/code/code_800F9280.c

@@ -3,6 +3,7 @@
 #include "soh/mixer.h"
 
 #include "soh/Enhancements/audio/AudioEditor.h"
+extern bool freezeGame;
 
 typedef struct {
     u8 unk_0;
@@ -372,6 +373,9 @@ extern f32 D_80130F24;
 extern f32 D_80130F28;
 
 void Audio_QueueSeqCmd(u32 cmd) {
+    if (freezeGame)
+        return; // No music during SFX rip.
+                //
     u8 op = cmd >> 28;
     if (op == 0 || op == 2 || op == 12) {
         u8 seqId = cmd & 0xFF;

soh/src/code/z_actor.c

@@ -80,6 +80,7 @@
 #include "textures/place_title_cards/g_pn_56.h"
 #include "textures/place_title_cards/g_pn_57.h"
 #endif
+bool freezeActors = false;
 
 static CollisionPoly* sCurCeilingPoly;
 static s32 sCurCeilingBgId;
@@ -1267,6 +1268,7 @@ void Actor_Init(Actor* actor, PlayState* play) {
 }
 
 void Actor_Destroy(Actor* actor, PlayState* play) {
+    GameInteractor_ExecuteOnActorDestroy(actor);
     if (actor->destroy != NULL) {
         actor->destroy(actor, play);
         actor->destroy = NULL;
@@ -2556,6 +2558,7 @@ u32 D_80116068[ACTORCAT_MAX] = {
 };
 
 void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
+
     Actor* refActor;
     Actor* actor;
     Player* player;
@@ -2571,6 +2574,11 @@ void Actor_UpdateAll(PlayState* play, ActorContext* actorCtx) {
     sp74 = NULL;
     unkFlag = 0;
 
+    if (freezeActors) {
+        GameInteractor_ExecuteOnPlayerUpdate(player);
+        return; // for AudioGlossary
+    }
+
     if (play->numSetupActors != 0) {
         actorEntry = &play->setupActorList[0];
         for (i = 0; i < play->numSetupActors; i++) {

soh/src/code/z_play.c

@@ -21,6 +21,7 @@
 #include <time.h>
 #include <assert.h>
 
+bool freezeGame = false; // Used for SFX ripper.
 TransitionUnk sTrnsnUnk;
 s32 gTrnsnUnkState;
 VisMono gPlayVisMono;
@@ -694,6 +695,11 @@ void Play_Update(PlayState* play) {
     s32 isPaused;
     s32 pad1;
 
+    if (freezeGame) {
+        GameInteractor_ExecuteOnGameStillFrozen();
+        return;
+    }
+
     if ((SREG(1) < 0) || (DREG(0) != 0)) {
         SREG(1) = 0;
         ZeldaArena_Display();