store sfx as raw data, removes explicit dr_wav dependency

This commit is contained in:
Demur Rumed 2025-04-25 20:05:50 +00:00
parent 4ffc47c237
commit 3ec02994fb
11 changed files with 124 additions and 136 deletions

3
.gitmodules vendored
View file

@ -7,9 +7,6 @@
[submodule "OTRExporter"]
path = OTRExporter
url = https://github.com/harbourmasters/OTRExporter
[submodule "dr_libs"]
path = soh/include/dr_libs
url = https://github.com/mackron/dr_libs
[submodule "miniaudio"]
path = soh/include/miniaudio
url = https://github.com/mackron/miniaudio

@ -1 +0,0 @@
Subproject commit 9cb7092ac8c75a82b5c6ea72652ca8d0091d7ffa

View file

@ -1,10 +1,3 @@
#define MINIAUDIO_IMPLEMENTATION
#define MA_NO_THREADING
#define MA_NO_DEVICE_IO
#define MA_NO_GENERATION
#define MA_NO_FLAC
#define MA_NO_MP3
#define MA_ENABLE_ONLY_SPECIFIC_BACKENDS
#define AAE_CHANNELS 2
#define AAE_SAMPLE_RATE 44100
#define AAE_MAX_BUFFER_SIZE AAE_SAMPLE_RATE / 10
@ -23,6 +16,8 @@ int AudioPlayer_GetDesiredBuffered();
#include <math.h>
#include <algorithm>
#include <stdexcept>
#include <spdlog/spdlog.h>
enum AAE_COMMANDS {
AAE_START = 0,
AAE_STOP,
@ -123,10 +118,9 @@ uint32_t AccessibleAudioEngine::retrieve(float* buffer, uint32_t nFrames) {
if (nFrames == 0)
return 0;
uint32_t ogNFrames = nFrames;
uint32_t framesObtained = 0;
while (nFrames > 0) {
void* readBuffer;
framesObtained = nFrames;
uint32_t framesObtained = nFrames;
ma_pcm_rb_acquire_read(&preparedOutput, &framesObtained, (void**)&readBuffer);
if (framesObtained > nFrames)
framesObtained = nFrames;
@ -246,6 +240,7 @@ void AccessibleAudioEngine::runThread() {
}
}
}
SoundSlot* AccessibleAudioEngine::findSound(SoundAction& action) {
if (action.slot < 0 || action.slot >= AAE_SLOTS_PER_HANDLE)
return NULL;
@ -257,6 +252,7 @@ SoundSlot* AccessibleAudioEngine::findSound(SoundAction& action) {
return NULL;
return &target;
}
void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
SoundSlot* sound;
if (sounds.contains(action.handle)) {
@ -266,7 +262,6 @@ void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
destroySound(sound);
}
}
else {
SoundSlots temp;
for (int i = 0; i < AAE_SLOTS_PER_HANDLE; i++)
@ -275,10 +270,14 @@ void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
sounds[action.handle] = temp;
sound = &sounds[action.handle][action.slot];
}
if (ma_sound_init_from_file(&engine, action.path.c_str(),
ma_result result = ma_sound_init_from_file(&engine, action.path.c_str(),
MA_SOUND_FLAG_NO_SPATIALIZATION | MA_SOUND_FLAG_NO_DEFAULT_ATTACHMENT, NULL, NULL,
&sound->sound) != MA_SUCCESS)
&sound->sound);
if (result != MA_SUCCESS) {
SPDLOG_ERROR("failed to play sound: {}", ma_result_description(result));
return;
}
initSoundExtras(sound);
ma_sound_set_looping(&sound->sound, action.looping);
@ -289,6 +288,7 @@ void AccessibleAudioEngine::doPlaySound(SoundAction& action) {
sound->active = true;
}
void AccessibleAudioEngine::doStopSound(SoundAction& action) {
SoundSlot* slot = findSound(action);
if (slot == NULL)
@ -416,6 +416,7 @@ bool AccessibleAudioEngine::initSoundExtras(SoundSlot* slot) {
ma_node_attach_output_bus(&slot->sound, 0, &slot->extras, 0);
return true;
}
void AccessibleAudioEngine::destroySound(SoundSlot* slot) {
ma_node_detach_all_output_buses(&slot->extras);
ma_sound_uninit(&slot->sound);
@ -461,7 +462,6 @@ AccessibleAudioEngine::~AccessibleAudioEngine() {
destroy();
}
void AccessibleAudioEngine::mix(int16_t* ogBuffer, uint32_t nFrames) {
uint32_t framesAvailable = ma_pcm_rb_available_read(&preparedOutput);
float sourceChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
float mixedChunk[AAE_MIX_CHUNK_SIZE * AAE_CHANNELS];
while (nFrames > 0) {
@ -596,9 +596,4 @@ void AccessibleAudioEngine::prepare() {
action.command = AAE_PREPARE;
// This is called once at the end of every frame, so now is the time to post all of the accumulated commands.
postSoundActions();
}
void AccessibleAudioEngine::cacheDecodedSample(const char* path, void* data, size_t size) {
// data stored as wave, so we register it with MiniAudio as an encoded asset as opposed to a decoded one
ma_resource_manager_register_encoded_data(&resourceManager, path, data, size);
}

View file

@ -1,11 +1,4 @@
#pragma once
#define MA_NO_FLAC
#define MA_NO_MP3
#define MA_NO_THREADING
#define MA_NO_DEVICE_IO
#define MA_NO_GENERATION
#include "miniaudio/miniaudio.h"
#include <stdint.h>
#include <thread>
#include <mutex>
@ -14,6 +7,9 @@
#include <string>
#include <unordered_map>
#include <array>
#include "soh/Enhancements/audio/miniaudio.h"
#define AAE_SOUND_ACTION_BATCH_SIZE 64
#define AAE_SLOTS_PER_HANDLE 16
class IResource;
@ -71,7 +67,6 @@ typedef std::array<SoundSlot, AAE_SLOTS_PER_HANDLE> SoundSlots;
class AccessibleAudioEngine {
int initialized;
ma_resource_manager resourceManager;
ma_engine engine;
ma_pcm_rb preparedOutput; // Lock-free single producer single consumer.
std::deque<SoundAction> soundActions; // A command cue.
@ -152,5 +147,6 @@ class AccessibleAudioEngine {
float maxDistance);
// Schedule the preparation of output for delivery.
void prepare();
void cacheDecodedSample(const char* path, void* data, size_t size);
ma_resource_manager resourceManager;
};

View file

@ -90,7 +90,7 @@ class ActorAccessibility {
// Maps internal sfx to external (prerendered) resources.
std::unordered_map<s16, SfxRecord> sfxMap;
// Similar to above, but this one maps raw audio samples as opposed to SFX.
std::unordered_set<const char*> sampleMap;
std::unordered_map<const char*, std::vector<uint8_t>> sampleMap;
int extractSfx = 0;
s16 currentScene = -1;
s8 currentRoom = -1;
@ -706,16 +706,17 @@ const char* ActorAccessibility_MapSfxToExternalAudio(s16 sfxId) {
std::stringstream ss;
ss << std::setw(4) << std::setfill('0') << std::hex << sfxId;
tempRecord.path = ss.str();
aa->sfxMap[sfxId] = tempRecord;
record = &aa->sfxMap[sfxId];
aa->audioEngine->cacheDecodedSample(record->path.c_str(), record->resource->Buffer->data(),
record->resource->Buffer->size());
auto pair = aa->sfxMap.insert({ sfxId, tempRecord });
record = &pair.first->second;
ma_resource_manager_register_decoded_data(&aa->audioEngine->resourceManager, record->path.c_str(),
record->resource->Buffer->data(), record->resource->Buffer->size() / 2, ma_format_s16, 1, 44100);
} else {
record = &it->second;
}
return record->path.c_str();
}
// Map the path to a raw sample to the external audio engine.
const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name) {
auto it = aa->sampleMap.find(name);
@ -728,11 +729,9 @@ const char* ActorAccessibility_MapRawSampleToExternalAudio(const char* name) {
return NULL; // Resource doesn't exist, user's gotta run the extractor.
AudioDecoder decoder;
decoder.setSample((SOH::AudioSample*)res.get());
// TODO track wav somehow & free it with drwav_free
s16* wav;
size_t wavSize = decoder.decodeToWav(&wav);
aa->sampleMap.insert(name);
aa->audioEngine->cacheDecodedSample(name, wav, wavSize);
auto pair = aa->sampleMap.insert({ name, decoder.decodeToWav() });
ma_resource_manager_register_encoded_data(&aa->audioEngine->resourceManager, name,
pair.first->second.data(), pair.first->second.size());
}
return name;

View file

@ -1,14 +1,8 @@
#include "SfxExtractor.h"
#include "soh/Enhancements/audio/AudioDecoder.h"
#include "soh/Enhancements/audio/miniaudio.h"
#include "soh/Enhancements/speechsynthesizer/SpeechSynthesizer.h"
#include "soh/Enhancements/tts/tts.h"
#include "dr_libs/dr_wav.h"
#define MA_NO_FLAC
#define MA_NO_MP3
#define MA_NO_THREADING
#define MA_NO_DEVICE_IO
#define MA_NO_GENERATION
#define MA_ENABLE_ONLY_SPECIFIC_BACKENDS
#include "miniaudio/miniaudio.h"
#include "soh/OTRGlobals.h"
#include "SfxTable.h"
#include <sstream>
@ -19,23 +13,7 @@ extern "C" {
void AudioMgr_CreateNextAudioBuffer(s16* samples, u32 num_samples);
extern bool freezeGame;
}
enum {
STEP_SETUP = 0,
STEP_MAIN,
STEP_FINISHED,
STEP_ERROR,
STEP_ERROR_OTR, // File exists.
} SFX_EXTRACTION_STEPS;
enum {
CT_WAITING, // for a sound to start ripping.
CT_PRIMING,
CT_READY, // to start ripping a sound.
CT_FINISHED, // ripping the current sound.
CT_SHUTDOWN,
} CAPTURE_THREAD_STATES;
#define SFX_EXTRACTION_BUFFER_SIZE 44100 * 15
#define SFX_EXTRACTION_ONE_FRAME 736
bool SfxExtractor::isAllZero(int16_t* buffer, size_t count) {
for (size_t i = 0; i < count; i++) {
if (buffer[i] != 0)
@ -71,34 +49,22 @@ bool SfxExtractor::renderOutput(size_t endOfInput) {
ma_channel_converter_config config =
ma_channel_converter_config_init(ma_format_s16, 2, NULL, 1, NULL, ma_channel_mix_mode_default);
ma_channel_converter converter;
if (ma_channel_converter_init(&config, NULL, &converter) != MA_SUCCESS)
throw std::runtime_error("SfxExtractor: Unable to initialize channel converter.");
drwav_data_format format;
format.bitsPerSample = 16;
format.channels = 1;
format.container = drwav_container_riff;
format.format = DR_WAVE_FORMAT_PCM;
format.sampleRate = 44100;
drwav wav;
if (ma_channel_converter_init(&config, NULL, &converter) != MA_SUCCESS) {
return false;
}
std::vector<uint8_t> fileData;
std::string fileName = getExternalFileName(currentSfx);
void* mem = NULL;
size_t size = 0;
if (!drwav_init_memory_write(&wav, &mem, &size, &format, NULL))
throw std::runtime_error("SfxExtractor: Unable to initialize wave writer.");
int16_t chunk[64];
int16_t* mark = tempBuffer + startOfInput;
size_t samplesLeft = endOfInput - startOfInput;
while (samplesLeft > 0) {
size_t thisChunk = std::min<size_t>(64, samplesLeft);
ma_channel_converter_process_pcm_frames(&converter, chunk, mark, thisChunk / 2);
drwav_write_pcm_frames(&wav, thisChunk / 2, chunk);
samplesLeft -= thisChunk;
mark += thisChunk;
while (mark < tempBuffer + endOfInput) {
size_t chunkSize = std::min<size_t>(64, ((tempBuffer + endOfInput) - mark) / 2);
ma_result converter_result = ma_channel_converter_process_pcm_frames(&converter, chunk, mark, chunkSize);
if (converter_result != MA_SUCCESS) {
return false;
}
fileData.insert(fileData.end(), (uint8_t*)chunk, (uint8_t*)(chunk + chunkSize));
mark += chunkSize * 2;
}
drwav_uninit(&wav);
std::vector<uint8_t> fileData((uint8_t*)mem, (uint8_t*)mem + size);
drwav_free(mem, nullptr);
return archive->WriteFile(fileName.c_str(), fileData);
}
@ -110,18 +76,15 @@ void SfxExtractor::setup() {
captureThreadState = CT_WAITING;
OTRAudio_InstallSfxCaptureThread();
// Make sure we're starting from a clean slate.
std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppDirectory("accessibility.o2r");
std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
if (std::filesystem::exists(sohAccessibilityPath)) {
currentStep = STEP_ERROR_OTR;
currentStep = STEP_ERROR_FILE_EXISTS;
return;
}
// Over-allocated just a tad because otherwise we'll overrun if the last frame is short.
tempStorage.resize((SFX_EXTRACTION_BUFFER_SIZE + (SFX_EXTRACTION_ONE_FRAME * 3)) * 2, 0);
tempBuffer = tempStorage.data();
sfxToRip = 0;
currentStep = STEP_MAIN;
archive = std::make_shared<Ship::O2rArchive>("accessibility.o2r");
archive = std::make_shared<Ship::O2rArchive>(sohAccessibilityPath);
archive->Open();
} catch (...) { currentStep = STEP_ERROR; }
}
@ -166,7 +129,7 @@ void SfxExtractor::finished() {
Audio_QueueSeqCmd(NA_BGM_TITLE);
if (currentStep == STEP_ERROR || currentStep == STEP_ERROR_OTR) {
if (currentStep >= STEP_ERROR) {
Audio_PlaySoundGeneral(NA_SE_SY_ERROR, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
&gSfxDefaultFreqAndVolScale, &gSfxDefaultReverb);
Audio_PlaySoundGeneral(NA_SE_EN_GANON_LAUGH, &gSfxDefaultPos, 4, &gSfxDefaultFreqAndVolScale,
@ -174,7 +137,7 @@ void SfxExtractor::finished() {
std::stringstream ss;
ss << "Sorry, we tried to extract the sound effects, but Ganondorf overruled us with an iron fist."
<< std::endl;
if (currentStep == STEP_ERROR_OTR)
if (currentStep == STEP_ERROR_FILE_EXISTS)
ss << "In all seriousness, please delete accessibility.o2r and try again.";
SpeechSynthesizer::Instance->Speak(ss.str().c_str(), "en-US");
} else
@ -209,8 +172,8 @@ void SfxExtractor::frameCallback() {
void SfxExtractor::prime() {
while (true) {
AudioMgr_CreateNextAudioBuffer(tempBuffer, SFX_EXTRACTION_ONE_FRAME);
if (isAllZero(tempBuffer, SFX_EXTRACTION_ONE_FRAME * 2))
AudioMgr_CreateNextAudioBuffer(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME);
if (isAllZero(tempBuffer + 0, SFX_EXTRACTION_ONE_FRAME * 2))
break;
}
captureThreadState = CT_FINISHED;
@ -222,10 +185,9 @@ void SfxExtractor::captureCallback() {
if (captureThreadState != CT_READY)
return; // No work to do at the moment.
memset(tempBuffer, 0, SFX_EXTRACTION_BUFFER_SIZE * 4);
int16_t* mark = tempBuffer;
int16_t* mark = tempBuffer + 0;
size_t samplesLeft = SFX_EXTRACTION_BUFFER_SIZE;
bool outputStarted = false;
size_t endOfInput = 0;
int waitTime = 0;
while (samplesLeft > 0) {
AudioMgr_CreateNextAudioBuffer(mark, SFX_EXTRACTION_ONE_FRAME);
@ -241,19 +203,19 @@ void SfxExtractor::captureCallback() {
}
outputStarted = true;
mark += (SFX_EXTRACTION_ONE_FRAME * 2);
endOfInput += (SFX_EXTRACTION_ONE_FRAME * 2);
samplesLeft -= std::min<size_t>(SFX_EXTRACTION_ONE_FRAME, samplesLeft);
size_t samples = std::min<size_t>(SFX_EXTRACTION_ONE_FRAME, samplesLeft);
mark += samples * 2;
samplesLeft -= samples;
}
if (renderOutput(endOfInput)) {
if (renderOutput(mark - tempBuffer)) {
captureThreadState = CT_FINISHED;
} else {
SPDLOG_ERROR("failed to write file to archive, trying again");
}
}
std::string SfxExtractor::getExternalFileName(int16_t sfxId) {
std::stringstream ss;
ss << "accessibility/audio/";
ss << std::hex << std::setw(4) << std::setfill('0') << sfxId << ".wav";
ss << "accessibility/audio/" << std::hex << std::setw(4) << std::setfill('0') << sfxId << ".wav";
return ss.str();
}

View file

@ -1,13 +1,33 @@
#pragma once
#include "libultraship/libultraship.h"
#define SFX_EXTRACTION_BUFFER_SIZE 44100 * 15
#define SFX_EXTRACTION_ONE_FRAME 736
enum CaptureThreadStates {
CT_WAITING, // for a sound to start ripping.
CT_PRIMING,
CT_READY, // to start ripping a sound.
CT_FINISHED, // ripping the current sound.
CT_SHUTDOWN,
};
enum SfxExtractionSteps {
STEP_SETUP = 0,
STEP_MAIN,
STEP_FINISHED,
STEP_ERROR,
STEP_ERROR_FILE_EXISTS,
};
class SfxExtractor {
std::shared_ptr<Ship::Archive> archive;
int currentStep;
int captureThreadState;
SfxExtractionSteps currentStep;
CaptureThreadStates captureThreadState;
int sfxToRip;
s16 currentSfx;
std::vector<int16_t> tempStorage; // Stores raw audio data for the sfx currently being ripped.
int16_t* tempBuffer; // Raw pointer to the above vector.
// Stores raw audio data for the sfx currently being ripped.
int16_t tempBuffer[(SFX_EXTRACTION_BUFFER_SIZE + SFX_EXTRACTION_ONE_FRAME * 3) * 2];
// Check if a buffer contains meaningful audio output.
bool isAllZero(int16_t* buffer, size_t count);
size_t adjustedStartOfInput();

View file

@ -1,7 +1,6 @@
#define MINIAUDIO_IMPLEMENTATION
#include "AudioDecoder.h"
#include "z64audio.h"
#define DR_WAV_IMPLEMENTATION
#include "dr_libs/dr_wav.h"
#include <stdexcept>
#define WAV_DECODE_CHUNK_SIZE 64
// A handful of definitions need to be copied from mixer.c.
@ -23,6 +22,7 @@ AudioDecoder::AudioDecoder() {
}
AudioDecoder::~AudioDecoder() {
}
void AudioDecoder::setSample(SOH::AudioSample* sample) {
this->sample.codec = sample->sample.codec;
this->sample.loop.start = sample->sample.loop->start;
@ -40,6 +40,7 @@ void AudioDecoder::setSample(SOH::AudioSample* sample) {
inStart = in;
inEnd = in + sample->sample.size;
}
void AudioDecoder::setSample(SoundFontSample* sample) {
this->sample.codec = sample->codec;
this->sample.loop.start = sample->loop->start;
@ -56,6 +57,7 @@ void AudioDecoder::setSample(SoundFontSample* sample) {
inStart = in;
inEnd = in + sample->size;
}
size_t AudioDecoder::decode(int16_t* out, size_t nSamples) {
size_t samplesOut = 0;
size_t nbytes = nSamples * 2;
@ -101,26 +103,39 @@ size_t AudioDecoder::decode(int16_t* out, size_t nSamples) {
return samplesOut;
}
size_t AudioDecoder::decodeToWav(int16_t** buffer) {
int16_t* wavOut = nullptr;
ma_result wavWrite(ma_encoder* pEncoder, const void* pBufferIn, size_t bytesToWrite, size_t* pBytesWritten) {
auto fileData = (std::vector<uint8_t>*)pEncoder->pUserData;
fileData->insert(fileData->end(), (uint8_t*)pBufferIn, (uint8_t*)pBufferIn + bytesToWrite);
if (pBytesWritten != NULL) {
*pBytesWritten = bytesToWrite;
}
return MA_SUCCESS;
}
drwav_data_format format;
format.bitsPerSample = 16;
format.channels = 1;
format.container = drwav_container_riff;
format.format = DR_WAVE_FORMAT_PCM;
ma_result wavSeek(ma_encoder* pEncoder, ma_int64 offset, ma_seek_origin origin) {
return MA_ERROR;
}
std::vector<uint8_t> AudioDecoder::decodeToWav() {
std::vector<uint8_t> fileData;
ma_uint32 sampleRate;
// Todo: figure out how to really determine the sample rate. CODEC_ADPCM tends to stream at higher rates (usually
// 20KHZ) while CODEC_SMALL_ADPCM is usually around 14000. They're still not consistent though.
if (sample.codec == CODEC_ADPCM)
format.sampleRate = 20000;
sampleRate = 20000;
else if (sample.codec = CODEC_SMALL_ADPCM)
format.sampleRate = 14000;
sampleRate = 14000;
else
throw std::runtime_error("AudioDecoder: Unsupported codec.");
drwav wav;
size_t wavSize;
if (!drwav_init_memory_write(&wav, (void**)&wavOut, &wavSize, &format, nullptr))
throw std::runtime_error("AudioDecoder: Unable to initialize wave writer.");
ma_encoder_config maconfig = ma_encoder_config_init(ma_encoding_format_wav, ma_format_s16, 1, sampleRate);
ma_encoder wavEncoder;
ma_result init_result = ma_encoder_init(wavWrite, wavSeek, &fileData, &maconfig, &wavEncoder);
if (init_result != MA_SUCCESS) {
return fileData;
}
int16_t chunk[WAV_DECODE_CHUNK_SIZE];
// Don't decode past the end of the loop.
size_t samplesLeft = sample.loop.end;
@ -135,14 +150,9 @@ size_t AudioDecoder::decodeToWav(int16_t** buffer) {
if (samplesRead == 0)
break;
if (drwav_write_pcm_frames(&wav, samplesRead, chunk) != samplesRead) {
drwav_uninit(&wav);
drwav_free(wavOut, nullptr);
throw std::runtime_error("AudioDecoder: Unable to write wave data.");
}
ma_encoder_write_pcm_frames(&wavEncoder, chunk, samplesRead, NULL);
samplesLeft -= samplesRead;
}
drwav_uninit(&wav);
*buffer = wavOut;
return wavSize;
ma_encoder_uninit(&wavEncoder);
return fileData;
}

View file

@ -2,6 +2,7 @@
// A standalone, incremental audio sample decoder.
// Based on the ADPCM decoding routines in mixer.c.
#include "miniaudio.h"
#include "libultraship/libultraship.h"
#include "soh/resource/type/AudioSample.h"
#include "z64audio.h"
@ -28,5 +29,5 @@ class AudioDecoder {
void setSample(SoundFontSample* sample);
size_t decode(int16_t* out, size_t nSamples);
size_t decodeToWav(int16_t** buffer);
std::vector<uint8_t> decodeToWav();
};

View file

@ -0,0 +1,9 @@
#pragma once
#define MA_NO_FLAC
#define MA_NO_MP3
#define MA_NO_THREADING
#define MA_NO_DEVICE_IO
#define MA_NO_GENERATION
#define MA_NO_STDIO
#define MA_ENABLE_ONLY_SPECIFIC_BACKENDS
#include "miniaudio/miniaudio.h"

View file

@ -294,7 +294,7 @@ OTRGlobals::OTRGlobals() {
}
}
std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppDirectory("accessibility.o2r");
std::string sohAccessibilityPath = Ship::Context::GetPathRelativeToAppBundle("accessibility.o2r");
if (std::filesystem::exists(sohAccessibilityPath)) {
OTRFiles.push_back(sohAccessibilityPath);
}