From 999fa2af615ef4e1558788a2120be751f9236a37 Mon Sep 17 00:00:00 2001 From: "Skyth (Asilkan)" <19259897+blueskythlikesclouds@users.noreply.github.com> Date: Wed, 26 Feb 2025 13:49:36 +0300 Subject: [PATCH] Implement shader archive decompressor for the build system. (#466) * Implement shader archive decompressor for the build system. * Fix Linux compilation error. --- UnleashedRecompLib/CMakeLists.txt | 11 +- .../io.github.hedge_dev.unleashedrecomp.json | 5 + tools/CMakeLists.txt | 1 + tools/x_decompress/CMakeLists.txt | 11 + tools/x_decompress/x_decompress.cpp | 247 ++++++++++++++++++ 5 files changed, 274 insertions(+), 1 deletion(-) create mode 100644 tools/x_decompress/CMakeLists.txt create mode 100644 tools/x_decompress/x_decompress.cpp diff --git a/UnleashedRecompLib/CMakeLists.txt b/UnleashedRecompLib/CMakeLists.txt index 6c33d0a0..53c3b2c2 100644 --- a/UnleashedRecompLib/CMakeLists.txt +++ b/UnleashedRecompLib/CMakeLists.txt @@ -37,6 +37,15 @@ add_custom_command( "${CMAKE_CURRENT_SOURCE_DIR}/config/SWA.toml" ) +add_custom_command( + OUTPUT + "${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar" + COMMAND + $ "${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar" "${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar" +) + set(XENOS_RECOMP_ROOT "${UNLEASHED_RECOMP_TOOLS_ROOT}/XenosRecomp/XenosRecomp") set(XENOS_RECOMP_INCLUDE "${XENOS_RECOMP_ROOT}/shader_common.h") @@ -58,7 +67,7 @@ add_custom_command( $ DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/private/default_patched.xex" - "${CMAKE_CURRENT_SOURCE_DIR}/private/shader.ar" + "${CMAKE_CURRENT_SOURCE_DIR}/private/shader_decompressed.ar" ${XENOS_RECOMP_SOURCES} ${XENOS_RECOMP_INCLUDE} ) diff --git a/flatpak/io.github.hedge_dev.unleashedrecomp.json b/flatpak/io.github.hedge_dev.unleashedrecomp.json index d4b3b3fe..a61c952a 100644 --- a/flatpak/io.github.hedge_dev.unleashedrecomp.json +++ b/flatpak/io.github.hedge_dev.unleashedrecomp.json @@ -52,6 +52,11 @@ "type": "file", "path": "private/shader.ar", "dest": "UnleashedRecompLib/private" + }, + { + "type": "file", + "path": "private/shader_decompressed.ar", + "dest": "UnleashedRecompLib/private" } ], "build-options": { diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 33b579e9..bdcf384a 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,5 +1,6 @@ add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/bc_diff) add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/file_to_c) add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/fshasher) +add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/x_decompress) add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp) add_subdirectory(${UNLEASHED_RECOMP_TOOLS_ROOT}/XenosRecomp) diff --git a/tools/x_decompress/CMakeLists.txt b/tools/x_decompress/CMakeLists.txt new file mode 100644 index 00000000..aecf1f79 --- /dev/null +++ b/tools/x_decompress/CMakeLists.txt @@ -0,0 +1,11 @@ +project("x_decompress") +set(CMAKE_CXX_STANDARD 17) + +add_executable(x_decompress + "x_decompress.cpp" + "${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp/thirdparty/libmspack/libmspack/mspack/lzxd.c" +) + +target_include_directories(x_decompress + PRIVATE "${UNLEASHED_RECOMP_TOOLS_ROOT}/XenonRecomp/thirdparty/libmspack/libmspack/mspack" +) diff --git a/tools/x_decompress/x_decompress.cpp b/tools/x_decompress/x_decompress.cpp new file mode 100644 index 00000000..211ec48a --- /dev/null +++ b/tools/x_decompress/x_decompress.cpp @@ -0,0 +1,247 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::vector readAllBytes(const char* path) +{ + std::ifstream file{ path, std::ios::binary }; + std::vector result{}; + + if (!file.good()) { + return result; + } + + file.seekg(0, std::ios::end); + result.resize(file.tellg()); + + file.seekg(0, std::ios::beg); + file.read(reinterpret_cast(result.data()), result.size()); + + return result; +} + +template +static T byteSwap(T value) +{ + if constexpr (sizeof(T) == 1) + return value; + else if constexpr (sizeof(T) == 2) + return static_cast(__builtin_bswap16(static_cast(value))); + else if constexpr (sizeof(T) == 4) + return static_cast(__builtin_bswap32(static_cast(value))); + else if constexpr (sizeof(T) == 8) + return static_cast(__builtin_bswap64(static_cast(value))); + + assert(false && "Unexpected byte size."); + return value; +} + +template +static void byteSwapInplace(T& value) +{ + value = byteSwap(value); +} + +struct ReadStream +{ + const uint8_t* data = nullptr; + int size = 0; // Size from every compressed block. +}; + +static int mspackRead(mspack_file* file, void* buffer, int bytes) +{ + ReadStream* stream = reinterpret_cast(file); + + if (stream->size == 0) + { + uint16_t size = byteSwap(*reinterpret_cast(stream->data)); + stream->data += sizeof(uint16_t); + + // This indicates there is an uncompressed block size available. We don't need it so we skip it. + if ((size & 0xFF00) == 0xFF00) + { + stream->data += 1; + size = byteSwap(*reinterpret_cast(stream->data)); + stream->data += sizeof(uint16_t); + } + + stream->size = size; + } + + int sizeToRead = std::min(stream->size, bytes); + + memcpy(buffer, stream->data, sizeToRead); + stream->data += sizeToRead; + stream->size -= sizeToRead; + + return sizeToRead; +} + +struct WriteStream +{ + uint8_t* data = nullptr; + std::size_t size = 0; // Remaining available space in the stream. +}; + +static int mspackWrite(mspack_file* file, void* buffer, int bytes) +{ + WriteStream* stream = reinterpret_cast(file); + + std::size_t sizeToWrite = std::min(stream->size, static_cast(bytes)); + + memcpy(stream->data, buffer, sizeToWrite); + stream->data += sizeToWrite; + stream->size -= sizeToWrite; + + return static_cast(sizeToWrite); +} + +static void* mspackAlloc(mspack_system* self, size_t bytes) +{ + return operator new(bytes); +} + +static void mspackFree(void* ptr) +{ + operator delete(ptr); +} + +static void mspackCopy(void* src, void* dst, size_t bytes) +{ + memcpy(dst, src, bytes); +} + +static mspack_system g_lzxSystem = +{ + nullptr, + nullptr, + mspackRead, + mspackWrite, + nullptr, + nullptr, + nullptr, + mspackAlloc, + mspackFree, + mspackCopy +}; + +// Xbox Compression header definitions. +static constexpr uint32_t XCompressSignature = 0xFF512EE; + +struct XCompressHeader +{ + uint32_t signature; + uint32_t field04; + uint32_t field08; + uint32_t field0C; + uint32_t windowSize; + uint32_t compressedBlockSize; + uint64_t uncompressedSize; + uint64_t compressedSize; + uint32_t uncompressedBlockSize; + uint32_t field2C; + + void byteSwap() + { + byteSwapInplace(signature); + byteSwapInplace(field04); + byteSwapInplace(field08); + byteSwapInplace(field0C); + byteSwapInplace(windowSize); + byteSwapInplace(compressedBlockSize); + byteSwapInplace(uncompressedSize); + byteSwapInplace(compressedSize); + byteSwapInplace(uncompressedBlockSize); + byteSwapInplace(field2C); + } +}; + +int main(int argc, char** argv) +{ + if (argc < 3) + { + printf("Usage: x_decompress [input file path] [output file path]"); + return EXIT_SUCCESS; + } + + std::vector file = readAllBytes(argv[1]); + if (file.empty()) + { + fprintf(stderr, "Input file \"%s\" not found or empty", argv[1]); + return EXIT_FAILURE; + } + + std::vector decompressedFile; + + if (file.size() >= sizeof(XCompressHeader) && byteSwap(*reinterpret_cast(file.data())) == XCompressSignature) + { + XCompressHeader* header = reinterpret_cast(file.data()); + header->byteSwap(); + + decompressedFile.resize(header->uncompressedSize); + + const uint8_t* srcBytes = file.data() + sizeof(XCompressHeader); + + WriteStream dstStream; + dstStream.data = decompressedFile.data(); + dstStream.size = decompressedFile.size(); + + // libmspack wants the bit index. This value is always guaranteed to be a power of two, + // so we can extract the bit index by counting the amount of leading zeroes. + int windowBits = 0; + uint32_t windowSize = header->windowSize; + while ((windowSize & 0x1) == 0) + { + ++windowBits; + windowSize >>= 1; + } + + // Loop over compressed blocks. + while (srcBytes < (file.data() + file.size()) && dstStream.data < (decompressedFile.data() + decompressedFile.size())) + { + uint32_t compressedSize = byteSwap(*reinterpret_cast(srcBytes)); + srcBytes += sizeof(uint32_t); + + ReadStream srcStream; + srcStream.data = srcBytes; + + std::size_t uncompressedBlockSize = std::min(static_cast(header->uncompressedBlockSize), dstStream.size); + + lzxd_stream* lzx = lzxd_init( + &g_lzxSystem, + reinterpret_cast(&srcStream), + reinterpret_cast(&dstStream), + windowBits, + 0, + static_cast(header->compressedBlockSize), + static_cast(uncompressedBlockSize), + 0); + + lzxd_decompress(lzx, uncompressedBlockSize); + lzxd_free(lzx); + + srcBytes += compressedSize; + } + } + else + { + decompressedFile = std::move(file); + } + + std::ofstream outputFile(argv[2], std::ios::binary); + if (!outputFile.good()) + { + fprintf(stderr, "Cannot open output file \"%s\" for writing", argv[2]); + return EXIT_FAILURE; + } + + outputFile.write(reinterpret_cast(decompressedFile.data()), decompressedFile.size()); + + return EXIT_SUCCESS; +}