From 04979ce11d7a36b0efa0e5736ceafffde98dee70 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 9 Nov 2022 09:13:06 +0100 Subject: [PATCH] feat: Implement compress operation --- oodle-cli.cpp | 442 ++++++++++++++++++++++++++++++++++------- oodle-cli.vcxproj | 8 +- oodle-cli.vcxproj.user | 2 +- 3 files changed, 378 insertions(+), 74 deletions(-) diff --git a/oodle-cli.cpp b/oodle-cli.cpp index a563c36..51196e0 100644 --- a/oodle-cli.cpp +++ b/oodle-cli.cpp @@ -1,6 +1,3 @@ -// oodle-cli.cpp : This file contains the 'main' function. Program execution begins and ends there. -// - #include #include #include @@ -11,6 +8,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include @@ -22,24 +24,42 @@ const LPCWSTR LIB_NAME = L"oo2core_8_win64"; // It's not like this is going to change often. const char* LIB_FILE = "oo2core_8_win64.dll"; +typedef decltype(OodleLZ_Compress)* compress_func; typedef decltype(OodleLZ_Decompress)* decompress_func; typedef decltype(OodleCore_Plugins_SetPrintf)* setprintf_func; +typedef std::uint8_t u8; +typedef std::uint32_t u32; +typedef std::uint64_t u64; + +const u32 CHUNK_RAW_SIZE = 512 * 1024; + void usage() { std::cerr << - "Usage: oodle-cli [OPTIONS] [--] \n" - "Decompress a to using the Oodle algorithm.\n" - "The size of the uncompressed data must be known.\n" + "Usage: oodle-cli [OPTIONS] [--] \n" + "Perform Oodle 2 compression or decompression for Darktide data.\n" + "Deflated data is a stream of chunks with 512k uncompressed size.\n" + "Inflated data is an arbitrary byte stream.\n" + "Data is read from stdin and written to stdout\n" + "\n" + " may be either 'compress' or 'decompress'.\n" "\n" "Options:\n" - " -v Increase Oodle's verbosity. May be specified up to three times.\n" - " -c Only check if the library can be found and used.\n" - " --fuzz-safe Use fuzz safe decompression.\n" - " --check-crc Check CRC during decompression.\n" + " -v Increase Oodle's verbosity.\n" + " May be specified up to three times.\n" + " -c Only check if the library can be found and used.\n" + " --fuzz-safe Use fuzz safe decompression.\n" + " --check-crc Check CRC during decompression.\n" + " --padding The amount of padding to assume at the start of the data.\n" + " Set this to `(x % 16)`, where `x` is the position at which\n" + " the data passed to oodle - cli starts in the source file.\n" + " --help Show this help message.\n" "\n" "The following environmental variables are recognized:\n" - " DLL_SEARCH_PATH: A color (':') separated list of directories to search for the Oodle library. May be relative to the current working directory.\n"; + " DLL_SEARCH_PATH: A color (':') separated list of directories to search\n" + " for the Oodle library. May be relative to the current\n" + " working directory.\n"; } @@ -61,7 +81,288 @@ void printf_callback(int verboseLevel, const char* file, int line, const char* f va_list args; va_start(args, fmt); - vfprintf(stdout, fmt, args); + vfprintf(stderr, fmt, args); +} + + +u32 read_u32(FILE* stream) { + u32 buf; + if (fread_s(&buf, 4, sizeof(u8), 4, stream) < 4) { + throw "ERROR: Failed to read u32 from stream!"; + } + + return _byteswap_ulong(buf); +} + + +size_t write_u32(FILE* stream, u32 val) { + val = _byteswap_ulong(val); + + if (fwrite(&val, 4, sizeof(u8), stream) < 4) { + throw "ERROR: Failed to read u32 from stream!"; + } + + return 4; +} + +size_t write_padding(FILE* stream, size_t offset) { + size_t pos = ftell(stream) + offset; + if (errno != 0) { + return 0; + } + + size_t padding_size = 16 - ((pos) % 16); + if (padding_size < 16) { + u64 padding = 0x0; + + _set_errno(0); + fwrite(&padding, sizeof(u8), padding_size, stream); + + return padding_size; + } + + return 0; +} + + +int do_compress(HMODULE hDLL, FILE* in_file, FILE* out_file, size_t padding_start) { + auto fn = reinterpret_cast(GetProcAddress(hDLL, "OodleLZ_Compress")); + if (fn == NULL) { + std::cerr << "ERROR: The library is incompatible, no 'OodleLZ_Compress'!\n"; + return 1; + } + + // I'm not sure how Fatshark does this efficiently. + // Within each chunk, there is a padding whos size depends on the position where the padding is written. + // And that position is affected by both the size of previous chunks and the total number of chunks, since + // list of chunk sizes is written before all chunks. + // Because of that, all chunks have to be compressed and buffered before any of them can be written to the output. + std::vector chunk_sizes; + std::vector final_buffer(CHUNK_RAW_SIZE); + + u8* write_address = final_buffer.data(); + + // This tracks the total amount of data written to `final_buffer`. + // It also doubles as the current position in the output stream, required + // to calculate padding sizes. + size_t final_size = 0; + + // Re-use the same buffers for each chunk. + // Allocating the full CHUNK_RAW_SIZE for compressed buffer is pretty + // much guranteed to be too much, but only slightly, and more performant than + // re-allocating anyways. + u8* raw_buffer = new u8[CHUNK_RAW_SIZE]; + + size_t chunk_index = 0; + boolean is_eof = false; + + do { + chunk_index++; + + _set_errno(0); + size_t read = fread_s(raw_buffer, CHUNK_RAW_SIZE, sizeof(u8), CHUNK_RAW_SIZE, in_file); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to read chunk " << chunk_index << " from input file : " << msg << std::endl; + return 1; + } + + if (read < CHUNK_RAW_SIZE) { + is_eof = true; + } + + size_t remaining = CHUNK_RAW_SIZE - read; + + // Bitsquid always writes full chunks. If the last chunk can't be filled completely + // with data the remainder is filled with `0x0`. + memset(raw_buffer + read, 0x0, remaining); + + // Bitsquid also adds an end marker consisting of four `0x66`. + if (remaining >= 4) { + memset(raw_buffer + read, 0x66, 4); + } + else if (CHUNK_RAW_SIZE - read >= 0) { + std::cerr << "ERROR: Not enough space left in chunk to add end marker. Don't know how to proceed.\n"; + return 1; + } + + intptr_t res = fn( + OodleLZ_Compressor_Kraken, + raw_buffer, + CHUNK_RAW_SIZE, + write_address, + OodleLZ_CompressionLevel_Optimal2, + nullptr, + 0, + nullptr, + nullptr, + 0 + ); + + if (res <= 0) { + std::cerr << "ERROR: Failed to compress chunk " << chunk_index << "!\n"; + return 1; + } + + chunk_sizes.push_back(res); + write_address += res; + } while (!is_eof); + + _set_errno(0); + write_u32(out_file, chunk_sizes.size()); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to write number of chunks to output file: " << msg << std::endl; + return 1; + } + + // `padding_start` is the point in the final file, passed from CLI. + // The `4` is added for the u32 at the start of our output, which is the + // number of chunks. + size_t pos = ftell(out_file) + 4; + _set_errno(0); + write_padding(out_file, pos); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to write chunk size padding to output file: " << msg << std::endl; + return 1; + } + + chunk_index = 0; + u8* read_address = final_buffer.data(); + + for (auto size : chunk_sizes) { + chunk_index++; + + _set_errno(0); + write_u32(out_file, size); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to write size for chunk " << chunk_index << " to output file : " << msg << std::endl; + return 1; + } + + size_t pos = ftell(out_file) + padding_start; + _set_errno(0); + write_padding(out_file, pos); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to write chunk size padding to output file: " << msg << std::endl; + return 1; + } + + _set_errno(0); + fwrite(&read_address, sizeof(u8), size, out_file); + if (errno != 0) { + char msg[80]; + _strerror_s(msg, 80, NULL); + std::cerr << "ERROR: Failed to write data for chunk " << chunk_index << " to output file : " << msg << std::endl; + return 1; + } + + read_address += size; + } + + + delete[] raw_buffer; + + return 0; +} + + +int do_decompress( + HMODULE hDLL, + FILE* in_file, + FILE* out_file, + OodleLZ_FuzzSafe fuzz_safe, + OodleLZ_CheckCRC check_crc, + OodleLZ_Verbosity verbosity, + size_t padding_start, + size_t num_chunks +) { + auto fn = reinterpret_cast(GetProcAddress(hDLL, "OodleLZ_Decompress")); + if (fn == NULL) { + std::cerr << "ERROR: The library is incompatible, no 'OodleLZ_Decompress'!\n"; + return 1; + } + + // Re-use the same buffers for each chunk. + // Allocating the full CHUNK_RAW_SIZE for the compressed buffer is pretty + // much guranteed to be too much, but should be more performant than the + // exact size allocating each time. And who cares about <512k RAM wasted? + u8* raw_buffer = new u8[CHUNK_RAW_SIZE]; + u8* compressed_buffer = new u8[CHUNK_RAW_SIZE]; + if (verbosity >= OodleLZ_Verbosity_Some) { + std::cerr << "DEBUG: " << num_chunks << " chunks\n"; + } + + for (size_t chunk_index = 0; chunk_index < num_chunks; chunk_index++) { + u32 chunk_size = read_u32(in_file); + + _set_errno(0); + size_t pos = ftell(in_file) + padding_start; + if (errno != 0) { + std::cerr << "ERROR: Failed to get position in input file: " << strerror(errno) << std::endl; + return 1; + } + + size_t padding_size = 16 - (pos % 16); + if (padding_size < 16) { + _set_errno(0); + if (fseek(in_file, (long)padding_size, SEEK_CUR) != 0) { + std::cerr << "ERROR: Failed to seek input file for chunk " << chunk_index << ": " << strerror(errno) << std::endl; + return 1; + } + } + + if (verbosity >= OodleLZ_Verbosity_Some) { + std::cerr << "DEBUG: Chunk " << chunk_index + 1 << ": " << chunk_size << "\n"; + } + + _set_errno(0); + if (fread_s(compressed_buffer, chunk_size, sizeof(u8), chunk_size, in_file) < chunk_size) { + std::cerr << "ERROR: Failed to read compressed data for chunk " << chunk_index << " from input file: " << strerror(errno) << "\n"; + return 1; + } + + intptr_t res = fn( + compressed_buffer, + chunk_size, + raw_buffer, + CHUNK_RAW_SIZE, + fuzz_safe, + check_crc, + verbosity, + nullptr, + 0, + nullptr, + nullptr, + nullptr, + 0, + OodleLZ_Decode_Unthreaded + ); + + if (res != CHUNK_RAW_SIZE) { + std::cerr << "ERROR: Failed to decompress chunk " << chunk_index << "!\n"; + return 1; + } + + _set_errno(0); + if (fwrite(raw_buffer, sizeof(u8), CHUNK_RAW_SIZE, out_file) < CHUNK_RAW_SIZE) { + std::cerr << "ERROR: Failed to write decompressed chunk " << chunk_index << " to output file" << strerror(errno) << "\n"; + return 1; + } + } + + delete[] raw_buffer; + delete[] compressed_buffer; + + return 0; } @@ -71,6 +372,8 @@ int main(int argc, char* argv[]) bool check_lib = FALSE; OodleLZ_FuzzSafe fuzz_safe = OodleLZ_FuzzSafe_No; OodleLZ_CheckCRC check_crc = OodleLZ_CheckCRC_No; + size_t padding_start = 0; + size_t num_chunks = 0; int i = 1; for (; i < argc; i++) { @@ -96,6 +399,14 @@ int main(int argc, char* argv[]) else if (strcmp(arg, "--check-crc") == 0) { check_crc = OodleLZ_CheckCRC_Yes; } + else if (strcmp(arg, "--padding") == 0) { + i++; + padding_start = std::stoi(argv[i]); + } + else if (strcmp(arg, "--chunks") == 0) { + i++; + num_chunks = std::stoi(argv[i]); + } else { std::cerr << "ERROR: Unknown option '" << arg << "'!\n\n"; usage(); @@ -140,91 +451,84 @@ int main(int argc, char* argv[]) return 1; } - auto decompress = reinterpret_cast(GetProcAddress((HMODULE)hDLL, "OodleLZ_Decompress")); - if (decompress == NULL) { - std::cerr << "ERROR: The library is incompatible!\n"; - return 1; - } - if (check_lib) { std::cout << "INFO: '" << LIB_FILE << "' found and loaded.\n"; return 0; } - if (argc - i < 3) { + if (num_chunks == 0) { + std::cerr << "ERROR: Number of chunks not specified!\n"; + usage(); + return 1; + } + + if (argc - i < 1) { std::cerr << "ERROR: Arguments missing!\n\n"; usage(); return 1; } - std::string in_name = argv[i]; - std::string out_name = argv[i + 1]; - size_t raw_buffer_size = std::stoi(argv[i + 2]); - - std::cout << "INFO: Attempting to decompress from '" << in_name << "' to '" << out_name << "' (" << raw_buffer_size << " bytes).\n"; - if (verbosity > 0) { auto fn = reinterpret_cast(GetProcAddress((HMODULE)hDLL, "OodleCore_Plugins_SetPrintf")); if (fn == NULL) { - std::cerr << "ERROR: The library is incompatible!\n"; + std::cerr << "ERROR: The library is incompatible, no 'OodleCore_Plugins_SetPrintf'!\n"; return 1; } reinterpret_cast(fn(printf_callback)); } - std::ifstream in_file(in_name, std::ios::binary | std::ios::ate); - if (!in_file) { - std::cerr << "ERROR: Failed to open compressed file!\n"; - return 1; + std::string operation = argv[i]; + FILE* in_file = nullptr; + FILE* out_file = nullptr; + + if (argc - i >= 2) { + errno_t err = fopen_s(&in_file, argv[i + 1], "rbS"); + + if (err != 0) { + std::cerr << "ERROR: Failed to open input file: " << strerror(errno) << "\n"; + return 1; + } + } + else { + if (_setmode(_fileno(stdin), _O_BINARY) < 0) { + std::cerr << "ERROR: Failed to prepare stdin for binary reading!\n"; + return 1; + } + in_file = stdin; } - std::streamsize compressed_buffer_size = in_file.tellg(); - in_file.seekg(0, std::ios::beg); + if (argc - i >= 3) { + errno_t err = fopen_s(&out_file, argv[i + 2], "wbS"); - std::vector compressed_buffer(compressed_buffer_size); - in_file.read(compressed_buffer.data(), compressed_buffer_size); - - if (in_file.fail()) { - std::cerr << "ERROR: Failed to read compressed file!\n"; - return 1; + if (err != 0) { + std::cerr << "ERROR: Failed to open output file: " << strerror(errno) << "\n"; + return 1; + } + } + else { + if (_setmode(_fileno(stdout), _O_BINARY) < 0) { + std::cerr << "ERROR: Failed to prepare stdout for binary writing!\n"; + return 1; + } + out_file = stdout; } - std::vector raw_buffer(raw_buffer_size); + int res; - intptr_t res = decompress( - compressed_buffer.data(), - compressed_buffer.size(), - raw_buffer.data(), - raw_buffer_size, - fuzz_safe, - check_crc, - (OodleLZ_Verbosity)verbosity, - nullptr, - 0, - nullptr, - nullptr, - nullptr, - 0, - OodleLZ_Decode_Unthreaded - ); - - if (res != raw_buffer_size) { - std::cerr << "ERROR: Failed to decompress!\n"; - return 1; + if (operation == "compress") { + res = do_compress(hDLL, in_file, out_file, padding_start); + } + else { + res = do_decompress(hDLL, in_file, out_file, fuzz_safe, check_crc, (OodleLZ_Verbosity)verbosity, padding_start, num_chunks); } - std::ofstream out_file(out_name, std::ios::binary); - if (!out_file) { - std::cerr << "ERROR: Failed to open output file!\n"; - return 1; + if (in_file != stdin) { + fclose(in_file); } - out_file.write(raw_buffer.data(), res); - - if (out_file.fail()) { - std::cerr << "ERROR: Failed to write output file!\n"; - return 1; + if (out_file != stdout) { + fclose(out_file); } - std::cout << "INFO: Done!\n"; + return res; } diff --git a/oodle-cli.vcxproj b/oodle-cli.vcxproj index 78a8e60..5ad73f3 100644 --- a/oodle-cli.vcxproj +++ b/oodle-cli.vcxproj @@ -74,7 +74,7 @@ Level3 true - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true @@ -88,7 +88,7 @@ true true true - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true @@ -102,7 +102,7 @@ Level3 true - _DEBUG;_CONSOLE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 @@ -117,7 +117,7 @@ true true true - NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + _CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) true stdcpp17 diff --git a/oodle-cli.vcxproj.user b/oodle-cli.vcxproj.user index 87e5899..7b8be40 100644 --- a/oodle-cli.vcxproj.user +++ b/oodle-cli.vcxproj.user @@ -1,7 +1,7 @@  - -v -v -v test\compressed.data 100 + compress .\test\compress_test.in.bin .\test\compress_test.out.bin WindowsLocalDebugger \ No newline at end of file