feat: Implement compress operation

oodle-cli.cpp

@@ -1,6 +1,3 @@
-// oodle-cli.cpp : This file contains the 'main' function. Program execution begins and ends there.
-//
-
 #include <iostream>
 #include <fstream>
 #include <string>
@@ -11,6 +8,11 @@
 #include <codecvt>
 #include <filesystem>
 #include <stdlib.h>
+#include <io.h>
+#include <fcntl.h>
+#include <share.h>
+#include <sys\stat.h>
+#include <string.h>
 
 #include <windows.h>
 #include <libloaderapi.h>
@@ -22,24 +24,42 @@ const LPCWSTR LIB_NAME = L"oo2core_8_win64";
 // It's not like this is going to change often.
 const char* LIB_FILE = "oo2core_8_win64.dll";
 
+typedef decltype(OodleLZ_Compress)* compress_func;
 typedef decltype(OodleLZ_Decompress)* decompress_func;
 typedef decltype(OodleCore_Plugins_SetPrintf)* setprintf_func;
 
+typedef std::uint8_t u8;
+typedef std::uint32_t u32;
+typedef std::uint64_t u64;
+
+const u32 CHUNK_RAW_SIZE = 512 * 1024;
+
 
 void usage() {
     std::cerr <<
-        "Usage: oodle-cli [OPTIONS] [--] <in_file> <out_file> <uncompressed_size>\n"
+        "Usage: oodle-cli [OPTIONS] [--] <operation>\n"
-        "Decompress a <in_file> to <out_file> using the Oodle algorithm.\n"
+        "Perform Oodle 2 compression or decompression for Darktide data.\n"
-        "The size of the uncompressed data must be known.\n"
+        "Deflated data is a stream of chunks with 512k uncompressed size.\n"
+        "Inflated data is an arbitrary byte stream.\n"
+        "Data is read from stdin and written to stdout\n"
+        "\n"
+        "<operation> may be either 'compress' or 'decompress'.\n"
         "\n"
         "Options:\n"
-        "  -v  Increase Oodle's verbosity. May be specified up to three times.\n"
+        "  -v            Increase Oodle's verbosity.\n"
+        "                May be specified up to three times.\n"
         "  -c            Only check if the library can be found and used.\n"
         "  --fuzz-safe   Use fuzz safe decompression.\n"
         "  --check-crc   Check CRC during decompression.\n"
+        "  --padding     The amount of padding to assume at the start of the data.\n"
+        "                Set this to `(x % 16)`, where `x` is the position at which\n"
+        "                the data passed to oodle - cli starts in the source file.\n"
+        "  --help        Show this help message.\n"
         "\n"
         "The following environmental variables are recognized:\n"
-        "  DLL_SEARCH_PATH: A color (':') separated list of directories to search for the Oodle library. May be relative to the current working directory.\n";
+        "  DLL_SEARCH_PATH: A color (':') separated list of directories to search\n"
+        "                   for the Oodle library. May be relative to the current\n"
+        "                   working directory.\n";
 }
 
 
@@ -61,7 +81,288 @@ void printf_callback(int verboseLevel, const char* file, int line, const char* f
 
     va_list args;
     va_start(args, fmt);
-    vfprintf(stdout, fmt, args);
+    vfprintf(stderr, fmt, args);
+}
+
+
+u32 read_u32(FILE* stream) {
+    u32 buf;
+    if (fread_s(&buf, 4, sizeof(u8), 4, stream) < 4) {
+        throw "ERROR: Failed to read u32 from stream!";
+    }
+
+    return _byteswap_ulong(buf);
+}
+
+
+size_t write_u32(FILE* stream, u32 val) {
+    val = _byteswap_ulong(val);
+
+    if (fwrite(&val, 4, sizeof(u8), stream) < 4) {
+        throw "ERROR: Failed to read u32 from stream!";
+    }
+
+    return 4;
+}
+
+size_t write_padding(FILE* stream, size_t offset) {
+    size_t pos = ftell(stream) + offset;
+    if (errno != 0) {
+        return 0;
+    }
+
+    size_t padding_size = 16 - ((pos) % 16);
+    if (padding_size < 16) {
+        u64 padding = 0x0;
+
+        _set_errno(0);
+        fwrite(&padding, sizeof(u8), padding_size, stream);
+
+        return padding_size;
+    }
+
+    return 0;
+}
+
+
+int do_compress(HMODULE hDLL, FILE* in_file, FILE* out_file, size_t padding_start) {
+    auto fn = reinterpret_cast<compress_func>(GetProcAddress(hDLL, "OodleLZ_Compress"));
+    if (fn == NULL) {
+        std::cerr << "ERROR: The library is incompatible, no 'OodleLZ_Compress'!\n";
+        return 1;
+    }
+
+    // I'm not sure how Fatshark does this efficiently.
+    // Within each chunk, there is a padding whos size depends on the position where the padding is written.
+    // And that position is affected by both the size of previous chunks and the total number of chunks, since
+    // list of chunk sizes is written before all chunks.
+    // Because of that, all chunks have to be compressed and buffered before any of them can be written to the output.
+    std::vector<u32> chunk_sizes;
+    std::vector<u8> final_buffer(CHUNK_RAW_SIZE);
+
+    u8* write_address = final_buffer.data();
+
+    // This tracks the total amount of data written to `final_buffer`.
+    // It also doubles as the current position in the output stream, required
+    // to calculate padding sizes.
+    size_t final_size = 0;
+
+    // Re-use the same buffers for each chunk.
+    // Allocating the full CHUNK_RAW_SIZE for compressed buffer is pretty
+    // much guranteed to be too much, but only slightly, and more performant than
+    // re-allocating anyways.
+    u8* raw_buffer = new u8[CHUNK_RAW_SIZE];
+
+    size_t chunk_index = 0;
+    boolean is_eof = false;
+
+    do {
+        chunk_index++;
+
+        _set_errno(0);
+        size_t read = fread_s(raw_buffer, CHUNK_RAW_SIZE, sizeof(u8), CHUNK_RAW_SIZE, in_file);
+        if (errno != 0) {
+            char msg[80];
+            _strerror_s(msg, 80, NULL);
+            std::cerr << "ERROR: Failed to read chunk " << chunk_index << " from input file : " << msg << std::endl;
+            return 1;
+        }
+
+        if (read < CHUNK_RAW_SIZE) {
+            is_eof = true;
+        }
+
+        size_t remaining = CHUNK_RAW_SIZE - read;
+
+        // Bitsquid always writes full chunks. If the last chunk can't be filled completely
+        // with data the remainder is filled with `0x0`.
+        memset(raw_buffer + read, 0x0, remaining);
+
+        // Bitsquid also adds an end marker consisting of four `0x66`.
+        if (remaining >= 4) {
+            memset(raw_buffer + read, 0x66, 4);
+        }
+        else if (CHUNK_RAW_SIZE - read >= 0) {
+            std::cerr << "ERROR: Not enough space left in chunk to add end marker. Don't know how to proceed.\n";
+            return 1;
+        }
+
+        intptr_t res = fn(
+            OodleLZ_Compressor_Kraken,
+            raw_buffer,
+            CHUNK_RAW_SIZE,
+            write_address,
+            OodleLZ_CompressionLevel_Optimal2,
+            nullptr,
+            0,
+            nullptr,
+            nullptr,
+            0
+        );
+
+        if (res <= 0) {
+            std::cerr << "ERROR: Failed to compress chunk " << chunk_index << "!\n";
+            return 1;
+        }
+
+        chunk_sizes.push_back(res);
+        write_address += res;
+    } while (!is_eof);
+
+    _set_errno(0);
+    write_u32(out_file, chunk_sizes.size());
+    if (errno != 0) {
+        char msg[80];
+        _strerror_s(msg, 80, NULL);
+        std::cerr << "ERROR: Failed to write number of chunks to output file: " << msg << std::endl;
+        return 1;
+    }
+
+    // `padding_start` is the point in the final file, passed from CLI.
+    // The `4` is added for the u32 at the start of our output, which is the
+    // number of chunks.
+    size_t pos = ftell(out_file) + 4;
+    _set_errno(0);
+    write_padding(out_file, pos);
+    if (errno != 0) {
+        char msg[80];
+        _strerror_s(msg, 80, NULL);
+        std::cerr << "ERROR: Failed to write chunk size padding to output file: " << msg << std::endl;
+        return 1;
+    }
+
+    chunk_index = 0;
+    u8* read_address = final_buffer.data();
+
+    for (auto size : chunk_sizes) {
+        chunk_index++;
+
+        _set_errno(0);
+        write_u32(out_file, size);
+        if (errno != 0) {
+            char msg[80];
+            _strerror_s(msg, 80, NULL);
+            std::cerr << "ERROR: Failed to write size for chunk " << chunk_index << " to output file : " << msg << std::endl;
+            return 1;
+        }
+
+        size_t pos = ftell(out_file) + padding_start;
+        _set_errno(0);
+        write_padding(out_file, pos);
+        if (errno != 0) {
+            char msg[80];
+            _strerror_s(msg, 80, NULL);
+            std::cerr << "ERROR: Failed to write chunk size padding to output file: " << msg << std::endl;
+            return 1;
+        }
+
+        _set_errno(0);
+        fwrite(&read_address, sizeof(u8), size, out_file);
+        if (errno != 0) {
+            char msg[80];
+            _strerror_s(msg, 80, NULL);
+            std::cerr << "ERROR: Failed to write data for chunk " << chunk_index << " to output file : " << msg << std::endl;
+            return 1;
+        }
+
+        read_address += size;
+    }
+
+
+    delete[] raw_buffer;
+
+    return 0;
+}
+
+
+int do_decompress(
+    HMODULE hDLL,
+    FILE* in_file,
+    FILE* out_file,
+    OodleLZ_FuzzSafe fuzz_safe,
+    OodleLZ_CheckCRC check_crc,
+    OodleLZ_Verbosity verbosity,
+    size_t padding_start,
+    size_t num_chunks
+) {
+    auto fn = reinterpret_cast<decompress_func>(GetProcAddress(hDLL, "OodleLZ_Decompress"));
+    if (fn == NULL) {
+        std::cerr << "ERROR: The library is incompatible, no 'OodleLZ_Decompress'!\n";
+        return 1;
+    }
+
+    // Re-use the same buffers for each chunk.
+    // Allocating the full CHUNK_RAW_SIZE for the compressed buffer is pretty
+    // much guranteed to be too much, but should be more performant than the
+    // exact size allocating each time. And who cares about <512k RAM wasted?
+    u8* raw_buffer = new u8[CHUNK_RAW_SIZE];
+    u8* compressed_buffer = new u8[CHUNK_RAW_SIZE];
+    if (verbosity >= OodleLZ_Verbosity_Some) {
+        std::cerr << "DEBUG: " << num_chunks << " chunks\n";
+    }
+
+    for (size_t chunk_index = 0; chunk_index < num_chunks; chunk_index++) {
+        u32 chunk_size = read_u32(in_file);
+
+        _set_errno(0);
+        size_t pos = ftell(in_file) + padding_start;
+        if (errno != 0) {
+            std::cerr << "ERROR: Failed to get position in input file: " << strerror(errno) << std::endl;
+            return 1;
+        }
+
+        size_t padding_size = 16 - (pos % 16);
+        if (padding_size < 16) {
+            _set_errno(0);
+            if (fseek(in_file, (long)padding_size, SEEK_CUR) != 0) {
+                std::cerr << "ERROR: Failed to seek input file for chunk " << chunk_index <<  ": " << strerror(errno) << std::endl;
+                return 1;
+            }
+        }
+
+        if (verbosity >= OodleLZ_Verbosity_Some) {
+            std::cerr << "DEBUG: Chunk " << chunk_index + 1 << ": " << chunk_size << "\n";
+        }
+
+        _set_errno(0);
+        if (fread_s(compressed_buffer, chunk_size, sizeof(u8), chunk_size, in_file) < chunk_size) {
+            std::cerr << "ERROR: Failed to read compressed data for chunk " << chunk_index << " from input file: " << strerror(errno) << "\n";
+            return 1;
+        }
+
+        intptr_t res = fn(
+            compressed_buffer,
+            chunk_size,
+            raw_buffer,
+            CHUNK_RAW_SIZE,
+            fuzz_safe,
+            check_crc,
+            verbosity,
+            nullptr,
+            0,
+            nullptr,
+            nullptr,
+            nullptr,
+            0,
+            OodleLZ_Decode_Unthreaded
+        );
+        
+        if (res != CHUNK_RAW_SIZE) {
+            std::cerr << "ERROR: Failed to decompress chunk " << chunk_index << "!\n";
+            return 1;
+        }
+
+        _set_errno(0);
+        if (fwrite(raw_buffer, sizeof(u8), CHUNK_RAW_SIZE, out_file) < CHUNK_RAW_SIZE) {
+            std::cerr << "ERROR: Failed to write decompressed chunk " << chunk_index << " to output file" << strerror(errno) << "\n";
+            return 1;
+        }
+    }
+
+    delete[] raw_buffer;
+    delete[] compressed_buffer;
+
+    return 0;
 }
 
 
@@ -71,6 +372,8 @@ int main(int argc, char* argv[])
     bool check_lib = FALSE;
     OodleLZ_FuzzSafe fuzz_safe = OodleLZ_FuzzSafe_No;
     OodleLZ_CheckCRC check_crc = OodleLZ_CheckCRC_No;
+    size_t padding_start = 0;
+    size_t num_chunks = 0;
     int i = 1;
 
     for (; i < argc; i++) {
@@ -96,6 +399,14 @@ int main(int argc, char* argv[])
         else if (strcmp(arg, "--check-crc") == 0) {
             check_crc = OodleLZ_CheckCRC_Yes;
         }
+        else if (strcmp(arg, "--padding") == 0) {
+            i++;
+            padding_start = std::stoi(argv[i]);
+        }
+        else if (strcmp(arg, "--chunks") == 0) {
+            i++;
+            num_chunks = std::stoi(argv[i]);
+        }
         else {
             std::cerr << "ERROR: Unknown option '" << arg << "'!\n\n";
             usage();
@@ -140,91 +451,84 @@ int main(int argc, char* argv[])
         return 1;
     }
 
-    auto decompress = reinterpret_cast<decompress_func>(GetProcAddress((HMODULE)hDLL, "OodleLZ_Decompress"));
-    if (decompress == NULL) {
-        std::cerr << "ERROR: The library is incompatible!\n";
-        return 1;
-    }
-
     if (check_lib) {
         std::cout << "INFO: '" << LIB_FILE << "' found and loaded.\n";
         return 0;
     }
 
-    if (argc - i < 3) {
+    if (num_chunks == 0) {
+        std::cerr << "ERROR: Number of chunks not specified!\n";
+        usage();
+        return 1;
+    }
+
+    if (argc - i < 1) {
         std::cerr << "ERROR: Arguments missing!\n\n";
         usage();
         return 1;
     }
 
-    std::string in_name = argv[i];
-    std::string out_name = argv[i + 1];
-    size_t raw_buffer_size = std::stoi(argv[i + 2]);
-
-    std::cout << "INFO: Attempting to decompress from '" << in_name << "' to '" << out_name << "' (" << raw_buffer_size << " bytes).\n";
-
     if (verbosity > 0) {
         auto fn = reinterpret_cast<setprintf_func>(GetProcAddress((HMODULE)hDLL, "OodleCore_Plugins_SetPrintf"));
         if (fn == NULL) {
-            std::cerr << "ERROR: The library is incompatible!\n";
+            std::cerr << "ERROR: The library is incompatible, no 'OodleCore_Plugins_SetPrintf'!\n";
             return 1;
         }
         reinterpret_cast<void*>(fn(printf_callback));
     }
 
-    std::ifstream in_file(in_name, std::ios::binary | std::ios::ate);
+    std::string operation = argv[i];
-    if (!in_file) {
+    FILE* in_file = nullptr;
-        std::cerr << "ERROR: Failed to open compressed file!\n";
+    FILE* out_file = nullptr;
+
+    if (argc - i >= 2) {
+        errno_t err = fopen_s(&in_file, argv[i + 1], "rbS");
+
+        if (err != 0) {
+            std::cerr << "ERROR: Failed to open input file: " << strerror(errno) << "\n";
             return 1;
         }
-
+    }
-    std::streamsize compressed_buffer_size = in_file.tellg();
+    else {
-    in_file.seekg(0, std::ios::beg);
+        if (_setmode(_fileno(stdin), _O_BINARY) < 0) {
-
+            std::cerr << "ERROR: Failed to prepare stdin for binary reading!\n";
-    std::vector<char> compressed_buffer(compressed_buffer_size);
-    in_file.read(compressed_buffer.data(), compressed_buffer_size);
-
-    if (in_file.fail()) {
-        std::cerr << "ERROR: Failed to read compressed file!\n";
             return 1;
         }
+        in_file = stdin;
+    }
 
-    std::vector<char> raw_buffer(raw_buffer_size);
+    if (argc - i >= 3) {
+        errno_t err = fopen_s(&out_file, argv[i + 2], "wbS");
 
-    intptr_t res = decompress(
+        if (err != 0) {
-        compressed_buffer.data(),
+            std::cerr << "ERROR: Failed to open output file: " << strerror(errno) << "\n";
-        compressed_buffer.size(),
-        raw_buffer.data(),
-        raw_buffer_size,
-        fuzz_safe,
-        check_crc,
-        (OodleLZ_Verbosity)verbosity,
-        nullptr,
-        0,
-        nullptr,
-        nullptr,
-        nullptr,
-        0,
-        OodleLZ_Decode_Unthreaded
-    );
-
-    if (res != raw_buffer_size) {
-        std::cerr << "ERROR: Failed to decompress!\n";
             return 1;
         }
-
+    }
-    std::ofstream out_file(out_name, std::ios::binary);
+    else {
-    if (!out_file) {
+        if (_setmode(_fileno(stdout), _O_BINARY) < 0) {
-        std::cerr << "ERROR: Failed to open output file!\n";
+            std::cerr << "ERROR: Failed to prepare stdout for binary writing!\n";
             return 1;
         }
-
+        out_file = stdout;
-    out_file.write(raw_buffer.data(), res);
-
-    if (out_file.fail()) {
-        std::cerr << "ERROR: Failed to write output file!\n";
-        return 1;
     }
 
-    std::cout << "INFO: Done!\n";
+    int res;
+
+    if (operation == "compress") {
+        res = do_compress(hDLL, in_file, out_file, padding_start);
+    }
+    else {
+        res = do_decompress(hDLL, in_file, out_file, fuzz_safe, check_crc, (OodleLZ_Verbosity)verbosity, padding_start, num_chunks);
+    }
+
+    if (in_file != stdin) {
+        fclose(in_file);
+    }
+
+    if (out_file != stdout) {
+        fclose(out_file);
+    }
+
+    return res;
 }

oodle-cli.vcxproj

@@ -74,7 +74,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
     </ClCompile>
     <Link>
@@ -88,7 +88,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
     </ClCompile>
     <Link>
@@ -102,7 +102,7 @@
     <ClCompile>
       <WarningLevel>Level3</WarningLevel>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_DEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>
@@ -117,7 +117,7 @@
       <FunctionLevelLinking>true</FunctionLevelLinking>
       <IntrinsicFunctions>true</IntrinsicFunctions>
       <SDLCheck>true</SDLCheck>
-      <PreprocessorDefinitions>NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;NDEBUG;_CONSOLE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <ConformanceMode>true</ConformanceMode>
       <LanguageStandard>stdcpp17</LanguageStandard>
     </ClCompile>

oodle-cli.vcxproj.user

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
-    <LocalDebuggerCommandArguments> -v -v -v test\compressed.data 100</LocalDebuggerCommandArguments>
+    <LocalDebuggerCommandArguments>compress .\test\compress_test.in.bin .\test\compress_test.out.bin</LocalDebuggerCommandArguments>
     <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
   </PropertyGroup>
 </Project>