feat: Implement static linking, second attempt

This is mostly just the code from the previous attempt. All that was
missing were the `.lib` files to link to on Windows.
This commit is contained in:
Lucas Schwiderski 2023-03-10 23:10:15 +01:00
parent ca56e562ea
commit ba753cf6bb
Signed by: lucas
GPG key ID: AA12679AAA6DF4D8
20 changed files with 2106 additions and 490 deletions

5
.gitignore vendored
View file

@ -1,6 +1,7 @@
/target
/data
.envrc
liboo2corelinux64.so
oo2core_8_win64.dll
*.so
*.dll
*.lib
dictionary.csv

89
Cargo.lock generated
View file

@ -142,6 +142,28 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
[[package]]
name = "bindgen"
version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"log",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash",
"shlex",
"syn",
"which",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@ -269,6 +291,15 @@ dependencies = [
"jobserver",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom 7.1.3",
]
[[package]]
name = "cfg-expr"
version = "0.11.0"
@ -293,6 +324,17 @@ dependencies = [
"generic-array",
]
[[package]]
name = "clang-sys"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "clap"
version = "4.1.8"
@ -778,7 +820,7 @@ dependencies = [
"futures",
"lazy_static",
"nexusmods",
"oodle-sys",
"oodle",
"path-slash",
"sdk",
"serde",
@ -811,7 +853,7 @@ dependencies = [
"libloading",
"nanorand",
"notify",
"oodle-sys",
"oodle",
"path-clean",
"path-slash",
"pin-project-lite",
@ -854,6 +896,12 @@ dependencies = [
"wio",
]
[[package]]
name = "either"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "encoding_rs"
version = "0.8.32"
@ -1770,6 +1818,12 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.140"
@ -2160,11 +2214,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
[[package]]
name = "oodle-sys"
name = "oodle"
version = "0.1.0"
dependencies = [
"libloading",
"thiserror",
"bindgen",
"color-eyre",
"tracing",
]
@ -2325,6 +2379,12 @@ dependencies = [
"sha2",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "percent-encoding"
version = "2.2.0"
@ -2839,7 +2899,7 @@ dependencies = [
"libloading",
"luajit2-sys",
"nanorand",
"oodle-sys",
"oodle",
"path-slash",
"pin-project-lite",
"serde",
@ -2968,6 +3028,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
@ -3840,6 +3906,17 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb"
[[package]]
name = "which"
version = "4.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269"
dependencies = [
"either",
"libc",
"once_cell",
]
[[package]]
name = "winapi"
version = "0.3.9"

View file

@ -4,7 +4,7 @@ members = [
"crates/dtmt",
"crates/dtmm",
"lib/dtmt-shared",
"lib/oodle-sys",
"lib/oodle",
"lib/sdk",
"lib/serde_sjson",
"lib/steamlocate-rs",

View file

@ -13,7 +13,7 @@ confy = "0.5.1"
druid = { version = "0.8", features = ["im", "serde", "image", "png", "jpeg", "bmp", "webp", "svg"] }
dtmt-shared = { path = "../../lib/dtmt-shared", version = "*" }
futures = "0.3.25"
oodle-sys = { path = "../../lib/oodle-sys", version = "*" }
oodle = { path = "../../lib/oodle", version = "*" }
sdk = { path = "../../lib/sdk", version = "*" }
nexusmods = { path = "../../lib/nexusmods", version = "*" }
serde_sjson = { path = "../../lib/serde_sjson", version = "*" }

View file

@ -37,12 +37,6 @@ fn main() -> Result<()> {
tracing::trace!(default_config_path = %default_config_path.display());
let matches = command!()
.arg(Arg::new("oodle").long("oodle").help(
"The oodle library to load. This may either be:\n\
- A library name that will be searched for in the system's default paths.\n\
- A file path relative to the current working directory.\n\
- An absolute file path.",
))
.arg(
Arg::new("config")
.long("config")
@ -56,10 +50,6 @@ fn main() -> Result<()> {
let (log_tx, log_rx) = tokio::sync::mpsc::unbounded_channel();
util::log::create_tracing_subscriber(log_tx);
unsafe {
oodle_sys::init(matches.get_one::<String>("oodle"));
}
let (action_tx, action_rx) = tokio::sync::mpsc::unbounded_channel();
// let config = util::config::read_config(&default_config_path, &matches)

View file

@ -15,7 +15,7 @@ futures-util = "0.3.24"
glob = "0.3.0"
libloading = "0.7.4"
nanorand = "0.7.0"
oodle-sys = { path = "../../lib/oodle-sys", version = "*" }
oodle = { path = "../../lib/oodle", version = "*" }
pin-project-lite = "0.2.9"
promptly = "0.3.1"
sdk = { path = "../../lib/sdk", version = "*" }

View file

@ -33,12 +33,6 @@ pub(crate) fn command_definition() -> Command {
If omitted, dtmt will search from the current working directory upward.",
),
)
.arg(Arg::new("oodle").long("oodle").help(
"The oodle library to load. This may either be:\n\
- A library name that will be searched for in the system's default paths.\n\
- A file path relative to the current working directory.\n\
- An absolute file path.",
))
.arg(
Arg::new("out")
.long("out")
@ -391,10 +385,6 @@ where
#[tracing::instrument(skip_all)]
pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
unsafe {
oodle_sys::init(matches.get_one::<String>("oodle"));
}
let cfg = read_project_config(matches.get_one::<PathBuf>("directory").cloned()).await?;
let game_dir = matches

View file

@ -1,4 +1,4 @@
use clap::{Arg, ArgMatches, Command};
use clap::{ArgMatches, Command};
use color_eyre::eyre::Result;
mod decompress;
@ -10,12 +10,6 @@ pub(crate) fn command_definition() -> Command {
Command::new("bundle")
.subcommand_required(true)
.about("Manipulate the game's bundle files")
.arg(Arg::new("oodle").long("oodle").help(
"The oodle library to load. This may either be:\n\
- A library name that will be searched for in the system's default paths.\n\
- A file path relative to the current working directory.\n\
- An absolute file path.",
))
.subcommand(decompress::command_definition())
.subcommand(extract::command_definition())
.subcommand(inject::command_definition())
@ -24,10 +18,6 @@ pub(crate) fn command_definition() -> Command {
#[tracing::instrument(skip_all)]
pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
unsafe {
oodle_sys::init(matches.get_one::<String>("oodle"));
}
match matches.subcommand() {
Some(("decompress", sub_matches)) => decompress::run(ctx, sub_matches).await,
Some(("extract", sub_matches)) => extract::run(ctx, sub_matches).await,

View file

@ -34,12 +34,6 @@ pub(crate) fn command_definition() -> Command {
If omitted, the current working directory is used.",
),
)
.arg(Arg::new("oodle").long("oodle").help(
"The oodle library to load. This may either be:\n\
- A library name that will be searched for in the system's default paths.\n\
- A file path relative to the current working directory.\n\
- An absolute file path.",
))
.arg(
Arg::new("out")
.long("out")
@ -104,10 +98,6 @@ where
#[tracing::instrument(skip_all)]
pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
unsafe {
oodle_sys::init(matches.get_one::<String>("oodle"));
}
let cfg = read_project_config(matches.get_one::<PathBuf>("directory").cloned())
.await
.wrap_err("failed to load project config")?;

View file

@ -1,2 +0,0 @@
/target
/Cargo.lock

View file

@ -1,77 +0,0 @@
#![feature(c_size_t)]
#![feature(once_cell)]
use std::ffi::OsStr;
use std::sync::OnceLock;
mod library;
mod types;
pub use library::Library;
pub use library::CHUNK_SIZE;
pub use types::*;
#[derive(thiserror::Error, Debug)]
pub enum OodleError {
#[error("{0}")]
Oodle(String),
#[error(transparent)]
Library(#[from] libloading::Error),
}
type Result<T> = std::result::Result<T, OodleError>;
static LIB: OnceLock<Library> = OnceLock::new();
/// Initialize the global library handle that this module's
/// functions operate on.
///
/// # Safety
///
/// The safety concerns as described by [`libloading::Library::new`] apply.
pub unsafe fn init<P: AsRef<OsStr>>(name: Option<P>) {
let lib = match name {
Some(name) => Library::with_name(name),
None => Library::new(),
};
let lib = lib.expect("Failed to load library.");
if LIB.set(lib).is_err() {
panic!("Library was already initialized. Did you call `init` twice?");
}
}
fn get() -> Result<&'static Library> {
match LIB.get() {
Some(lib) => Ok(lib),
None => {
let err = OodleError::Oodle(String::from("Library has not been initialized, yet."));
Err(err)
}
}
}
pub fn decompress<I>(
data: I,
fuzz_safe: OodleLZ_FuzzSafe,
check_crc: OodleLZ_CheckCRC,
) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let lib = get()?;
lib.decompress(data, fuzz_safe, check_crc)
}
pub fn compress<I>(data: I) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let lib = get()?;
lib.compress(data)
}
pub fn get_decode_buffer_size(raw_size: usize, corruption_possible: bool) -> Result<usize> {
let lib = get()?;
lib.get_decode_buffer_size(raw_size, corruption_possible)
}

View file

@ -1,154 +0,0 @@
use std::{ffi::OsStr, ptr};
use libloading::Symbol;
use super::Result;
use crate::{types::*, OodleError};
// Hardcoded chunk size of Bitsquid's bundle compression
pub const CHUNK_SIZE: usize = 512 * 1024;
pub const COMPRESSOR: OodleLZ_Compressor = OodleLZ_Compressor::Kraken;
pub const LEVEL: OodleLZ_CompressionLevel = OodleLZ_CompressionLevel::Optimal2;
#[cfg(target_os = "windows")]
const OODLE_LIB_NAME: &str = "oo2core_8_win64";
#[cfg(target_os = "linux")]
const OODLE_LIB_NAME: &str = "liboo2corelinux64.so";
pub struct Library {
inner: libloading::Library,
}
impl Library {
/// Load the Oodle library by its default name.
///
/// The default name is platform-specific:
/// - Windows: `oo2core_8_win64`
/// - Linux: `liboo2corelinux64.so`
///
/// # Safety
///
/// The safety concerns as described by [`libloading::Library::new`] apply.
pub unsafe fn new() -> Result<Self> {
Self::with_name(OODLE_LIB_NAME)
}
/// Load the Oodle library by the given name or path.
///
/// See [`libloading::Library::new`] for how the `name` parameter is handled.
///
/// # Safety
///
/// The safety concerns as described by [`libloading::Library::new`] apply.
pub unsafe fn with_name<P: AsRef<OsStr>>(name: P) -> Result<Self> {
let inner = libloading::Library::new(name)?;
Ok(Self { inner })
}
#[tracing::instrument(skip(self, data))]
pub fn decompress<I>(
&self,
data: I,
fuzz_safe: OodleLZ_FuzzSafe,
check_crc: OodleLZ_CheckCRC,
) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let data = data.as_ref();
let mut out = vec![0; CHUNK_SIZE];
let verbosity = if tracing::enabled!(tracing::Level::INFO) {
OodleLZ_Verbosity::Minimal
} else if tracing::enabled!(tracing::Level::DEBUG) {
OodleLZ_Verbosity::Some
} else if tracing::enabled!(tracing::Level::TRACE) {
OodleLZ_Verbosity::Lots
} else {
OodleLZ_Verbosity::None
};
let ret = unsafe {
let decompress: Symbol<OodleLZ_Decompress> = self.inner.get(b"OodleLZ_Decompress\0")?;
decompress(
data.as_ptr() as *const _,
data.len(),
out.as_mut_ptr() as *mut _,
out.len(),
fuzz_safe,
check_crc,
verbosity,
ptr::null_mut(),
0,
ptr::null_mut(),
ptr::null_mut(),
ptr::null_mut(),
0,
OodleLZ_Decode_ThreadPhase::UNTHREADED,
)
};
if ret == 0 {
let err = OodleError::Oodle(String::from("Decompression failed."));
return Err(err);
}
Ok(out)
}
#[tracing::instrument(name = "Oodle::compress", skip(self, data))]
pub fn compress<I>(&self, data: I) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let mut raw = Vec::from(data.as_ref());
raw.resize(CHUNK_SIZE, 0);
// TODO: Query oodle for buffer size
let mut out = vec![0u8; CHUNK_SIZE];
let ret = unsafe {
let compress: Symbol<OodleLZ_Compress> = self.inner.get(b"OodleLZ_Compress\0")?;
compress(
COMPRESSOR,
raw.as_ptr() as *const _,
raw.len(),
out.as_mut_ptr() as *mut _,
LEVEL,
ptr::null_mut(),
0,
ptr::null_mut(),
ptr::null_mut(),
0,
)
};
tracing::debug!(compressed_size = ret, "Compressed chunk");
if ret == 0 {
let err = OodleError::Oodle(String::from("Compression failed."));
return Err(err);
}
out.resize(ret as usize, 0);
Ok(out)
}
pub fn get_decode_buffer_size(
&self,
raw_size: usize,
corruption_possible: bool,
) -> Result<usize> {
unsafe {
let f: Symbol<OodleLZ_GetDecodeBufferSize> =
self.inner.get(b"OodleLZ_GetDecodeBufferSize\0")?;
let size = f(COMPRESSOR, raw_size, corruption_possible);
Ok(size)
}
}
}

View file

@ -1,197 +0,0 @@
#![allow(dead_code)]
use core::ffi::{c_char, c_int, c_size_t, c_ulonglong, c_void};
// Type definitions taken from Unreal Engine's `oodle2.h`
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_FuzzSafe {
No = 0,
Yes = 1,
}
impl From<bool> for OodleLZ_FuzzSafe {
fn from(value: bool) -> Self {
if value {
Self::Yes
} else {
Self::No
}
}
}
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_CheckCRC {
No = 0,
Yes = 1,
Force32 = 0x40000000,
}
impl From<bool> for OodleLZ_CheckCRC {
fn from(value: bool) -> Self {
if value {
Self::Yes
} else {
Self::No
}
}
}
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_Verbosity {
None = 0,
Minimal = 1,
Some = 2,
Lots = 3,
Force32 = 0x40000000,
}
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_Decode_ThreadPhase {
Phase1 = 1,
Phase2 = 2,
PhaseAll = 3,
}
impl OodleLZ_Decode_ThreadPhase {
pub const UNTHREADED: Self = OodleLZ_Decode_ThreadPhase::PhaseAll;
}
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_Compressor {
Invalid = -1,
// None = memcpy, pass through uncompressed bytes
None = 3,
// NEW COMPRESSORS:
// Fast decompression and high compression ratios, amazing!
Kraken = 8,
// Leviathan = Kraken's big brother with higher compression, slightly slower decompression.
Leviathan = 13,
// Mermaid is between Kraken & Selkie - crazy fast, still decent compression.
Mermaid = 9,
// Selkie is a super-fast relative of Mermaid. For maximum decode speed.
Selkie = 11,
// Hydra, the many-headed beast = Leviathan, Kraken, Mermaid, or Selkie (see $OodleLZ_About_Hydra)
Hydra = 12,
BitKnit = 10,
// DEPRECATED but still supported
Lzb16 = 4,
Lzna = 7,
Lzh = 0,
Lzhlw = 1,
Lznib = 2,
Lzblw = 5,
Lza = 6,
Count = 14,
Force32 = 0x40000000,
}
#[repr(C)]
#[allow(non_camel_case_types)]
#[derive(Clone, Copy, Debug)]
pub enum OodleLZ_CompressionLevel {
// don't compress, just copy raw bytes
None = 0,
// super fast mode, lower compression ratio
SuperFast = 1,
// fastest LZ mode with still decent compression ratio
VeryFast = 2,
// fast - good for daily use
Fast = 3,
// standard medium speed LZ mode
Normal = 4,
// optimal parse level 1 (faster optimal encoder)
Optimal1 = 5,
// optimal parse level 2 (recommended baseline optimal encoder)
Optimal2 = 6,
// optimal parse level 3 (slower optimal encoder)
Optimal3 = 7,
// optimal parse level 4 (very slow optimal encoder)
Optimal4 = 8,
// optimal parse level 5 (don't care about encode speed, maximum compression)
Optimal5 = 9,
// faster than SuperFast, less compression
HyperFast1 = -1,
// faster than HyperFast1, less compression
HyperFast2 = -2,
// faster than HyperFast2, less compression
HyperFast3 = -3,
// fastest, less compression
HyperFast4 = -4,
Force32 = 0x40000000,
}
impl OodleLZ_CompressionLevel {
// alias hyperfast base level
pub const HYPERFAST: Self = OodleLZ_CompressionLevel::HyperFast1;
// alias optimal standard level
pub const OPTIMAL: Self = OodleLZ_CompressionLevel::Optimal2;
// maximum compression level
pub const MAX: Self = OodleLZ_CompressionLevel::Optimal5;
// fastest compression level
pub const MIN: Self = OodleLZ_CompressionLevel::HyperFast4;
pub const INVALID: Self = OodleLZ_CompressionLevel::Force32;
}
#[allow(non_camel_case_types)]
pub type t_fp_OodleCore_Plugin_Printf =
extern "C" fn(level: c_int, file: *const c_char, line: c_int, fmt: *const c_char);
#[allow(non_camel_case_types)]
pub type OodleLZ_Decompress = extern "C" fn(
compressed_buffer: *const c_void,
compressed_length: c_size_t,
raw_buffer: *mut c_void,
raw_length: c_size_t,
fuzz_safe: OodleLZ_FuzzSafe,
check_crc: OodleLZ_CheckCRC,
verbosity: OodleLZ_Verbosity,
decBufBase: *mut c_void,
decBufSize: c_size_t,
callback: *const c_void,
callback_user_data: *const c_void,
decoder_memory: *mut c_void,
decoder_memory_size: c_size_t,
thread_phase: OodleLZ_Decode_ThreadPhase,
) -> c_ulonglong;
#[allow(non_camel_case_types)]
pub type OodleLZ_Compress = extern "C" fn(
compressor: OodleLZ_Compressor,
raw_buffer: *const c_void,
raw_len: c_size_t,
compressed_buffer: *mut c_void,
level: OodleLZ_CompressionLevel,
options: *const c_void,
dictionary_base: c_size_t,
lrm: *const c_void,
scratch_memory: *mut c_void,
scratch_size: c_size_t,
) -> c_ulonglong;
#[allow(non_camel_case_types)]
pub type OodleLZ_GetDecodeBufferSize = extern "C" fn(
compressor: OodleLZ_Compressor,
raw_size: c_size_t,
corruption_possible: bool,
) -> c_size_t;
#[allow(non_camel_case_types)]
pub type OodleCore_Plugins_SetPrintf =
extern "C" fn(f: t_fp_OodleCore_Plugin_Printf) -> t_fp_OodleCore_Plugin_Printf;
#[allow(non_camel_case_types)]
pub type OodleCore_Plugin_Printf_Verbose = t_fp_OodleCore_Plugin_Printf;
#[allow(non_camel_case_types)]
pub type OodleCore_Plugin_Printf_Default = t_fp_OodleCore_Plugin_Printf;

View file

@ -1,11 +1,13 @@
[package]
name = "oodle-sys"
name = "oodle"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
libloading = "0.7.4"
thiserror = "1.0.38"
color-eyre = "0.6.2"
tracing = "0.1.37"
[build-dependencies]
bindgen = "0.64.0"

51
lib/oodle/build.rs Normal file
View file

@ -0,0 +1,51 @@
extern crate bindgen;
use std::env;
use std::path::PathBuf;
fn main() {
// Tell cargo to look for shared libraries in the specified directory
if let Ok(manifest_dir) = std::env::var("CARGO_MANIFEST_DIR") {
println!("cargo:rustc-link-search={}", manifest_dir);
dbg!(&manifest_dir);
}
let lib_name = if std::env::var("CARGO_CFG_WINDOWS").is_ok() {
if cfg!(debug_assertions) {
"oo2core_win64_debug"
} else {
"oo2core_win64"
}
} else {
"oo2corelinux64"
};
println!("cargo:rustc-link-lib={}", lib_name);
dbg!(&lib_name);
println!("cargo:rerun-if-changed=oodle2.h");
// The bindgen::Builder is the main entry point
// to bindgen, and lets you build up options for
// the resulting bindings.
let bindings = bindgen::Builder::default()
// The input header we would like to generate
// bindings for.
.header("oodle2base.h")
.header("oodle2.h")
.blocklist_file("stdint.h")
.blocklist_file("stdlib.h")
// Tell cargo to invalidate the built crate whenever any of the
// included header files changed.
.parse_callbacks(Box::new(bindgen::CargoCallbacks))
// Finish the builder and generate the bindings.
.generate()
// Unwrap the Result and panic on failure.
.expect("Unable to generate bindings");
// Write the bindings to the $OUT_DIR/bindings.rs file.
let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
bindings
.write_to_file(out_path.join("bindings.rs"))
.expect("Couldn't write bindings!");
}

1643
lib/oodle/oodle2.h Normal file

File diff suppressed because it is too large Load diff

167
lib/oodle/oodle2base.h Normal file
View file

@ -0,0 +1,167 @@
//===================================================
// Oodle2 Base header
// (C) Copyright 1994-2021 Epic Games Tools LLC
//===================================================
#ifndef __OODLE2BASE_H_INCLUDED__
#define __OODLE2BASE_H_INCLUDED__
#ifndef OODLE2BASE_PUBLIC_HEADER
#define OODLE2BASE_PUBLIC_HEADER 1
#endif
#ifdef _MSC_VER
#pragma pack(push, Oodle, 8)
#pragma warning(push)
#pragma warning(disable : 4127) // conditional is constant
#endif
#ifndef OODLE_BASE_TYPES_H
#define OODLE_BASE_TYPES_H
#include <stdint.h>
#define OOCOPYRIGHT "Copyright (C) 1994-2021, Epic Games Tools LLC"
// Typedefs
typedef int8_t OO_S8;
typedef uint8_t OO_U8;
typedef int16_t OO_S16;
typedef uint16_t OO_U16;
typedef int32_t OO_S32;
typedef uint32_t OO_U32;
typedef int64_t OO_S64;
typedef uint64_t OO_U64;
typedef float OO_F32;
typedef double OO_F64;
typedef intptr_t OO_SINTa;
typedef uintptr_t OO_UINTa;
typedef int32_t OO_BOOL;
// Struct packing handling and inlining
#if defined(__GNUC__) || defined(__clang__)
#define OOSTRUCT struct __attribute__((__packed__))
#define OOINLINEFUNC inline
#elif defined(_MSC_VER)
// on VC++, we use pragmas for the struct packing
#define OOSTRUCT struct
#define OOINLINEFUNC __inline
#endif
// Linkage stuff
#if defined(_WIN32)
#define OOLINK __stdcall
#define OOEXPLINK __stdcall
#else
#define OOLINK
#define OOEXPLINK
#endif
// C++ name demangaling
#ifdef __cplusplus
#define OODEFFUNC extern "C"
#define OODEFSTART extern "C" {
#define OODEFEND }
#define OODEFAULT( val ) =val
#else
#define OODEFFUNC
#define OODEFSTART
#define OODEFEND
#define OODEFAULT( val )
#endif
// ========================================================
// Exported function declarations
#define OOEXPFUNC OODEFFUNC
//===========================================================================
// OO_STRING_JOIN joins strings in the preprocessor and works with LINESTRING
#define OO_STRING_JOIN(arg1, arg2) OO_STRING_JOIN_DELAY(arg1, arg2)
#define OO_STRING_JOIN_DELAY(arg1, arg2) OO_STRING_JOIN_IMMEDIATE(arg1, arg2)
#define OO_STRING_JOIN_IMMEDIATE(arg1, arg2) arg1 ## arg2
//===========================================================================
// OO_NUMBERNAME is a macro to make a name unique, so that you can use it to declare
// variable names and they won't conflict with each other
// using __LINE__ is broken in MSVC with /ZI , but __COUNTER__ is an MSVC extension that works
#ifdef _MSC_VER
#define OO_NUMBERNAME(name) OO_STRING_JOIN(name,__COUNTER__)
#else
#define OO_NUMBERNAME(name) OO_STRING_JOIN(name,__LINE__)
#endif
//===================================================================
// simple compiler assert
// this happens at declaration time, so if it's inside a function in a C file, drop {} around it
#ifndef OO_COMPILER_ASSERT
#if defined(__clang__)
#define OO_COMPILER_ASSERT_UNUSED __attribute__((unused)) // hides warnings when compiler_asserts are in a local scope
#else
#define OO_COMPILER_ASSERT_UNUSED
#endif
#define OO_COMPILER_ASSERT(exp) typedef char OO_NUMBERNAME(_dummy_array) [ (exp) ? 1 : -1 ] OO_COMPILER_ASSERT_UNUSED
#endif
#endif
// Oodle2 base header
#ifndef OODLE2_PUBLIC_CORE_DEFINES
#define OODLE2_PUBLIC_CORE_DEFINES 1
#define OOFUNC1 OOEXPFUNC
#define OOFUNC2 OOEXPLINK
#define OOFUNCSTART
#define OODLE_CALLBACK OOLINK
// Check build flags
#if defined(OODLE_BUILDING_LIB) || defined(OODLE_BUILDING_DLL)
#error Should not see OODLE_BUILDING set for users of oodle.h
#endif
#ifndef NULL
#define NULL (0)
#endif
// OODLE_MALLOC_MINIMUM_ALIGNMENT is 8 in 32-bit, 16 in 64-bit
#define OODLE_MALLOC_MINIMUM_ALIGNMENT ((OO_SINTa)(2*sizeof(void *)))
typedef void (OODLE_CALLBACK t_OodleFPVoidVoid)(void);
/* void-void callback func pointer
takes void, returns void
*/
typedef void (OODLE_CALLBACK t_OodleFPVoidVoidStar)(void *);
/* void-void-star callback func pointer
takes void pointer, returns void
*/
#define OODLE_JOB_MAX_DEPENDENCIES (4) /* Maximum number of dependencies Oodle will ever pass to a RunJob callback
*/
#define OODLE_JOB_NULL_HANDLE (0) /* Value 0 of Jobify handles is reserved to mean none
* Wait(OODLE_JOB_NULL_HANDLE) is a nop
* if RunJob returns OODLE_JOB_NULL_HANDLE it means the job
* was run synchronously and no wait is required
*/
#define t_fp_Oodle_Job t_OodleFPVoidVoidStar /* Job function pointer for Plugin Jobify system
takes void pointer returns void
*/
#endif // OODLE2_PUBLIC_CORE_DEFINES
#ifdef _MSC_VER
#pragma warning(pop)
#pragma pack(pop, Oodle)
#endif
#endif // __OODLE2BASE_H_INCLUDED__

145
lib/oodle/src/lib.rs Normal file
View file

@ -0,0 +1,145 @@
#![allow(non_upper_case_globals)]
#![allow(non_camel_case_types)]
#![allow(non_snake_case)]
use std::ptr;
use color_eyre::{eyre, Result};
#[allow(dead_code)]
mod bindings {
include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
}
// Hardcoded chunk size of Bitsquid's bundle compression
pub const CHUNK_SIZE: usize = 512 * 1024;
pub const COMPRESSOR: bindings::OodleLZ_Compressor =
bindings::OodleLZ_Compressor_OodleLZ_Compressor_Kraken;
pub const LEVEL: bindings::OodleLZ_CompressionLevel =
bindings::OodleLZ_CompressionLevel_OodleLZ_CompressionLevel_Optimal2;
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum OodleLZ_FuzzSafe {
Yes,
No,
}
impl From<OodleLZ_FuzzSafe> for bindings::OodleLZ_FuzzSafe {
fn from(value: OodleLZ_FuzzSafe) -> Self {
match value {
OodleLZ_FuzzSafe::Yes => bindings::OodleLZ_FuzzSafe_OodleLZ_FuzzSafe_Yes,
OodleLZ_FuzzSafe::No => bindings::OodleLZ_FuzzSafe_OodleLZ_FuzzSafe_No,
}
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum OodleLZ_CheckCRC {
Yes,
No,
}
impl From<OodleLZ_CheckCRC> for bindings::OodleLZ_CheckCRC {
fn from(value: OodleLZ_CheckCRC) -> Self {
match value {
OodleLZ_CheckCRC::Yes => bindings::OodleLZ_CheckCRC_OodleLZ_CheckCRC_Yes,
OodleLZ_CheckCRC::No => bindings::OodleLZ_CheckCRC_OodleLZ_CheckCRC_No,
}
}
}
#[tracing::instrument(skip(data))]
pub fn decompress<I>(
data: I,
fuzz_safe: OodleLZ_FuzzSafe,
check_crc: OodleLZ_CheckCRC,
) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let data = data.as_ref();
let mut out = vec![0; CHUNK_SIZE];
let verbosity = if tracing::enabled!(tracing::Level::INFO) {
bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_Minimal
} else if tracing::enabled!(tracing::Level::DEBUG) {
bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_Some
} else if tracing::enabled!(tracing::Level::TRACE) {
bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_Lots
} else {
bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_None
};
let ret = unsafe {
bindings::OodleLZ_Decompress(
data.as_ptr() as *const _,
data.len() as isize,
out.as_mut_ptr() as *mut _,
out.len() as isize,
fuzz_safe.into(),
check_crc.into(),
verbosity,
ptr::null_mut(),
0,
None,
ptr::null_mut(),
ptr::null_mut(),
0,
bindings::OodleLZ_Decode_ThreadPhase_OodleLZ_Decode_Unthreaded,
)
};
if ret == 0 {
eyre::bail!("Decompression failed");
}
Ok(out)
}
#[tracing::instrument(skip(data))]
pub fn compress<I>(data: I) -> Result<Vec<u8>>
where
I: AsRef<[u8]>,
{
let mut raw = Vec::from(data.as_ref());
raw.resize(CHUNK_SIZE, 0);
// TODO: Query oodle for buffer size
let mut out = vec![0u8; CHUNK_SIZE];
let ret = unsafe {
bindings::OodleLZ_Compress(
COMPRESSOR,
raw.as_ptr() as *const _,
raw.len() as isize,
out.as_mut_ptr() as *mut _,
LEVEL,
ptr::null_mut(),
ptr::null_mut(),
ptr::null_mut(),
ptr::null_mut(),
0,
)
};
tracing::debug!(compressed_size = ret, "Compressed chunk");
if ret == 0 {
eyre::bail!("Compression failed");
}
out.resize(ret as usize, 0);
Ok(out)
}
pub fn get_decode_buffer_size(raw_size: usize, corruption_possible: bool) -> Result<usize> {
let size = unsafe {
bindings::OodleLZ_GetDecodeBufferSize(
COMPRESSOR,
raw_size as isize,
if corruption_possible { 1 } else { 0 },
)
};
Ok(size as usize)
}

View file

@ -17,7 +17,7 @@ nanorand = "0.7.0"
pin-project-lite = "0.2.9"
serde = { version = "1.0.147", features = ["derive"] }
serde_sjson = { path = "../../lib/serde_sjson", version = "*" }
oodle-sys = { path = "../../lib/oodle-sys", version = "*" }
oodle = { path = "../../lib/oodle", version = "*" }
tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] }
tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] }
tracing = { version = "0.1.37", features = ["async-await"] }

View file

@ -4,7 +4,7 @@ use std::path::Path;
use color_eyre::eyre::{self, Context, Result};
use color_eyre::{Help, Report, SectionExt};
use oodle_sys::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE};
use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE};
use crate::binary::sync::*;
use crate::bundle::file::Properties;
@ -159,7 +159,7 @@ impl Bundle {
decompressed.append(&mut compressed_buffer);
} else {
// TODO: Optimize to not reallocate?
let mut raw_buffer = oodle_sys::decompress(
let mut raw_buffer = oodle::decompress(
&compressed_buffer,
OodleLZ_FuzzSafe::No,
OodleLZ_CheckCRC::No,
@ -257,7 +257,7 @@ impl Bundle {
let mut chunk_sizes = Vec::with_capacity(num_chunks);
for chunk in chunks {
let compressed = oodle_sys::compress(chunk)?;
let compressed = oodle::compress(chunk)?;
tracing::trace!(
raw_chunk_size = chunk.len(),
compressed_chunk_size = compressed.len()
@ -359,7 +359,7 @@ where
r.read_exact(&mut compressed_buffer)?;
// TODO: Optimize to not reallocate?
let mut raw_buffer = oodle_sys::decompress(
let mut raw_buffer = oodle::decompress(
&compressed_buffer,
OodleLZ_FuzzSafe::No,
OodleLZ_CheckCRC::No,