From 3cb66878554c2d7601ad005e2b2a8b4edb103a1b Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 3 Mar 2023 16:55:22 +0100 Subject: [PATCH 01/35] feat(sdk): Implement partial texture decompilation --- crates/dtmt/src/cmd/bundle/extract.rs | 28 ++ crates/dtmt/src/cmd/dictionary.rs | 11 +- crates/dtmt/src/cmd/experiment/mod.rs | 21 + .../dtmt/src/cmd/experiment/texture_meta.rs | 117 ++++++ crates/dtmt/src/main.rs | 9 +- lib/oodle/src/lib.rs | 3 +- lib/sdk/src/binary.rs | 26 +- lib/sdk/src/bundle/file.rs | 67 ++- lib/sdk/src/bundle/filetype.rs | 10 + lib/sdk/src/bundle/mod.rs | 2 + lib/sdk/src/context.rs | 9 +- lib/sdk/src/filetype/mod.rs | 1 + lib/sdk/src/filetype/texture.rs | 387 ++++++++++++++++++ lib/sdk/src/murmur/dictionary.rs | 12 +- 14 files changed, 667 insertions(+), 36 deletions(-) create mode 100644 crates/dtmt/src/cmd/experiment/mod.rs create mode 100644 crates/dtmt/src/cmd/experiment/texture_meta.rs create mode 100644 lib/sdk/src/filetype/texture.rs diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index 75f1360..b595dba 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -287,6 +287,34 @@ where P1: AsRef + std::fmt::Debug, P2: AsRef + std::fmt::Debug, { + let ctx = if ctx.game_dir.is_some() { + tracing::debug!( + "Got game directory from config: {}", + ctx.game_dir.as_ref().unwrap().display() + ); + + ctx + } else { + let game_dir = path + .as_ref() + .parent() + .and_then(|parent| parent.parent()) + .map(|p| p.to_path_buf()); + + tracing::info!( + "No game directory configured, guessing from bundle path: {:?}", + game_dir + ); + + Arc::new(sdk::Context { + game_dir, + lookup: Arc::clone(&ctx.lookup), + ljd: ctx.ljd.clone(), + revorb: ctx.revorb.clone(), + ww2ogg: ctx.ww2ogg.clone(), + }) + }; + let bundle = { let data = fs::read(path.as_ref()).await?; let name = Bundle::get_name_from_path(&ctx, path.as_ref()); diff --git a/crates/dtmt/src/cmd/dictionary.rs b/crates/dtmt/src/cmd/dictionary.rs index 4c54c34..8f0d32c 100644 --- a/crates/dtmt/src/cmd/dictionary.rs +++ b/crates/dtmt/src/cmd/dictionary.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum}; use cli_table::{print_stdout, WithTitle}; @@ -156,6 +157,8 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( BufReader::new(Box::new(f)) }; + let lookup = Arc::make_mut(&mut ctx.lookup); + let group = sdk::murmur::HashGroup::from(*group); let mut added = 0; @@ -165,15 +168,15 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( let total = { for line in lines.into_iter() { let value = line?; - if ctx.lookup.find(&value, group).is_some() { + if lookup.find(&value, group).is_some() { skipped += 1; } else { - ctx.lookup.add(value, group); + lookup.add(value, group); added += 1; } } - ctx.lookup.len() + lookup.len() }; let out_path = matches @@ -190,7 +193,7 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( }) .with_section(|| out_path.display().to_string().header("Path:"))?; - ctx.lookup + lookup .to_csv(f) .await .wrap_err("Failed to write dictionary to disk")?; diff --git a/crates/dtmt/src/cmd/experiment/mod.rs b/crates/dtmt/src/cmd/experiment/mod.rs new file mode 100644 index 0000000..50ba706 --- /dev/null +++ b/crates/dtmt/src/cmd/experiment/mod.rs @@ -0,0 +1,21 @@ +use clap::{ArgMatches, Command}; +use color_eyre::Result; + +mod texture_meta; + +pub(crate) fn command_definition() -> Command { + Command::new("experiment") + .subcommand_required(true) + .about("A collection of utilities and experiments.") + .subcommand(texture_meta::command_definition()) +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + match matches.subcommand() { + Some(("texture-meta", sub_matches)) => texture_meta::run(ctx, sub_matches).await, + _ => unreachable!( + "clap is configured to require a subcommand, and they're all handled above" + ), + } +} diff --git a/crates/dtmt/src/cmd/experiment/texture_meta.rs b/crates/dtmt/src/cmd/experiment/texture_meta.rs new file mode 100644 index 0000000..98d0035 --- /dev/null +++ b/crates/dtmt/src/cmd/experiment/texture_meta.rs @@ -0,0 +1,117 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::Context; +use color_eyre::Result; +use futures_util::StreamExt; +use sdk::Bundle; +use tokio::fs; + +use crate::cmd::util::resolve_bundle_paths; + +pub(crate) fn command_definition() -> Command { + Command::new("texture-meta") + .about( + "Iterates over the provided bundles and lists certain meta data. + Primarily intended to help spot patterns between dependend data fields and values.", + ) + .arg( + Arg::new("bundle") + .required(true) + .action(ArgAction::Append) + .value_parser(value_parser!(PathBuf)) + .help( + "Path to the bundle(s) to read. If this points to a directory instead \ + of a file, all files in that directory will be checked.", + ), + ) + // TODO: Maybe provide JSON and CSV + // TODO: Maybe allow toggling certain fields +} + +#[tracing::instrument(skip(ctx))] +async fn handle_bundle(ctx: &sdk::Context, path: &PathBuf) -> Result<()> { + let bundle = { + let binary = fs::read(path).await?; + let name = Bundle::get_name_from_path(ctx, path); + Bundle::from_binary(ctx, name, binary)? + }; + + let bundle_dir = ctx + .game_dir + .as_deref() + .map(|dir| dir.join("bundle")) + .or_else(|| path.parent().map(|p| p.to_path_buf())) + .unwrap_or_default(); + + for f in bundle.files().iter() { + for (i, v) in f.variants().iter().enumerate() { + let data_file_name = v.data_file_name(); + + let data_file_length = if let Some(file_name) = data_file_name { + let path = bundle_dir.join(file_name); + + match fs::metadata(&path).await { + Ok(meta) => meta.len(), + Err(err) => { + return Err(err).wrap_err_with(|| { + format!("Failed to open data file {}", path.display()) + }) + } + } + } else { + 0 + }; + + println!( + "{},{},{},{},{:b},{},{},{:?},{},{:#010b}", + bundle.name().display(), + f.name(false, None), + f.file_type().ext_name(), + i, + v.property(), + v.data().len(), + v.external(), + data_file_name, + data_file_length, + v.unknown_1(), + ); + } + } + + Ok(()) +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + let bundles = matches + .get_many::("bundle") + .unwrap_or_default() + .cloned(); + + let paths = resolve_bundle_paths(bundles); + + let ctx = Arc::new(ctx); + + println!( + "Bundle Name,File Name,File Type,Variant,Property,Bundle Data Length,External,Data File,Data File Length,Unknown 1" + ); + + paths + .for_each_concurrent(10, |p| async { + let ctx = ctx.clone(); + async move { + if let Err(err) = handle_bundle(&ctx, &p) + .await + .wrap_err_with(|| format!("Failed to list contents of bundle {}", p.display())) + { + tracing::error!("Failed to handle bundle: {}", format!("{:#}", err)); + } + } + .await; + }) + .await; + + Ok(()) +} diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index 2e10b17..a36e58d 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -12,6 +12,7 @@ use clap::value_parser; use clap::{command, Arg}; use color_eyre::eyre; use color_eyre::eyre::{Context, Result}; +use sdk::murmur::Dictionary; use serde::{Deserialize, Serialize}; use tokio::fs::File; use tokio::io::BufReader; @@ -21,6 +22,7 @@ mod cmd { pub mod build; pub mod bundle; pub mod dictionary; + pub mod experiment; pub mod migrate; pub mod murmur; pub mod new; @@ -56,6 +58,7 @@ async fn main() -> Result<()> { .subcommand(cmd::build::command_definition()) .subcommand(cmd::bundle::command_definition()) .subcommand(cmd::dictionary::command_definition()) + .subcommand(cmd::experiment::command_definition()) .subcommand(cmd::migrate::command_definition()) .subcommand(cmd::murmur::command_definition()) .subcommand(cmd::new::command_definition()) @@ -96,8 +99,9 @@ async fn main() -> Result<()> { let r = BufReader::new(f); let mut ctx = ctx.write().await; - if let Err(err) = ctx.lookup.from_csv(r).await { - tracing::error!("{:#}", err); + match Dictionary::from_csv(r).await { + Ok(lookup) => ctx.lookup = Arc::new(lookup), + Err(err) => tracing::error!("{:#}", err), } }) }; @@ -133,6 +137,7 @@ async fn main() -> Result<()> { Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?, Some(("bundle", sub_matches)) => cmd::bundle::run(ctx, sub_matches).await?, Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?, + Some(("experiment", sub_matches)) => cmd::experiment::run(ctx, sub_matches).await?, Some(("migrate", sub_matches)) => cmd::migrate::run(ctx, sub_matches).await?, Some(("murmur", sub_matches)) => cmd::murmur::run(ctx, sub_matches).await?, Some(("new", sub_matches)) => cmd::new::run(ctx, sub_matches).await?, diff --git a/lib/oodle/src/lib.rs b/lib/oodle/src/lib.rs index 871daab..7edadee 100644 --- a/lib/oodle/src/lib.rs +++ b/lib/oodle/src/lib.rs @@ -52,6 +52,7 @@ impl From for bindings::OodleLZ_CheckCRC { #[tracing::instrument(skip(data))] pub fn decompress( data: I, + out_size: usize, fuzz_safe: OodleLZ_FuzzSafe, check_crc: OodleLZ_CheckCRC, ) -> Result> @@ -59,7 +60,7 @@ where I: AsRef<[u8]>, { let data = data.as_ref(); - let mut out = vec![0; CHUNK_SIZE]; + let mut out = vec![0; out_size]; let verbosity = if tracing::enabled!(tracing::Level::INFO) { bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_Minimal diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 9348e1b..83ccca0 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -44,10 +44,10 @@ impl FromBinary for Vec { pub mod sync { use std::ffi::CStr; - use std::io::{self, Read, Seek, SeekFrom}; + use std::io::{self, Read, Seek, SeekFrom, Write}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; - use color_eyre::eyre::WrapErr; + use color_eyre::eyre::{self, WrapErr}; use color_eyre::{Help, Report, Result, SectionExt}; macro_rules! make_read { @@ -123,15 +123,17 @@ pub mod sync { }; } - pub trait ReadExt: ReadBytesExt + Seek { + pub trait ReadExt: Read + Seek { fn read_u8(&mut self) -> io::Result { ReadBytesExt::read_u8(self) } + make_read!(read_u16, read_u16_le, u16); make_read!(read_u32, read_u32_le, u32); make_read!(read_u64, read_u64_le, u64); make_skip!(skip_u8, read_u8, u8); + make_skip!(skip_u16, read_u16, u16); make_skip!(skip_u32, read_u32, u32); // Implementation based on https://en.wikipedia.com/wiki/LEB128 @@ -181,9 +183,17 @@ pub mod sync { res } } + + fn read_bool(&mut self) -> Result { + match ReadExt::read_u8(self)? { + 0 => Ok(false), + 1 => Ok(true), + v => eyre::bail!("Invalid value for boolean '{}'", v), + } + } } - pub trait WriteExt: WriteBytesExt + Seek { + pub trait WriteExt: Write + Seek { fn write_u8(&mut self, val: u8) -> io::Result<()> { WriteBytesExt::write_u8(self, val) } @@ -191,6 +201,10 @@ pub mod sync { make_write!(write_u32, write_u32_le, u32); make_write!(write_u64, write_u64_le, u64); + fn write_bool(&mut self, val: bool) -> io::Result<()> { + WriteBytesExt::write_u8(self, if val { 1 } else { 0 }) + } + fn write_padding(&mut self) -> io::Result { let pos = self.stream_position()?; let size = 16 - (pos % 16) as usize; @@ -207,8 +221,8 @@ pub mod sync { } } - impl ReadExt for R {} - impl WriteExt for W {} + impl ReadExt for R {} + impl WriteExt for W {} pub(crate) fn _read_up_to(r: &mut R, buf: &mut Vec) -> Result where diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index f387409..aa18184 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -15,8 +15,9 @@ use super::filetype::BundleFileType; #[derive(Debug)] struct BundleFileHeader { variant: u32, - unknown_1: u8, + external: bool, size: usize, + unknown_1: u8, len_data_file_name: usize, } @@ -24,7 +25,7 @@ pub struct BundleFileVariant { property: u32, data: Vec, data_file_name: Option, - // Seems to be related to whether there is a data path. + external: bool, unknown_1: u8, } @@ -38,6 +39,7 @@ impl BundleFileVariant { property: 0, data: Vec::new(), data_file_name: None, + external: false, unknown_1: 0, } } @@ -62,21 +64,30 @@ impl BundleFileVariant { self.data_file_name.as_ref() } + pub fn external(&self) -> bool { + self.external + } + + pub fn unknown_1(&self) -> u8 { + self.unknown_1 + } + #[tracing::instrument(skip_all)] fn read_header(r: &mut R) -> Result where R: Read + Seek, { let variant = r.read_u32()?; - let unknown_1 = r.read_u8()?; + let external = r.read_bool()?; let size = r.read_u32()? as usize; - r.skip_u8(1)?; + let unknown_1 = r.read_u8()?; let len_data_file_name = r.read_u32()? as usize; Ok(BundleFileHeader { size, - unknown_1, + external, variant, + unknown_1, len_data_file_name, }) } @@ -87,7 +98,7 @@ impl BundleFileVariant { W: Write + Seek, { w.write_u32(self.property)?; - w.write_u8(self.unknown_1)?; + w.write_bool(self.external)?; let len_data_file_name = self.data_file_name.as_ref().map(|s| s.len()).unwrap_or(0); @@ -105,6 +116,26 @@ impl BundleFileVariant { } } +impl std::fmt::Debug for BundleFileVariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut out = f.debug_struct("BundleFileVariant"); + out.field("property", &self.property); + + if self.data.len() <= 5 { + out.field("data", &format!("{:x?}", &self.data)); + } else { + out.field( + "data", + &format!("{:x?}.. ({} bytes)", &self.data[..5], &self.data.len()), + ); + } + + out.field("data_file_name", &self.data_file_name) + .field("external", &self.external) + .finish() + } +} + bitflags! { #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { @@ -188,6 +219,7 @@ impl BundleFile { let s = r .read_string_len(header.len_data_file_name) .wrap_err("Failed to read data file name")?; + Some(s) } else { None @@ -200,6 +232,7 @@ impl BundleFile { property: header.variant, data, data_file_name, + external: header.external, unknown_1: header.unknown_1, }; @@ -227,7 +260,7 @@ impl BundleFile { for variant in self.variants.iter() { w.write_u32(variant.property())?; - w.write_u8(variant.unknown_1)?; + w.write_bool(variant.external)?; let len_data_file_name = variant.data_file_name().map(|s| s.len()).unwrap_or(0); @@ -261,6 +294,9 @@ impl BundleFile { ) -> Result { match file_type { BundleFileType::Lua => lua::compile(name, sjson).wrap_err("Failed to compile Lua file"), + BundleFileType::Texture => texture::compile(name, sjson, root) + .await + .wrap_err("Failed to compile Texture file"), BundleFileType::Unknown(_) => { eyre::bail!("Unknown file type. Cannot compile from SJSON"); } @@ -344,18 +380,16 @@ impl BundleFile { Ok(files) } - #[tracing::instrument(name = "File::decompiled", skip_all)] + #[tracing::instrument( + name = "File::decompiled", + skip_all, + fields(file = self.name(false, None), file_type = self.file_type().ext_name(), variants = self.variants.len()) + )] pub async fn decompiled(&self, ctx: &crate::Context) -> Result> { let file_type = self.file_type(); - if tracing::enabled!(tracing::Level::DEBUG) { - tracing::debug!( - name = self.name(true, None), - variants = self.variants.len(), - "Attempting to decompile" - ); - } - + // The `Strings` type handles all variants combined. + // For the other ones, each variant will be its own file. if file_type == BundleFileType::Strings { return strings::decompile(ctx, &self.variants); } @@ -371,6 +405,7 @@ impl BundleFile { let res = match file_type { BundleFileType::Lua => lua::decompile(ctx, data).await, BundleFileType::Package => package::decompile(ctx, name.clone(), data), + BundleFileType::Texture => texture::decompile(ctx, name.clone(), variant).await, _ => { tracing::debug!("Can't decompile, unknown file type"); Ok(vec![UserFile::with_name(data.to_vec(), name.clone())]) diff --git a/lib/sdk/src/bundle/filetype.rs b/lib/sdk/src/bundle/filetype.rs index 0b4f292..65209b4 100644 --- a/lib/sdk/src/bundle/filetype.rs +++ b/lib/sdk/src/bundle/filetype.rs @@ -67,6 +67,8 @@ pub enum BundleFileType { WwiseMetadata, WwiseStream, Xml, + Theme, + MissionThemes, Unknown(Murmur64), } @@ -136,6 +138,8 @@ impl BundleFileType { BundleFileType::WwiseMetadata => String::from("wwise_metadata"), BundleFileType::WwiseStream => String::from("wwise_stream"), BundleFileType::Xml => String::from("xml"), + BundleFileType::Theme => String::from("theme"), + BundleFileType::MissionThemes => String::from("mission_themes"), BundleFileType::Unknown(s) => format!("{s:016X}"), } @@ -222,6 +226,8 @@ impl std::str::FromStr for BundleFileType { "wwise_metadata" => BundleFileType::WwiseMetadata, "wwise_stream" => BundleFileType::WwiseStream, "xml" => BundleFileType::Xml, + "theme" => BundleFileType::Theme, + "mission_themes" => BundleFileType::MissionThemes, s => eyre::bail!("Unknown type string '{}'", s), }; @@ -310,6 +316,8 @@ impl From for BundleFileType { 0xd50a8b7e1c82b110 => BundleFileType::WwiseMetadata, 0x504b55235d21440e => BundleFileType::WwiseStream, 0x76015845a6003765 => BundleFileType::Xml, + 0x38BB9442048A7FBD => Self::Theme, + 0x80F2DE893657F83A => Self::MissionThemes, _ => BundleFileType::Unknown(Murmur64::from(hash)), } @@ -381,6 +389,8 @@ impl From for u64 { BundleFileType::WwiseMetadata => 0xd50a8b7e1c82b110, BundleFileType::WwiseStream => 0x504b55235d21440e, BundleFileType::Xml => 0x76015845a6003765, + BundleFileType::Theme => 0x38BB9442048A7FBD, + BundleFileType::MissionThemes => 0x80F2DE893657F83A, BundleFileType::Unknown(hash) => hash.into(), } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 075f4d2..e969f39 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -163,6 +163,7 @@ impl Bundle { // TODO: Optimize to not reallocate? let mut raw_buffer = oodle::decompress( &compressed_buffer, + oodle::CHUNK_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, ) @@ -360,6 +361,7 @@ where // TODO: Optimize to not reallocate? let mut raw_buffer = oodle::decompress( &compressed_buffer, + oodle::CHUNK_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, )?; diff --git a/lib/sdk/src/context.rs b/lib/sdk/src/context.rs index 1500290..8c10b3c 100644 --- a/lib/sdk/src/context.rs +++ b/lib/sdk/src/context.rs @@ -1,8 +1,11 @@ +use std::ffi::OsString; +use std::path::PathBuf; use std::process::Command; -use std::{ffi::OsString, path::PathBuf}; +use std::sync::Arc; use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64}; +#[derive(Clone)] pub struct CmdLine { cmd: OsString, args: Vec, @@ -52,7 +55,7 @@ impl From<&CmdLine> for Command { } pub struct Context { - pub lookup: Dictionary, + pub lookup: Arc, pub ljd: Option, pub revorb: Option, pub ww2ogg: Option, @@ -62,7 +65,7 @@ pub struct Context { impl Context { pub fn new() -> Self { Self { - lookup: Dictionary::new(), + lookup: Arc::new(Dictionary::new()), ljd: None, revorb: None, ww2ogg: None, diff --git a/lib/sdk/src/filetype/mod.rs b/lib/sdk/src/filetype/mod.rs index c62c503..b837993 100644 --- a/lib/sdk/src/filetype/mod.rs +++ b/lib/sdk/src/filetype/mod.rs @@ -1,3 +1,4 @@ pub mod lua; pub mod package; pub mod strings; +pub mod texture; diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs new file mode 100644 index 0000000..a8c85cd --- /dev/null +++ b/lib/sdk/src/filetype/texture.rs @@ -0,0 +1,387 @@ +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use bitflags::bitflags; +use color_eyre::eyre::Context; +use color_eyre::{eyre, SectionExt}; +use color_eyre::{Help, Result}; +use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; +use serde::Deserialize; +use tokio::fs; + +use crate::binary::sync::{ReadExt, WriteExt}; +use crate::bundle::file::UserFile; +use crate::murmur::{IdString32, IdString64}; +use crate::{BundleFile, BundleFileType, BundleFileVariant}; + +bitflags! { + #[derive(Clone, Copy, Debug)] + struct TextureFlags: u32 { + const STREAMABLE = 0b0000_0001; + const UNKNOWN = 1 << 1; + const SRGB = 1 << 8; + } +} + +#[derive(Clone, Debug)] +struct TextureHeader { + flags: TextureFlags, + n_streamable_mipmaps: u32, + width: u32, + height: u32, +} + +impl TextureHeader { + #[tracing::instrument(skip(r))] + fn from_binary(mut r: impl ReadExt) -> Result { + let flags = r.read_u32().and_then(|bits| { + TextureFlags::from_bits(bits) + .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) + })?; + let n_streamable_mipmaps = r.read_u32()?; + let width = r.read_u32()?; + let height = r.read_u32()?; + + // Don't quite know yet what this is, only that it is related to mipmaps. + // The reference to "streamable mipmaps" comes from VT2, so far. + // As such, it might be related to the stream file, but since all texture files have it, + // The engine calculates some offset and then moves 68 bytes at that offset to the beginning. + // Hence the split between `68` and `60` in the length. + r.seek(SeekFrom::Current(68 + 60))?; + + Ok(Self { + flags, + n_streamable_mipmaps, + width, + height, + }) + } + + #[tracing::instrument(skip(w))] + fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + eyre::ensure!( + self.flags.is_empty() && self.n_streamable_mipmaps == 0, + "Only textures are supported where `flags == 0` and `n_streamable_mipmaps == 0`." + ); + + w.write_u32(self.flags.bits())?; + w.write_u32(self.n_streamable_mipmaps)?; + w.write_u32(self.width)?; + w.write_u32(self.height)?; + + // See `from_binary` about this unknown section. + let buf = [0; 148]; + w.write_all(&buf)?; + + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct Texture { + header: TextureHeader, + data: Vec, + stream: Option>, + category: IdString32, +} + +impl Texture { + #[tracing::instrument(skip(r, stream_r))] + fn from_binary(mut r: impl Read + Seek, mut stream_r: Option) -> Result { + // Looking at the executable in IDA, there is one other valid value: `2`. + // If this ever comes up in the game data, I'll have to reverse engineer the + // (de)compression algorithm through IDA. + let compression_type = r.read_u32()?; + eyre::ensure!( + compression_type == 1, + "Unknown compression type for texture '{}'", + compression_type + ); + + let compressed_size = r.read_u32()? as usize; + let uncompressed_size = r.read_u32()? as usize; + + let out_buf = { + let mut comp_buf = vec![0; compressed_size]; + r.read_exact(&mut comp_buf)?; + + oodle::decompress( + comp_buf, + uncompressed_size, + OodleLZ_FuzzSafe::No, + OodleLZ_CheckCRC::No, + )? + }; + + eyre::ensure!( + out_buf.len() == uncompressed_size, + "Length of decompressed buffer did not match expected value. Expected {}, got {}", + uncompressed_size, + out_buf.len() + ); + + // No idea what this number is supposed to mean. + // Even the game engine just skips this one. + r.skip_u32(0x43)?; + + let header = TextureHeader::from_binary(&mut r)?; + + let meta_size = r.read_u32()?; + + eyre::ensure!( + meta_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. meta_size = {}, stream = {}", + meta_size, + stream_r.is_some() + ); + + let stream = if let Some(stream_r) = stream_r.as_mut() { + // Number of compression chunks in the stream file + let num_chunks = r.read_u32()?; + r.skip_u16(0)?; + + { + let num_chunks_1 = r.read_u16()? as u32; + + eyre::ensure!( + num_chunks == num_chunks_1, + "Chunk numbers don't match. first = {}, second = {}", + num_chunks, + num_chunks_1 + ); + } + + const RAW_SIZE: usize = 0x10000; + let mut stream_raw = Vec::new(); + let mut last = 0; + + for i in 0..num_chunks { + let offset_next = r.read_u32()? as usize; + let size = offset_next - last; + + let span = tracing::info_span!( + "read stream chunk", + num_chunks, + i, + chunk_size = size, + offset = last + ); + let _enter = span.enter(); + + let mut buf = vec![0; size]; + stream_r + .read_exact(&mut buf) + .wrap_err("Failed to read chunk from stream file")?; + + let raw = + oodle::decompress(&buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) + .wrap_err("Failed to decompress stream chunk")?; + + stream_raw.extend_from_slice(&raw); + + last = offset_next; + } + + Some(stream_raw) + } else { + None + }; + + let category = r.read_u32().map(IdString32::from)?; + + Ok(Self { + category, + header, + data: out_buf, + stream, + }) + } + + #[tracing::instrument(skip(w))] + fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + let compression_type = 1; + w.write_u32(compression_type)?; + + let comp_buf = oodle::compress(&self.data).wrap_err("Failed to compress DDS data")?; + + w.write_u32(comp_buf.len() as u32)?; + w.write_u32(self.data.len() as u32)?; + w.write_all(&comp_buf)?; + + // Unknown field, which the engine seems to ignore. + // All game files have the same value here, so we just mirror that. + w.write_u32(0x43)?; + + self.header.to_binary(&mut w)?; + + // More data not fully figured out, yet. + let meta_size = 0; + w.write_u32(meta_size)?; + + w.write_u32(self.category.to_murmur32().into())?; + Ok(()) + } + + #[tracing::instrument] + fn to_user_files(&self, name: String) -> Vec { + let mut files = Vec::with_capacity(2); + + // TODO: Don't clone. + + if let Some(stream) = &self.stream { + let stream_name = PathBuf::from(&name).with_extension("stream"); + files.push(UserFile::with_name( + stream.clone(), + stream_name.display().to_string(), + )); + } + + files.push(UserFile::with_name(self.data.clone(), name)); + files + } +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinition { + common: TextureDefinitionPlatform, + // Stingray supports per-platform sections here, where you can create overrides with the same + // values as in `common`. But since we only support PC, we don't need to implement + // that. +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionPlatform { + input: TextureDefinitionInput, + output: TextureDefinitionOutput, +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionInput { + filename: String, +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionOutput { + category: String, +} + +#[tracing::instrument(skip(data), fields(buf_len = data.as_ref().len()))] +pub(crate) async fn decompile_data( + name: String, + data: impl AsRef<[u8]>, + stream_file_name: Option, +) -> Result> { + let mut r = Cursor::new(data.as_ref()); + let mut stream_r = if let Some(file_name) = stream_file_name { + let stream_data = fs::read(&file_name) + .await + .wrap_err_with(|| format!("Failed to read stream file '{}'", file_name.display()))?; + Some(Cursor::new(stream_data)) + } else { + None + }; + + let texture = Texture::from_binary(&mut r, stream_r.as_mut())?; + let files = texture.to_user_files(name); + Ok(files) +} + +#[tracing::instrument(skip(ctx))] +pub(crate) async fn decompile( + ctx: &crate::Context, + name: String, + variant: &BundleFileVariant, +) -> Result> { + if !variant.external() { + tracing::debug!("Decompiling texture from bundle data"); + + let stream_file_name = variant.data_file_name().map(|name| match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(name), + None => PathBuf::from("bundle").join(name), + }); + + return decompile_data(name, variant.data(), stream_file_name).await; + } + + let Some(file_name) = variant.data_file_name() else { + eyre::bail!("Texture file has no data and no data file"); + }; + + tracing::debug!("Decompiling texture from external file '{}'", file_name); + + let path = match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(file_name), + None => PathBuf::from("bundle").join(file_name), + }; + + tracing::trace!(path = %path.display()); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + + decompile_data(name, &data, None).await +} + +#[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] +pub async fn compile( + name: IdString64, + sjson: impl AsRef, + root: impl AsRef + std::fmt::Debug, +) -> Result { + let definitions: TextureDefinition = serde_sjson::from_str(sjson.as_ref()) + .wrap_err("Failed to deserialize SJSON") + .with_section(|| sjson.as_ref().to_string().header("SJSON:"))?; + + let dds = { + let path = root.as_ref().join(definitions.common.input.filename); + fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read DDS file '{}'", path.display()))? + }; + + let (width, height) = { + let mut r = Cursor::new(&dds); + + let magic = r.read_u32()?; + eyre::ensure!( + magic == 0x20534444, + "Invalid magic bytes for DDS. Expected 0x20534444, got {:08x}", + magic + ); + + r.seek(SeekFrom::Current(5))?; + + let width = r.read_u16()? as u32; + let height = r.read_u16()? as u32; + + (width, height) + }; + + let mut w = Cursor::new(Vec::new()); + + let texture = Texture { + header: TextureHeader { + // As long as we can't handle mipmaps, these two need be `0` + flags: TextureFlags::empty(), + n_streamable_mipmaps: 0, + width, + height, + }, + data: dds, + stream: None, + category: IdString32::String(definitions.common.output.category), + }; + texture.to_binary(&mut w)?; + + let mut variant = BundleFileVariant::new(); + variant.set_data(w.into_inner()); + + let mut file = BundleFile::new(name, BundleFileType::Texture); + file.add_variant(variant); + + Ok(file) +} diff --git a/lib/sdk/src/murmur/dictionary.rs b/lib/sdk/src/murmur/dictionary.rs index 267f0a4..c1b5636 100644 --- a/lib/sdk/src/murmur/dictionary.rs +++ b/lib/sdk/src/murmur/dictionary.rs @@ -48,6 +48,7 @@ struct Row { group: HashGroup, } +#[derive(Clone)] pub struct Entry { value: String, long: Murmur64, @@ -73,6 +74,7 @@ impl Entry { } } +#[derive(Clone)] pub struct Dictionary { entries: Vec, } @@ -88,10 +90,12 @@ impl Dictionary { Self { entries: vec![] } } - pub async fn from_csv(&mut self, r: R) -> Result<()> + pub async fn from_csv(r: R) -> Result where R: AsyncRead + std::marker::Unpin + std::marker::Send, { + let mut entries = vec![]; + let r = AsyncDeserializer::from_reader(r); let mut records = r.into_deserialize::(); @@ -112,10 +116,10 @@ impl Dictionary { group: record.group, }; - self.entries.push(entry); + entries.push(entry); } - Ok(()) + Ok(Self { entries }) } pub async fn to_csv(&self, w: W) -> Result<()> @@ -161,7 +165,7 @@ impl Dictionary { self.entries.push(entry); } - pub fn find(&mut self, value: &String, group: HashGroup) -> Option<&Entry> { + pub fn find(&self, value: &String, group: HashGroup) -> Option<&Entry> { self.entries .iter() .find(|e| e.value == *value && e.group == group) From 66d1e375b9a819d564178ae36d74d1d5bc715437 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 22 Sep 2023 15:42:16 +0200 Subject: [PATCH 02/35] dtmt: Add option to compile file when injecting --- crates/dtmt/src/cmd/bundle/inject.rs | 114 +++++++++++++++------------ 1 file changed, 64 insertions(+), 50 deletions(-) diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 2b86691..4e5cbac 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -3,19 +3,28 @@ use std::path::PathBuf; use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::{self, Context, Result}; use color_eyre::Help; +use sdk::murmur::IdString64; use sdk::Bundle; use tokio::fs::{self, File}; use tokio::io::AsyncReadExt; pub(crate) fn command_definition() -> Command { Command::new("inject") - .about("Inject a file into a bundle.") + .about("Inject a file into a bundle.\n\ + Raw binary data can be used to directly replace the file's variant data blob without affecting the metadata.\n\ + Alternatively, a compiler format may be specified, and a complete bundle file is created.") .arg( Arg::new("replace") - .help("The name of a file in the bundle whos content should be replaced.") + .help("The name of a file in the bundle whose content should be replaced.") .short('r') .long("replace"), ) + .arg( + Arg::new("compile") + .help("Compile the file with the given data format before injecting.") + .long("compile") + .short('c') + ) .arg( Arg::new("output") .help( @@ -58,55 +67,60 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { Bundle::from_binary(&ctx, name, binary).wrap_err("Failed to open bundle file")? }; - if let Some(name) = matches.get_one::("replace") { - let mut file = File::open(&file_path) + let name = match matches.get_one::("replace") { + Some(name) => match u64::from_str_radix(name, 16) { + Ok(id) => IdString64::from(id), + Err(_) => IdString64::String(name.clone()), + }, + None => eyre::bail!("Currently, only the '--replace' operation is supported."), + }; + + let mut file = File::open(&file_path) + .await + .wrap_err_with(|| format!("Failed to open '{}'", file_path.display()))?; + + if let Some(variant) = bundle + .files_mut() + .filter(|file| file.matches_name(name.clone())) + // TODO: Handle file variants + .find_map(|file| file.variants_mut().next()) + { + let mut data = Vec::new(); + file.read_to_end(&mut data) .await - .wrap_err_with(|| format!("Failed to open '{}'", file_path.display()))?; - - if let Some(variant) = bundle - .files_mut() - .filter(|file| file.matches_name(name.clone())) - // TODO: Handle file variants - .find_map(|file| file.variants_mut().next()) - { - let mut data = Vec::new(); - file.read_to_end(&mut data) - .await - .wrap_err("Failed to read input file")?; - variant.set_data(data); - } else { - let err = eyre::eyre!("No file '{}' in this bundle.", name) - .with_suggestion(|| { - format!( - "Run '{} bundle list {}' to list the files in this bundle.", - clap::crate_name!(), - bundle_path.display() - ) - }) - .with_suggestion(|| { - format!( - "Use '{} bundle inject --add {} {} {}' to add it as a new file", - clap::crate_name!(), - name, - bundle_path.display(), - file_path.display() - ) - }); - - return Err(err); - } - - let out_path = matches.get_one::("output").unwrap_or(bundle_path); - let data = bundle - .to_binary() - .wrap_err("Failed to write changed bundle to output")?; - - fs::write(out_path, &data) - .await - .wrap_err("Failed to write data to output file")?; - - Ok(()) + .wrap_err("Failed to read input file")?; + variant.set_data(data); } else { - eyre::bail!("Currently, only the '--replace' operation is supported."); + let err = + eyre::eyre!("No file '{}' in this bundle.", name.display()).with_suggestion(|| { + format!( + "Run '{} bundle list {}' to list the files in this bundle.", + clap::crate_name!(), + bundle_path.display() + ) + // Not yet supported. + // }) + // .with_suggestion(|| { + // format!( + // "Use '{} bundle inject --add {} {} {}' to add it as a new file", + // clap::crate_name!(), + // name.display(), + // bundle_path.display(), + // file_path.display() + // ) + }); + + return Err(err); } + + let out_path = matches.get_one::("output").unwrap_or(bundle_path); + let data = bundle + .to_binary() + .wrap_err("Failed to write changed bundle to output")?; + + fs::write(out_path, &data) + .await + .wrap_err("Failed to write data to output file")?; + + Ok(()) } From 6926dabbab6bf068a5c11e08b665bc09d678eec3 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 6 Oct 2023 10:01:18 +0200 Subject: [PATCH 03/35] sdk: Add dictionary group for texture categories --- lib/sdk/src/context.rs | 8 ++++---- lib/sdk/src/filetype/strings.rs | 6 +++--- lib/sdk/src/filetype/texture.rs | 21 +++++++++++++-------- lib/sdk/src/murmur/dictionary.rs | 12 ++++++++++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/sdk/src/context.rs b/lib/sdk/src/context.rs index 8c10b3c..c565429 100644 --- a/lib/sdk/src/context.rs +++ b/lib/sdk/src/context.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::process::Command; use std::sync::Arc; -use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64}; +use crate::murmur::{Dictionary, HashGroup, IdString32, IdString64, Murmur32, Murmur64}; #[derive(Clone)] pub struct CmdLine { @@ -87,17 +87,17 @@ impl Context { } } - pub fn lookup_hash_short(&self, hash: M, group: HashGroup) -> String + pub fn lookup_hash_short(&self, hash: M, group: HashGroup) -> IdString32 where M: Into, { let hash = hash.into(); if let Some(s) = self.lookup.lookup_short(hash, group) { tracing::debug!(%hash, string = s, "Murmur32 lookup successful"); - s.to_owned() + s.to_string().into() } else { tracing::debug!(%hash, "Murmur32 lookup failed"); - format!("{hash:08X}") + hash.into() } } } diff --git a/lib/sdk/src/filetype/strings.rs b/lib/sdk/src/filetype/strings.rs index 8643266..c7ad6f9 100644 --- a/lib/sdk/src/filetype/strings.rs +++ b/lib/sdk/src/filetype/strings.rs @@ -5,7 +5,7 @@ use color_eyre::{Report, Result}; use crate::binary::sync::ReadExt; use crate::bundle::file::{BundleFileVariant, UserFile}; -use crate::murmur::HashGroup; +use crate::murmur::{HashGroup, IdString32}; #[derive(Copy, Clone, PartialEq, Eq, Hash, serde::Serialize)] #[serde(untagged)] @@ -26,7 +26,7 @@ impl Language { } #[derive(serde::Serialize)] -pub struct Strings(HashMap>); +pub struct Strings(HashMap>); #[inline(always)] fn read_string(r: R) -> Result @@ -46,7 +46,7 @@ where impl Strings { #[tracing::instrument(skip_all, fields(languages = variants.len()))] pub fn from_variants(ctx: &crate::Context, variants: &[BundleFileVariant]) -> Result { - let mut map: HashMap> = HashMap::new(); + let mut map: HashMap> = HashMap::new(); for (i, variant) in variants.iter().enumerate() { let _span = tracing::trace_span!("variant {}", i); diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index a8c85cd..7486fb8 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -11,7 +11,7 @@ use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; -use crate::murmur::{IdString32, IdString64}; +use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; bitflags! { @@ -86,8 +86,12 @@ struct Texture { } impl Texture { - #[tracing::instrument(skip(r, stream_r))] - fn from_binary(mut r: impl Read + Seek, mut stream_r: Option) -> Result { + #[tracing::instrument(skip(ctx, r, stream_r))] + fn from_binary( + ctx: &crate::Context, + mut r: impl Read + Seek, + mut stream_r: Option, + ) -> Result { // Looking at the executable in IDA, there is one other valid value: `2`. // If this ever comes up in the game data, I'll have to reverse engineer the // (de)compression algorithm through IDA. @@ -187,7 +191,7 @@ impl Texture { None }; - let category = r.read_u32().map(IdString32::from)?; + let category = ctx.lookup_hash_short(r.read_u32()?, HashGroup::TextureCategory); Ok(Self { category, @@ -265,8 +269,9 @@ struct TextureDefinitionOutput { category: String, } -#[tracing::instrument(skip(data), fields(buf_len = data.as_ref().len()))] +#[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] pub(crate) async fn decompile_data( + ctx: &crate::Context, name: String, data: impl AsRef<[u8]>, stream_file_name: Option, @@ -281,7 +286,7 @@ pub(crate) async fn decompile_data( None }; - let texture = Texture::from_binary(&mut r, stream_r.as_mut())?; + let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; let files = texture.to_user_files(name); Ok(files) } @@ -300,7 +305,7 @@ pub(crate) async fn decompile( None => PathBuf::from("bundle").join(name), }); - return decompile_data(name, variant.data(), stream_file_name).await; + return decompile_data(ctx, name, variant.data(), stream_file_name).await; } let Some(file_name) = variant.data_file_name() else { @@ -323,7 +328,7 @@ pub(crate) async fn decompile( "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." })?; - decompile_data(name, &data, None).await + decompile_data(ctx, name, &data, None).await } #[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] diff --git a/lib/sdk/src/murmur/dictionary.rs b/lib/sdk/src/murmur/dictionary.rs index c1b5636..55e0966 100644 --- a/lib/sdk/src/murmur/dictionary.rs +++ b/lib/sdk/src/murmur/dictionary.rs @@ -12,12 +12,19 @@ pub enum HashGroup { Filename, Filetype, Strings, + TextureCategory, Other, } impl HashGroup { - pub fn all() -> [Self; 3] { - [Self::Filename, Self::Filetype, Self::Other] + pub fn all() -> [Self; 5] { + [ + Self::Filename, + Self::Filetype, + Self::Strings, + Self::TextureCategory, + Self::Other, + ] } } @@ -27,6 +34,7 @@ impl std::fmt::Display for HashGroup { HashGroup::Filename => write!(f, "filename"), HashGroup::Filetype => write!(f, "filetype"), HashGroup::Strings => write!(f, "strings"), + HashGroup::TextureCategory => write!(f, "texture-category"), HashGroup::Other => write!(f, "other"), } } From 5e19d5cb348199d1be65fc4e39b3dc4ae8c892ca Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 6 Oct 2023 11:01:15 +0200 Subject: [PATCH 04/35] sdk: Add decompiled SJSON texture file In addition to the actual image file, also write a `.texture` engine file. --- lib/sdk/src/filetype/texture.rs | 83 +++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 7486fb8..7912e45 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -6,7 +6,7 @@ use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; @@ -14,6 +14,30 @@ use crate::bundle::file::UserFile; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinition { + common: TextureDefinitionPlatform, + // Stingray supports per-platform sections here, where you can create overrides with the same + // values as in `common`. But since we only support PC, we don't need to implement + // that. +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionPlatform { + input: TextureDefinitionInput, + output: TextureDefinitionOutput, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionInput { + filename: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionOutput { + category: String, +} + bitflags! { #[derive(Clone, Copy, Debug)] struct TextureFlags: u32 { @@ -227,8 +251,21 @@ impl Texture { } #[tracing::instrument] - fn to_user_files(&self, name: String) -> Vec { - let mut files = Vec::with_capacity(2); + fn to_sjson(&self, filename: String) -> Result { + let texture = TextureDefinition { + common: TextureDefinitionPlatform { + input: TextureDefinitionInput { filename }, + output: TextureDefinitionOutput { + category: self.category.display().to_string(), + }, + }, + }; + serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") + } + + #[tracing::instrument] + fn to_user_files(&self, name: String) -> Result> { + let mut files = Vec::with_capacity(3); // TODO: Don't clone. @@ -240,35 +277,20 @@ impl Texture { )); } + { + let data = self.to_sjson(name.clone())?.as_bytes().to_vec(); + let name = PathBuf::from(&name) + .with_extension("texture") + .display() + .to_string(); + files.push(UserFile::with_name(data, name)); + } + files.push(UserFile::with_name(self.data.clone(), name)); - files + Ok(files) } } -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinition { - common: TextureDefinitionPlatform, - // Stingray supports per-platform sections here, where you can create overrides with the same - // values as in `common`. But since we only support PC, we don't need to implement - // that. -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionPlatform { - input: TextureDefinitionInput, - output: TextureDefinitionOutput, -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionInput { - filename: String, -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionOutput { - category: String, -} - #[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] pub(crate) async fn decompile_data( ctx: &crate::Context, @@ -287,8 +309,9 @@ pub(crate) async fn decompile_data( }; let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; - let files = texture.to_user_files(name); - Ok(files) + texture + .to_user_files(name) + .wrap_err("Failed to build user files") } #[tracing::instrument(skip(ctx))] From aeed53bf88bc198d195b03fc174b4c0f32a4e235 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 22 Jul 2024 11:28:36 +0200 Subject: [PATCH 05/35] Reverse DDSImage::load Decompiling the game binary shows a rather elaborate algorithm to load DDS images from binary. Though comparing it to Microsoft's documentation on DDS, most of it seems to be pretty standard handling. However, we don't actually need all of it. The part about calculating pitch and reading blocks only accesses a subset of the `ImageFormat` struct, so we can strip our implementation to just that. --- lib/sdk/src/filetype/texture.rs | 29 ++-- lib/sdk/src/filetype/texture/dds.rs | 203 ++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 lib/sdk/src/filetype/texture/dds.rs diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 7912e45..58c0dbf 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -14,6 +14,8 @@ use crate::bundle::file::UserFile; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; +mod dds; + #[derive(Clone, Debug, Deserialize, Serialize)] struct TextureDefinition { common: TextureDefinitionPlatform, @@ -53,6 +55,7 @@ struct TextureHeader { n_streamable_mipmaps: u32, width: u32, height: u32, + mip_info_size: u32, } impl TextureHeader { @@ -66,18 +69,19 @@ impl TextureHeader { let width = r.read_u32()?; let height = r.read_u32()?; - // Don't quite know yet what this is, only that it is related to mipmaps. - // The reference to "streamable mipmaps" comes from VT2, so far. - // As such, it might be related to the stream file, but since all texture files have it, - // The engine calculates some offset and then moves 68 bytes at that offset to the beginning. - // Hence the split between `68` and `60` in the length. - r.seek(SeekFrom::Current(68 + 60))?; + r.skip_u32(0)?; + + // A section of 15 pairs of two u32 + r.seek(SeekFrom::Current(2 * 4 * 15))?; + + let mip_info_size = r.read_u32()?; Ok(Self { flags, n_streamable_mipmaps, width, height, + mip_info_size, }) } @@ -94,9 +98,12 @@ impl TextureHeader { w.write_u32(self.height)?; // See `from_binary` about this unknown section. - let buf = [0; 148]; + let buf = [0; (2 * 4 * 15) + 4]; w.write_all(&buf)?; + // TODO: For now we write `0` here, until the mipmap section is figured out + w.write_u32(0)?; + Ok(()) } } @@ -154,12 +161,10 @@ impl Texture { let header = TextureHeader::from_binary(&mut r)?; - let meta_size = r.read_u32()?; - eyre::ensure!( - meta_size == 0 || stream_r.is_some(), - "Compression chunks and stream file don't match up. meta_size = {}, stream = {}", - meta_size, + header.mip_info_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. mip_info_size = {}, has_stream = {}", + header.mip_info_size, stream_r.is_some() ); diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs new file mode 100644 index 0000000..135d24c --- /dev/null +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -0,0 +1,203 @@ +use bitflags::bitflags; +use color_eyre::eyre; +use color_eyre::Result; + +bitflags! { + #[derive(Clone, Copy, Debug)] + pub struct DDSD: u32 { + /// Required + const CAPS = 0x1; + /// Required + const HEIGHT = 0x2; + /// Required + const WIDTH = 0x4; + /// Pitch for an uncompressed texture + const PITCH = 0x8; + /// Required + const PIXELFORMAT = 0x1000; + /// Required in a mipmapped texture + const MIPMAPCOUNT = 0x20000; + /// Pitch for a compressed texture + const LINEARSIZE = 0x80000; + /// Required in a depth texture + const DEPTH = 0x800000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DDSCAPS: u32 { + const COMPLEX = 0x8; + const MIPMAP = 0x400000; + const TEXTURE = 0x1000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DDSCAPS2: u32 { + const CUBEMAP = 0x200; + const CUBEMAP_POSITIVEX = 0x400; + const CUBEMAP_NEGATIVEX = 0x800; + const CUBEMAP_POSITIVEY = 0x1000; + const CUBEMAP_NEGATIVEY = 0x2000; + const CUBEMAP_POSITIVEZ = 0x4000; + const CUBEMAP_NEGATIVEZ = 0x8000; + const VOLUME = 0x200000; + + const CUBEMAP_ALLFACES = Self::CUBEMAP_POSITIVEX.bits() + | Self::CUBEMAP_NEGATIVEX.bits() + | Self::CUBEMAP_POSITIVEY.bits() + | Self::CUBEMAP_NEGATIVEY.bits() + | Self::CUBEMAP_POSITIVEZ.bits() + | Self::CUBEMAP_NEGATIVEZ.bits(); + } + + #[derive(Clone, Copy, Debug)] + pub struct DDPF: u32 { + const ALPHAPIXELS = 0x1; + const ALPHA = 0x2; + const FOURCC = 0x4; + const RGB = 0x40; + const YUV = 0x200; + const LUMINANCE = 0x20000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DdsResourceMiscFlags: u32 { + const TEXTURECUBE = 0x4; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum D3D10ResourceDimension { + Unknown = 0, + Buffer = 1, + Texture1D = 2, + Texture2D = 3, + Texture3D = 4, +} + +pub struct Dx10Header { + /// Resource data formats, including fully-typed and typeless formats. + /// See https://learn.microsoft.com/en-us/windows/win32/api/dxgiformat/ne-dxgiformat-dxgi_format + dxgi_format: u32, + resource_dimension: D3D10ResourceDimension, + misc_flag: DdsResourceMiscFlags, + array_size: u32, + misc_flags2: u32, +} + +pub struct DDSPixelFormat { + /// Structure size. Must be `32`. + size: u32, + flags: DDPF, + four_cc: u32, + rgb_bit_count: u32, + r_bit_mask: u32, + g_bit_mask: u32, + b_bit_mask: u32, + a_bit_mask: u32, +} + +pub struct DDSHeader { + /// Size of this structure. Must be `124`. + size: u32, + /// Flags to indicate which members contain valid data. + flags: DDSD, + height: u32, + width: u32, + pitch_or_linear_size: u32, + depth: u32, + mipmap_count: u32, + reserved_1: [u8; 11], + pixel_format: DDSPixelFormat, + caps: DDSCAPS, + caps_2: DDSCAPS2, + reserved_2: [u8; 3], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImageType { + Image2D = 0, + Image3D = 1, + ImageCube = 2, + Unknown = 3, + Image2dArray = 4, + ImagecubeArray = 5, +} + +/// A stripped version of `ImageType` that only contains just the data needed +/// to read a DDS image stream. +pub struct StrippedImageFormat { + pub image_type: ImageType, + pub width: u32, + pub height: u32, + pub layers: u32, + pub mip_levels: u32, +} + +// This is a stripped down version of the logic that the engine implements to fill +// `stingray::ImageFormat`. With the `type` field we need to distinguish between `IMAGE3D` +// and everything else, and we need the various dimensions filled to calculate the chunks. +pub fn stripped_format_from_header( + dds_header: &DDSHeader, + dx10_header: &Dx10Header, +) -> Result { + let mut image_format = StrippedImageFormat { + image_type: ImageType::Unknown, + width: dds_header.width, + height: dds_header.height, + layers: 0, + mip_levels: 0, + }; + + if dds_header.mipmap_count > 0 { + image_format.mip_levels = dds_header.mipmap_count; + } else { + image_format.mip_levels = 1; + } + + // INFO: These next two sections are conditional in the engine code, + // based on a lot of stuff in "fourcc" and other fields. But it might + // actually be fine to just do it like this, as this seems universal + // to DDS. + // Will have to check how it plays out with actual assets. + + if dds_header.caps_2.contains(DDSCAPS2::CUBEMAP) { + image_format.image_type = ImageType::ImageCube; + image_format.layers = 6; + } else if dds_header.caps_2.contains(DDSCAPS2::VOLUME) { + image_format.image_type = ImageType::Image3D; + image_format.layers = dds_header.depth; + } else { + image_format.image_type = ImageType::Image2D; + image_format.layers = 1; + } + + if dx10_header.resource_dimension == D3D10ResourceDimension::Texture2D { + if dx10_header.misc_flag == DdsResourceMiscFlags::TextureCube { + image_format.image_type = ImageType::ImageCube; + if dx10_header.array_size > 1 { + image_format.layers = dx10_header.array_size; + } else { + image_format.layers = 6; + } + } else { + image_format.image_type = ImageType::Image2D; + image_format.layers = dx10_header.array_size; + } + } else if dx10_header.resource_dimension == D3D10ResourceDimension::Texture3D { + image_format.image_type = ImageType::Image3D; + image_format.layers = dds_header.depth; + } + + if dx10_header.array_size > 1 { + match image_format.image_type { + ImageType::Image2D => image_format.image_type = ImageType::Image2dArray, + ImageType::ImageCube => image_format.image_type = ImageType::ImagecubeArray, + ImageType::Image3D => { + eyre::bail!("3D-Arrays are not a supported image format") + } + _ => {} + } + } + + Ok(image_format) +} From 634fc310eec4813fb503e74443a65a378b8ec331 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 24 Jul 2024 14:15:51 +0200 Subject: [PATCH 06/35] sdk: Implement decompiling streamed mipmaps For now, we only extract the largest mipmap. --- Cargo.lock | 36 +++ Cargo.toml | 3 + lib/sdk/Cargo.toml | 3 + lib/sdk/src/filetype/texture.rs | 319 +++++++++++++++++------ lib/sdk/src/filetype/texture/dds.rs | 384 +++++++++++++++++++++++++--- lib/sdk/src/lib.rs | 1 + 6 files changed, 639 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4ec109..6080763 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2480,6 +2480,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3335,11 +3346,14 @@ dependencies = [ "glob", "luajit2-sys", "nanorand", + "num-derive", + "num-traits", "oodle", "path-slash", "pin-project-lite", "serde", "serde_sjson", + "strum", "tokio", "tokio-stream", "tracing", @@ -3601,6 +3615,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.100", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 87d9ea6..8c104d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,8 @@ luajit2-sys = { path = "lib/luajit2-sys" } minijinja = { version = "2.0.1", default-features = false, features = ["serde"] } nanorand = "0.7.0" nexusmods = { path = "lib/nexusmods" } +num-derive = "0.4.2" +num-traits = "0.2.19" notify = "8.0.0" oodle = { path = "lib/oodle" } open = "5.0.1" @@ -49,6 +51,7 @@ serde = { version = "1.0.152", features = ["derive", "rc"] } serde_sjson = "1.2.1" steamlocate = "2.0.0-beta.2" strip-ansi-escapes = "0.2.0" +strum = { version = "0.26.3", features = ["derive", "strum_macros"] } time = { version = "0.3.20", features = ["serde", "serde-well-known", "local-offset", "formatting", "macros"] } tokio = { version = "1.23.0", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] } tokio-stream = { version = "0.1.12", features = ["fs", "io-util"] } diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 4667a1c..0cd0e4e 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -15,11 +15,14 @@ futures-util = { workspace = true } glob = { workspace = true } luajit2-sys = { workspace = true } nanorand = { workspace = true } +num-derive = { workspace = true } +num-traits = { workspace = true } oodle = { workspace = true } path-slash = { workspace = true } pin-project-lite = { workspace = true } serde = { workspace = true } serde_sjson = { workspace = true } +strum = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 58c0dbf..1565b8c 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -1,16 +1,18 @@ -use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::io::{Cursor, Read, Seek, SeekFrom, Write as _}; use std::path::{Path, PathBuf}; use bitflags::bitflags; use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; +use num_traits::ToPrimitive as _; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; +use crate::filetype::texture::dds::{DXGIFormat, ImageType}; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; @@ -49,13 +51,20 @@ bitflags! { } } +#[derive(Copy, Clone, Debug, Default)] +struct TextureHeaderMipInfo { + offset: usize, + size: usize, +} + #[derive(Clone, Debug)] struct TextureHeader { flags: TextureFlags, - n_streamable_mipmaps: u32, - width: u32, - height: u32, - mip_info_size: u32, + n_streamable_mipmaps: usize, + width: usize, + height: usize, + mip_infos: [TextureHeaderMipInfo; 16], + meta_size: usize, } impl TextureHeader { @@ -65,23 +74,26 @@ impl TextureHeader { TextureFlags::from_bits(bits) .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) })?; - let n_streamable_mipmaps = r.read_u32()?; - let width = r.read_u32()?; - let height = r.read_u32()?; + let n_streamable_mipmaps = r.read_u32()? as usize; + let width = r.read_u32()? as usize; + let height = r.read_u32()? as usize; - r.skip_u32(0)?; + let mut mip_infos = [TextureHeaderMipInfo::default(); 16]; - // A section of 15 pairs of two u32 - r.seek(SeekFrom::Current(2 * 4 * 15))?; + for info in mip_infos.iter_mut() { + info.offset = r.read_u32()? as usize; + info.size = r.read_u32()? as usize; + } - let mip_info_size = r.read_u32()?; + let meta_size = r.read_u32()? as usize; Ok(Self { flags, n_streamable_mipmaps, width, height, - mip_info_size, + mip_infos, + meta_size, }) } @@ -93,15 +105,16 @@ impl TextureHeader { ); w.write_u32(self.flags.bits())?; - w.write_u32(self.n_streamable_mipmaps)?; - w.write_u32(self.width)?; - w.write_u32(self.height)?; + w.write_u32(self.n_streamable_mipmaps as u32)?; + w.write_u32(self.width as u32)?; + w.write_u32(self.height as u32)?; - // See `from_binary` about this unknown section. - let buf = [0; (2 * 4 * 15) + 4]; - w.write_all(&buf)?; + for info in self.mip_infos { + w.write_u32(info.offset as u32)?; + w.write_u32(info.size as u32)?; + } - // TODO: For now we write `0` here, until the mipmap section is figured out + // TODO: For now we write `0` here, until the meta section is figured out w.write_u32(0)?; Ok(()) @@ -117,6 +130,90 @@ struct Texture { } impl Texture { + #[tracing::instrument(skip(data, chunks))] + fn decompress_stream_data(mut data: impl Read, chunks: impl AsRef<[usize]>) -> Result> { + const RAW_SIZE: usize = 0x10000; + + let chunks = chunks.as_ref(); + + let max_size = chunks.iter().max().copied().unwrap_or(RAW_SIZE); + let mut read_buf = vec![0; max_size]; + + let mut stream_raw = Vec::with_capacity(chunks.iter().sum()); + let mut last = 0; + + for offset_next in chunks { + let size = offset_next - last; + + let span = tracing::info_span!( + "stream chunk", + num_chunks = chunks.len(), + chunk_size_comp = size, + offset = last + ); + let _enter = span.enter(); + + let buf = &mut read_buf[0..size]; + data.read_exact(buf) + .wrap_err("Failed to read chunk from stream file")?; + + let raw = oodle::decompress(buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) + .wrap_err("Failed to decompress stream chunk")?; + eyre::ensure!( + raw.len() == RAW_SIZE, + "Invalid chunk length after decompression" + ); + + stream_raw.extend_from_slice(&raw); + + last = *offset_next; + } + Ok(stream_raw) + } + + #[tracing::instrument(skip(data), fields(data_len = data.as_ref().len()))] + fn reorder_stream_mipmap( + data: impl AsRef<[u8]>, + bits_per_block: usize, + bytes_per_block: usize, + block_size: usize, + pitch: usize, + ) -> Result> { + const CHUNK_SIZE: usize = 0x10000; + let data = data.as_ref(); + + let mut out = Vec::with_capacity(data.len()); + let mut window = vec![0u8; pitch * 64]; + + let row_size = bits_per_block * block_size; + tracing::Span::current().record("row_size", row_size); + + eyre::ensure!( + data.len() % CHUNK_SIZE == 0, + "Stream data does not divide evenly into chunks" + ); + + for (i, chunk) in data.chunks_exact(CHUNK_SIZE).enumerate() { + let chunk_x = (i % bytes_per_block) * row_size; + + let span = tracing::trace_span!("chunk", i, chunk_x = chunk_x); + let _guard = span.enter(); + + if i > 0 && i % bytes_per_block == 0 { + out.extend_from_slice(&window); + } + + for (j, row) in chunk.chunks_exact(row_size).enumerate() { + let start = chunk_x + j * pitch; + let end = start + row_size; + tracing::trace!("{i}/{j} at {}:{}", start, end); + window[start..end].copy_from_slice(row); + } + } + + Ok(out) + } + #[tracing::instrument(skip(ctx, r, stream_r))] fn from_binary( ctx: &crate::Context, @@ -162,19 +259,19 @@ impl Texture { let header = TextureHeader::from_binary(&mut r)?; eyre::ensure!( - header.mip_info_size == 0 || stream_r.is_some(), - "Compression chunks and stream file don't match up. mip_info_size = {}, has_stream = {}", - header.mip_info_size, + header.meta_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. meta_size = {}, has_stream = {}", + header.meta_size, stream_r.is_some() ); let stream = if let Some(stream_r) = stream_r.as_mut() { // Number of compression chunks in the stream file - let num_chunks = r.read_u32()?; + let num_chunks = r.read_u32()? as usize; r.skip_u16(0)?; { - let num_chunks_1 = r.read_u16()? as u32; + let num_chunks_1 = r.read_u16()? as usize; eyre::ensure!( num_chunks == num_chunks_1, @@ -184,37 +281,15 @@ impl Texture { ); } - const RAW_SIZE: usize = 0x10000; - let mut stream_raw = Vec::new(); - let mut last = 0; + let mut chunks = Vec::with_capacity(num_chunks); - for i in 0..num_chunks { - let offset_next = r.read_u32()? as usize; - let size = offset_next - last; - - let span = tracing::info_span!( - "read stream chunk", - num_chunks, - i, - chunk_size = size, - offset = last - ); - let _enter = span.enter(); - - let mut buf = vec![0; size]; - stream_r - .read_exact(&mut buf) - .wrap_err("Failed to read chunk from stream file")?; - - let raw = - oodle::decompress(&buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) - .wrap_err("Failed to decompress stream chunk")?; - - stream_raw.extend_from_slice(&raw); - - last = offset_next; + for _ in 0..num_chunks { + chunks.push(r.read_u32()? as usize); } + let stream_raw = Self::decompress_stream_data(stream_r, chunks) + .wrap_err("Failed to decompress stream data")?; + Some(stream_raw) } else { None @@ -247,10 +322,6 @@ impl Texture { self.header.to_binary(&mut w)?; - // More data not fully figured out, yet. - let meta_size = 0; - w.write_u32(meta_size)?; - w.write_u32(self.category.to_murmur32().into())?; Ok(()) } @@ -268,19 +339,9 @@ impl Texture { serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") } - #[tracing::instrument] + #[tracing::instrument(skip(self))] fn to_user_files(&self, name: String) -> Result> { - let mut files = Vec::with_capacity(3); - - // TODO: Don't clone. - - if let Some(stream) = &self.stream { - let stream_name = PathBuf::from(&name).with_extension("stream"); - files.push(UserFile::with_name( - stream.clone(), - stream_name.display().to_string(), - )); - } + let mut files = Vec::with_capacity(2); { let data = self.to_sjson(name.clone())?.as_bytes().to_vec(); @@ -291,7 +352,119 @@ impl Texture { files.push(UserFile::with_name(data, name)); } - files.push(UserFile::with_name(self.data.clone(), name)); + // For debugging purposes, also extract the raw files + if cfg!(debug_assertions) { + if let Some(stream) = &self.stream { + let stream_name = PathBuf::from(&name).with_extension("stream"); + files.push(UserFile::with_name( + stream.clone(), + stream_name.display().to_string(), + )); + } + + let name = PathBuf::from(&name) + .with_extension("raw.dds") + .display() + .to_string(); + files.push(UserFile::with_name(self.data.clone(), name)); + } + + { + let mut data = Cursor::new(&self.data); + let mut dds_header = + dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; + + eyre::ensure!( + dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) + && dds_header.pixel_format.four_cc == dds::FOURCC_DX10, + "Only DX10 textures are currently supported." + ); + + let dx10_header = + dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; + + match dx10_header.dxgi_format { + DXGIFormat::BC1_UNORM + | DXGIFormat::BC3_UNORM + | DXGIFormat::BC4_UNORM + | DXGIFormat::BC5_UNORM + | DXGIFormat::BC6H_UF16 + | DXGIFormat::BC7_UNORM => {} + _ => { + eyre::bail!( + "Unsupported DXGI format: {} (0x{:0X})", + dx10_header.dxgi_format, + dx10_header.dxgi_format.to_u32().unwrap_or_default() + ); + } + } + + let stingray_image_format = + dds::stripped_format_from_header(&dds_header, &dx10_header)?; + eyre::ensure!( + stingray_image_format.image_type == ImageType::Image2D, + "Unsupported image type: {}", + stingray_image_format.image_type, + ); + + let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; + let bits_per_block: usize = match block_size { + 8 => 128, + 16 => 64, + block_size => eyre::bail!("Unsupported block size {}", block_size), + }; + + let pitch = self.header.width / 4 * block_size; + let bytes_per_block = self.header.width / bits_per_block / 4; + + tracing::debug!( + "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", + block_size, + pitch, + bits_per_block, + bytes_per_block + ); + + let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); + + // Currently, we only extract the largest mipmap, + // so we need to set the dimensions accordingly, and remove the + // flag. + dds_header.width = self.header.width; + dds_header.height = self.header.height; + dds_header.mipmap_count = 0; + dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; + + dds_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DDS header")?; + + dx10_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DX10 header")?; + + if let Some(stream) = &self.stream { + let data = Self::reorder_stream_mipmap( + stream, + bits_per_block, + bytes_per_block, + block_size, + pitch, + ) + .wrap_err("Failed to reorder stream chunks")?; + + out_data + .write_all(&data) + .wrap_err("Failed to write streamed mipmap data")?; + } else { + out_data + .write_all(data.split().1) + .wrap_err("Failed to write texture data")?; + }; + + files.push(UserFile::with_name(out_data.into_inner(), name)); + } + Ok(files) } } @@ -388,8 +561,8 @@ pub async fn compile( r.seek(SeekFrom::Current(5))?; - let width = r.read_u16()? as u32; - let height = r.read_u16()? as u32; + let width = r.read_u32()? as usize; + let height = r.read_u32()? as usize; (width, height) }; @@ -403,6 +576,8 @@ pub async fn compile( n_streamable_mipmaps: 0, width, height, + mip_infos: [TextureHeaderMipInfo::default(); 16], + meta_size: 0, }, data: dds, stream: None, diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs index 135d24c..5ef1b90 100644 --- a/lib/sdk/src/filetype/texture/dds.rs +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -1,6 +1,16 @@ +use std::io::SeekFrom; + use bitflags::bitflags; -use color_eyre::eyre; +use color_eyre::eyre::Context as _; +use color_eyre::eyre::{self, OptionExt as _}; use color_eyre::Result; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::{FromPrimitive as _, ToPrimitive as _}; + +use crate::binary::sync::{ReadExt, WriteExt}; + +const MAGIC_DDS: u32 = 0x20534444; +pub const FOURCC_DX10: u32 = 0x30315844; bitflags! { #[derive(Clone, Copy, Debug)] @@ -65,7 +75,27 @@ bitflags! { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +fn flags_from_bits(bits: T::Bits) -> T +where + ::Bits: std::fmt::Binary, +{ + if let Some(flags) = T::from_bits(bits) { + flags + } else { + let unknown = bits & !T::all().bits(); + + tracing::warn!( + "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", + std::any::type_name::(), + T::all().bits(), + unknown + ); + + T::from_bits_truncate(bits) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, FromPrimitive, ToPrimitive)] pub enum D3D10ResourceDimension { Unknown = 0, Buffer = 1, @@ -74,46 +104,325 @@ pub enum D3D10ResourceDimension { Texture3D = 4, } +#[allow(clippy::upper_case_acronyms)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, strum::Display, FromPrimitive, ToPrimitive)] +pub enum DXGIFormat { + UNKNOWN = 0, + R32G32B32A32_TYPELESS = 1, + R32G32B32A32_FLOAT = 2, + R32G32B32A32_UINT = 3, + R32G32B32A32_SINT = 4, + R32G32B32_TYPELESS = 5, + R32G32B32_FLOAT = 6, + R32G32B32_UINT = 7, + R32G32B32_SINT = 8, + R16G16B16A16_TYPELESS = 9, + R16G16B16A16_FLOAT = 10, + R16G16B16A16_UNORM = 11, + R16G16B16A16_UINT = 12, + R16G16B16A16_SNORM = 13, + R16G16B16A16_SINT = 14, + R32G32_TYPELESS = 15, + R32G32_FLOAT = 16, + R32G32_UINT = 17, + R32G32_SINT = 18, + R32G8X24_TYPELESS = 19, + D32_FLOAT_S8X24_UINT = 20, + R32_FLOAT_X8X24_TYPELESS = 21, + X32_TYPELESS_G8X24_UINT = 22, + R10G10B10A2_TYPELESS = 23, + R10G10B10A2_UNORM = 24, + R10G10B10A2_UINT = 25, + R11G11B10_FLOAT = 26, + R8G8B8A8_TYPELESS = 27, + R8G8B8A8_UNORM = 28, + R8G8B8A8_UNORM_SRGB = 29, + R8G8B8A8_UINT = 30, + R8G8B8A8_SNORM = 31, + R8G8B8A8_SINT = 32, + R16G16_TYPELESS = 33, + R16G16_FLOAT = 34, + R16G16_UNORM = 35, + R16G16_UINT = 36, + R16G16_SNORM = 37, + R16G16_SINT = 38, + R32_TYPELESS = 39, + D32_FLOAT = 40, + R32_FLOAT = 41, + R32_UINT = 42, + R32_SINT = 43, + R24G8_TYPELESS = 44, + D24_UNORM_S8_UINT = 45, + R24_UNORM_X8_TYPELESS = 46, + X24_TYPELESS_G8_UINT = 47, + R8G8_TYPELESS = 48, + R8G8_UNORM = 49, + R8G8_UINT = 50, + R8G8_SNORM = 51, + R8G8_SINT = 52, + R16_TYPELESS = 53, + R16_FLOAT = 54, + D16_UNORM = 55, + R16_UNORM = 56, + R16_UINT = 57, + R16_SNORM = 58, + R16_SINT = 59, + R8_TYPELESS = 60, + R8_UNORM = 61, + R8_UINT = 62, + R8_SNORM = 63, + R8_SINT = 64, + A8_UNORM = 65, + R1_UNORM = 66, + R9G9B9E5_SHAREDEXP = 67, + R8G8_B8G8_UNORM = 68, + G8R8_G8B8_UNORM = 69, + BC1_TYPELESS = 70, + BC1_UNORM = 71, + BC1_UNORM_SRGB = 72, + BC2_TYPELESS = 73, + BC2_UNORM = 74, + BC2_UNORM_SRGB = 75, + BC3_TYPELESS = 76, + BC3_UNORM = 77, + BC3_UNORM_SRGB = 78, + BC4_TYPELESS = 79, + BC4_UNORM = 80, + BC4_SNORM = 81, + BC5_TYPELESS = 82, + BC5_UNORM = 83, + BC5_SNORM = 84, + B5G6R5_UNORM = 85, + B5G5R5A1_UNORM = 86, + B8G8R8A8_UNORM = 87, + B8G8R8X8_UNORM = 88, + R10G10B10_XR_BIAS_A2_UNORM = 89, + B8G8R8A8_TYPELESS = 90, + B8G8R8A8_UNORM_SRGB = 91, + B8G8R8X8_TYPELESS = 92, + B8G8R8X8_UNORM_SRGB = 93, + BC6H_TYPELESS = 94, + BC6H_UF16 = 95, + BC6H_SF16 = 96, + BC7_TYPELESS = 97, + BC7_UNORM = 98, + BC7_UNORM_SRGB = 99, + AYUV = 100, + Y410 = 101, + Y416 = 102, + NV12 = 103, + P010 = 104, + P016 = 105, + OPAQUE = 106, + YUY2 = 107, + Y210 = 108, + Y216 = 109, + NV11 = 110, + AI44 = 111, + IA44 = 112, + P8 = 113, + A8P8 = 114, + B4G4R4A4_UNORM = 115, + P208 = 130, + V208 = 131, + V408 = 132, + SAMPLER_FEEDBACK_MIN_MIP_OPAQUE, + SAMPLER_FEEDBACK_MIP_REGION_USED_OPAQUE, +} + +#[derive(Clone, Copy, Debug)] pub struct Dx10Header { /// Resource data formats, including fully-typed and typeless formats. /// See https://learn.microsoft.com/en-us/windows/win32/api/dxgiformat/ne-dxgiformat-dxgi_format - dxgi_format: u32, - resource_dimension: D3D10ResourceDimension, - misc_flag: DdsResourceMiscFlags, - array_size: u32, - misc_flags2: u32, + pub dxgi_format: DXGIFormat, + pub resource_dimension: D3D10ResourceDimension, + pub misc_flag: DdsResourceMiscFlags, + pub array_size: usize, + pub misc_flags2: u32, } +impl Dx10Header { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + let dxgi_format = r + .read_u32() + .map(|val| DXGIFormat::from_u32(val).unwrap_or(DXGIFormat::UNKNOWN))?; + let resource_dimension = r.read_u32().map(|val| { + D3D10ResourceDimension::from_u32(val).unwrap_or(D3D10ResourceDimension::Unknown) + })?; + let misc_flag = r.read_u32().map(flags_from_bits)?; + let array_size = r.read_u32()? as usize; + let misc_flags2 = r.read_u32()?; + + Ok(Self { + dxgi_format, + resource_dimension, + misc_flag, + array_size, + misc_flags2, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + w.write_u32( + self.dxgi_format + .to_u32() + .ok_or_eyre("DXGIFormat should fit in a u32")?, + )?; + w.write_u32( + self.resource_dimension + .to_u32() + .ok_or_eyre("DXGIFormat should fit in a u32")?, + )?; + w.write_u32(self.misc_flag.bits())?; + w.write_u32(self.array_size as u32)?; + w.write_u32(self.misc_flags2)?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] pub struct DDSPixelFormat { - /// Structure size. Must be `32`. - size: u32, - flags: DDPF, - four_cc: u32, - rgb_bit_count: u32, - r_bit_mask: u32, - g_bit_mask: u32, - b_bit_mask: u32, - a_bit_mask: u32, + pub flags: DDPF, + pub four_cc: u32, + pub rgb_bit_count: u32, + pub r_bit_mask: u32, + pub g_bit_mask: u32, + pub b_bit_mask: u32, + pub a_bit_mask: u32, } +impl DDSPixelFormat { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + let size = r.read_u32()? as usize; + eyre::ensure!( + size == 32, + "Invalid structure size. Got 0X{:0X}, expected 0x20", + size + ); + + let flags = r.read_u32().map(flags_from_bits)?; + let four_cc = r.read_u32()?; + let rgb_bit_count = r.read_u32()?; + let r_bit_mask = r.read_u32()?; + let g_bit_mask = r.read_u32()?; + let b_bit_mask = r.read_u32()?; + let a_bit_mask = r.read_u32()?; + + Ok(Self { + flags, + four_cc, + rgb_bit_count, + r_bit_mask, + g_bit_mask, + b_bit_mask, + a_bit_mask, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + // Structure size + w.write_u32(32)?; + + w.write_u32(self.flags.bits())?; + w.write_u32(self.four_cc)?; + w.write_u32(self.rgb_bit_count)?; + w.write_u32(self.r_bit_mask)?; + w.write_u32(self.g_bit_mask)?; + w.write_u32(self.b_bit_mask)?; + w.write_u32(self.a_bit_mask)?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] pub struct DDSHeader { - /// Size of this structure. Must be `124`. - size: u32, /// Flags to indicate which members contain valid data. - flags: DDSD, - height: u32, - width: u32, - pitch_or_linear_size: u32, - depth: u32, - mipmap_count: u32, - reserved_1: [u8; 11], - pixel_format: DDSPixelFormat, - caps: DDSCAPS, - caps_2: DDSCAPS2, - reserved_2: [u8; 3], + pub flags: DDSD, + pub height: usize, + pub width: usize, + pub pitch_or_linear_size: usize, + pub depth: usize, + pub mipmap_count: usize, + pub pixel_format: DDSPixelFormat, + pub caps: DDSCAPS, + pub caps_2: DDSCAPS2, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +impl DDSHeader { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + r.skip_u32(MAGIC_DDS).wrap_err("Invalid magic bytes")?; + + let size = r.read_u32()?; + eyre::ensure!( + size == 124, + "Invalid structure size. Got 0x{:0X}, expected 0x7C", + size + ); + + let flags = r.read_u32().map(flags_from_bits)?; + let height = r.read_u32()? as usize; + let width = r.read_u32()? as usize; + let pitch_or_linear_size = r.read_u32()? as usize; + let depth = r.read_u32()? as usize; + let mipmap_count = r.read_u32()? as usize; + + // Skip reserved bytes + r.seek(SeekFrom::Current(11 * 4))?; + + let pixel_format = DDSPixelFormat::from_binary(&mut r)?; + let caps = r.read_u32().map(flags_from_bits)?; + let caps_2 = r.read_u32().map(flags_from_bits)?; + + // Skip unused and reserved bytes + r.seek(SeekFrom::Current(3 * 4))?; + + Ok(Self { + flags, + height, + width, + pitch_or_linear_size, + depth, + mipmap_count, + pixel_format, + caps, + caps_2, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + w.write_u32(MAGIC_DDS)?; + + // Structure size in bytes + w.write_u32(124)?; + w.write_u32(self.flags.bits())?; + w.write_u32(self.height as u32)?; + w.write_u32(self.width as u32)?; + w.write_u32(self.pitch_or_linear_size as u32)?; + w.write_u32(self.depth as u32)?; + w.write_u32(self.mipmap_count as u32)?; + + w.write_all(&[0u8; 11 * 4])?; + + self.pixel_format.to_binary(&mut w)?; + w.write_u32(self.caps.bits())?; + w.write_u32(self.caps_2.bits())?; + + w.write_all(&[0u8; 3 * 4])?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] pub enum ImageType { Image2D = 0, Image3D = 1, @@ -125,12 +434,14 @@ pub enum ImageType { /// A stripped version of `ImageType` that only contains just the data needed /// to read a DDS image stream. +#[allow(dead_code)] +#[derive(Clone, Copy, Debug)] pub struct StrippedImageFormat { pub image_type: ImageType, - pub width: u32, - pub height: u32, - pub layers: u32, - pub mip_levels: u32, + pub width: usize, + pub height: usize, + pub layers: usize, + pub mip_levels: usize, } // This is a stripped down version of the logic that the engine implements to fill @@ -172,7 +483,10 @@ pub fn stripped_format_from_header( } if dx10_header.resource_dimension == D3D10ResourceDimension::Texture2D { - if dx10_header.misc_flag == DdsResourceMiscFlags::TextureCube { + if dx10_header + .misc_flag + .contains(DdsResourceMiscFlags::TEXTURECUBE) + { image_format.image_type = ImageType::ImageCube; if dx10_header.array_size > 1 { image_format.layers = dx10_header.array_size; diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index a24b3bd..bf10826 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(cursor_split)] #![feature(test)] mod binary; From c137cfb90d5fd9702a6ad5cf7adb4947aea8eb85 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 10:43:42 +0200 Subject: [PATCH 07/35] Implement more texture formats The second compression method found in the game's code seems to be Zlib, but it doesn't seem to be used in the game files. What does get used is a compression type of `0`, which appears to be uncompressed data. For DDS formats, all the ones that are currently used by in the game files can be emitted as is. Though for some of them, other tools might not be able to display them. --- Cargo.lock | 13 + Cargo.toml | 1 + crates/dtmt/src/cmd/bundle/extract.rs | 13 +- lib/sdk/Cargo.toml | 1 + lib/sdk/src/binary.rs | 20 ++ lib/sdk/src/filetype/texture.rs | 456 +++++++++++++++++--------- lib/sdk/src/filetype/texture/dds.rs | 82 +++-- 7 files changed, 395 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6080763..d36f510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1123,6 +1123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", + "libz-sys", "miniz_oxide 0.8.0", ] @@ -2187,6 +2188,17 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libz-sys" +version = "1.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -3341,6 +3353,7 @@ dependencies = [ "color-eyre", "csv-async", "fastrand", + "flate2", "futures", "futures-util", "glob", diff --git a/Cargo.toml b/Cargo.toml index 8c104d7..68751f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ druid = { version = "0.8", features = ["im", "serde", "image", "png", "jpeg", "b druid-widget-nursery = "0.1" dtmt-shared = { path = "lib/dtmt-shared" } fastrand = "2.1.0" +flate2 = { version = "1.0.30", features = ["zlib"] } futures = "0.3.25" futures-util = "0.3.24" glob = "0.3.0" diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index b595dba..3790181 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -275,7 +275,13 @@ struct ExtractOptions<'a> { #[tracing::instrument( skip(ctx, options), - fields(decompile = options.decompile, flatten = options.flatten, dry_run = options.dry_run) + fields( + bundle_name = tracing::field::Empty, + bundle_hash = tracing::field::Empty, + decompile = options.decompile, + flatten = options.flatten, + dry_run = options.dry_run, + ) )] async fn extract_bundle( ctx: Arc, @@ -318,6 +324,11 @@ where let bundle = { let data = fs::read(path.as_ref()).await?; let name = Bundle::get_name_from_path(&ctx, path.as_ref()); + { + let span = tracing::span::Span::current(); + span.record("bundle_hash", format!("{:X}", name)); + span.record("bundle_name", name.display().to_string()); + } Bundle::from_binary(&ctx, name, data)? }; diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 0cd0e4e..9abbb23 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -10,6 +10,7 @@ byteorder = { workspace = true } color-eyre = { workspace = true } csv-async = { workspace = true } fastrand = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } glob = { workspace = true } diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 83ccca0..40f9e9a 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -42,6 +42,26 @@ impl FromBinary for Vec { } } +pub fn flags_from_bits(bits: T::Bits) -> T +where + ::Bits: std::fmt::Binary, +{ + if let Some(flags) = T::from_bits(bits) { + flags + } else { + let unknown = bits & !T::all().bits(); + + tracing::warn!( + "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", + std::any::type_name::(), + T::all().bits(), + unknown + ); + + T::from_bits_truncate(bits) + } +} + pub mod sync { use std::ffi::CStr; use std::io::{self, Read, Seek, SeekFrom, Write}; diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 1565b8c..88314ea 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -5,16 +5,15 @@ use bitflags::bitflags; use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; -use num_traits::ToPrimitive as _; +use flate2::read::ZlibDecoder; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; -use crate::filetype::texture::dds::{DXGIFormat, ImageType}; use crate::murmur::{HashGroup, IdString32, IdString64}; -use crate::{BundleFile, BundleFileType, BundleFileVariant}; +use crate::{binary, BundleFile, BundleFileType, BundleFileVariant}; mod dds; @@ -43,7 +42,7 @@ struct TextureDefinitionOutput { } bitflags! { - #[derive(Clone, Copy, Debug)] + #[derive(Clone, Copy, Debug, Default)] struct TextureFlags: u32 { const STREAMABLE = 0b0000_0001; const UNKNOWN = 1 << 1; @@ -57,7 +56,7 @@ struct TextureHeaderMipInfo { size: usize, } -#[derive(Clone, Debug)] +#[derive(Clone, Default)] struct TextureHeader { flags: TextureFlags, n_streamable_mipmaps: usize, @@ -67,13 +66,33 @@ struct TextureHeader { meta_size: usize, } +impl std::fmt::Debug for TextureHeader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextureHeader") + .field("flags", &self.flags) + .field("n_streamable_mipmaps", &self.n_streamable_mipmaps) + .field("width", &self.width) + .field("height", &self.height) + .field("mip_infos", &{ + let mut s = self + .mip_infos + .iter() + .fold(String::from("["), |mut s, info| { + s.push_str(&format!("{}/{}, ", info.offset, info.size)); + s + }); + s.push(']'); + s + }) + .field("meta_size", &self.meta_size) + .finish() + } +} + impl TextureHeader { #[tracing::instrument(skip(r))] fn from_binary(mut r: impl ReadExt) -> Result { - let flags = r.read_u32().and_then(|bits| { - TextureFlags::from_bits(bits) - .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) - })?; + let flags = r.read_u32().map(binary::flags_from_bits)?; let n_streamable_mipmaps = r.read_u32()? as usize; let width = r.read_u32()? as usize; let height = r.read_u32()? as usize; @@ -121,7 +140,7 @@ impl TextureHeader { } } -#[derive(Clone, Debug)] +#[derive(Clone)] struct Texture { header: TextureHeader, data: Vec, @@ -129,6 +148,37 @@ struct Texture { category: IdString32, } +impl std::fmt::Debug for Texture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut out = f.debug_struct("Texture"); + out.field("header", &self.header); + + if self.data.len() <= 5 { + out.field("data", &format!("{:x?}", &self.data)); + } else { + out.field( + "data", + &format!("{:x?}.. ({} bytes)", &self.data[..5], &self.data.len()), + ); + } + + if let Some(stream) = self.stream.as_ref() { + if stream.len() <= 5 { + out.field("stream", &format!("{:x?}", &stream)); + } else { + out.field( + "stream", + &format!("{:x?}.. ({} bytes)", &stream[..5], &stream.len()), + ); + } + } else { + out.field("stream", &"None"); + } + + out.field("category", &self.category).finish() + } +} + impl Texture { #[tracing::instrument(skip(data, chunks))] fn decompress_stream_data(mut data: impl Read, chunks: impl AsRef<[usize]>) -> Result> { @@ -214,35 +264,83 @@ impl Texture { Ok(out) } - #[tracing::instrument(skip(ctx, r, stream_r))] + #[tracing::instrument( + "Texture::from_binary", + skip(ctx, r, stream_r), + fields( + compression_type = tracing::field::Empty, + compressed_size = tracing::field::Empty, + uncompressed_size = tracing::field::Empty, + ) + )] fn from_binary( ctx: &crate::Context, mut r: impl Read + Seek, mut stream_r: Option, ) -> Result { - // Looking at the executable in IDA, there is one other valid value: `2`. - // If this ever comes up in the game data, I'll have to reverse engineer the - // (de)compression algorithm through IDA. let compression_type = r.read_u32()?; - eyre::ensure!( - compression_type == 1, - "Unknown compression type for texture '{}'", - compression_type - ); - let compressed_size = r.read_u32()? as usize; let uncompressed_size = r.read_u32()? as usize; - let out_buf = { - let mut comp_buf = vec![0; compressed_size]; - r.read_exact(&mut comp_buf)?; + { + let span = tracing::Span::current(); + span.record("compression_type", compression_type); + span.record("compressed_size", compressed_size); + span.record("uncompressed_size", uncompressed_size); + } - oodle::decompress( + let mut comp_buf = vec![0; compressed_size]; + r.read_exact(&mut comp_buf)?; + + let out_buf = match compression_type { + // Uncompressed + // This one never seems to contain the additional `TextureHeader` metadata, + // so we return early in this branch. + 0 => { + eyre::ensure!( + compressed_size == 0 && uncompressed_size == 0, + "Cannot handle texture with compression_type == 0, but buffer sizes > 0" + ); + tracing::trace!("Found raw texture"); + + let pos = r.stream_position()?; + let end = { + r.seek(SeekFrom::End(0))?; + let end = r.stream_position()?; + r.seek(SeekFrom::Start(pos))?; + end + }; + + // Reads until the last u32. + let mut data = vec![0u8; (end - pos - 4) as usize]; + r.read_exact(&mut data)?; + + let category = r.read_u32().map(IdString32::from)?; + + return Ok(Self { + header: TextureHeader::default(), + data, + stream: None, + category, + }); + } + 1 => oodle::decompress( comp_buf, uncompressed_size, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, - )? + )?, + 2 => { + let mut decoder = ZlibDecoder::new(comp_buf.as_slice()); + let mut buf = Vec::with_capacity(uncompressed_size); + + decoder.read_to_end(&mut buf)?; + buf + } + _ => eyre::bail!( + "Unknown compression type for texture '{}'", + compression_type + ), }; eyre::ensure!( @@ -339,6 +437,129 @@ impl Texture { serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") } + #[tracing::instrument(fields( + dds_header = tracing::field::Empty, + dx10_header = tracing::field::Empty, + image_format = tracing::field::Empty, + ))] + fn create_dds_user_file(&self, name: String) -> Result { + let mut data = Cursor::new(&self.data); + let mut dds_header = + dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; + + { + let span = tracing::Span::current(); + span.record("dds_header", format!("{:?}", dds_header)); + } + + if !dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) { + tracing::debug!("Found DDS without FourCC. Dumping raw data"); + return Ok(UserFile::with_name(self.data.clone(), name)); + } + + // eyre::ensure!( + // dds_header.pixel_format.four_cc == dds::FourCC::DX10, + // "Only DX10 textures are currently supported. FourCC == {}", + // dds_header.pixel_format.four_cc, + // ); + + let dx10_header = + dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; + + { + let span = tracing::Span::current(); + span.record("dx10_header", format!("{:?}", dx10_header)); + } + + // match dx10_header.dxgi_format { + // DXGIFormat::BC1_UNORM + // | DXGIFormat::BC3_UNORM + // | DXGIFormat::BC4_UNORM + // | DXGIFormat::BC5_UNORM + // | DXGIFormat::BC6H_UF16 + // | DXGIFormat::BC7_UNORM => {} + // _ => { + // eyre::bail!( + // "Unsupported DXGI format: {} (0x{:0X})", + // dx10_header.dxgi_format, + // dx10_header.dxgi_format.to_u32().unwrap_or_default() + // ); + // } + // } + + let stingray_image_format = dds::stripped_format_from_header(&dds_header, &dx10_header)?; + { + let span = tracing::Span::current(); + span.record("image_format", format!("{:?}", stingray_image_format)); + } + + // eyre::ensure!( + // stingray_image_format.image_type == ImageType::Image2D, + // "Unsupported image type: {}", + // stingray_image_format.image_type, + // ); + + let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; + let bits_per_block: usize = match block_size { + 8 => 128, + 16 => 64, + block_size => eyre::bail!("Unsupported block size {}", block_size), + }; + + let pitch = self.header.width / 4 * block_size; + let bytes_per_block = self.header.width / bits_per_block / 4; + + tracing::debug!( + "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", + block_size, + pitch, + bits_per_block, + bytes_per_block + ); + + let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); + + // Currently, we only extract the largest mipmap, + // so we need to set the dimensions accordingly, and remove the + // flag. + dds_header.width = self.header.width; + dds_header.height = self.header.height; + dds_header.mipmap_count = 0; + dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; + + dds_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DDS header")?; + + dx10_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DX10 header")?; + + // If there is stream data, we build the mipmap data from it. + // If not, we take whatever is left in the bundle file. + if let Some(stream) = &self.stream { + let data = Self::reorder_stream_mipmap( + stream, + bits_per_block, + bytes_per_block, + block_size, + pitch, + ) + .wrap_err("Failed to reorder stream chunks")?; + + out_data + .write_all(&data) + .wrap_err("Failed to write streamed mipmap data")?; + } else { + let (_, remaining) = data.split(); + out_data + .write_all(remaining) + .wrap_err("Failed to write texture data")?; + }; + + Ok(UserFile::with_name(out_data.into_inner(), name)) + } + #[tracing::instrument(skip(self))] fn to_user_files(&self, name: String) -> Result> { let mut files = Vec::with_capacity(2); @@ -369,122 +590,38 @@ impl Texture { files.push(UserFile::with_name(self.data.clone(), name)); } + match self + .create_dds_user_file(name) + .wrap_err("Failed to create DDS file") { - let mut data = Cursor::new(&self.data); - let mut dds_header = - dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; - - eyre::ensure!( - dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) - && dds_header.pixel_format.four_cc == dds::FOURCC_DX10, - "Only DX10 textures are currently supported." - ); - - let dx10_header = - dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; - - match dx10_header.dxgi_format { - DXGIFormat::BC1_UNORM - | DXGIFormat::BC3_UNORM - | DXGIFormat::BC4_UNORM - | DXGIFormat::BC5_UNORM - | DXGIFormat::BC6H_UF16 - | DXGIFormat::BC7_UNORM => {} - _ => { - eyre::bail!( - "Unsupported DXGI format: {} (0x{:0X})", - dx10_header.dxgi_format, - dx10_header.dxgi_format.to_u32().unwrap_or_default() + Ok(dds) => files.push(dds), + Err(err) => { + if cfg!(debug_assertions) { + tracing::error!( + "{:?}", + err.with_section(|| { + "Running in debug mode, continuing to produce raw files".header("Note:") + }) ); + } else { + return Err(err); } } - - let stingray_image_format = - dds::stripped_format_from_header(&dds_header, &dx10_header)?; - eyre::ensure!( - stingray_image_format.image_type == ImageType::Image2D, - "Unsupported image type: {}", - stingray_image_format.image_type, - ); - - let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; - let bits_per_block: usize = match block_size { - 8 => 128, - 16 => 64, - block_size => eyre::bail!("Unsupported block size {}", block_size), - }; - - let pitch = self.header.width / 4 * block_size; - let bytes_per_block = self.header.width / bits_per_block / 4; - - tracing::debug!( - "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", - block_size, - pitch, - bits_per_block, - bytes_per_block - ); - - let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); - - // Currently, we only extract the largest mipmap, - // so we need to set the dimensions accordingly, and remove the - // flag. - dds_header.width = self.header.width; - dds_header.height = self.header.height; - dds_header.mipmap_count = 0; - dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; - - dds_header - .to_binary(&mut out_data) - .wrap_err("Failed to write DDS header")?; - - dx10_header - .to_binary(&mut out_data) - .wrap_err("Failed to write DX10 header")?; - - if let Some(stream) = &self.stream { - let data = Self::reorder_stream_mipmap( - stream, - bits_per_block, - bytes_per_block, - block_size, - pitch, - ) - .wrap_err("Failed to reorder stream chunks")?; - - out_data - .write_all(&data) - .wrap_err("Failed to write streamed mipmap data")?; - } else { - out_data - .write_all(data.split().1) - .wrap_err("Failed to write texture data")?; - }; - - files.push(UserFile::with_name(out_data.into_inner(), name)); - } + }; Ok(files) } } -#[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] +#[tracing::instrument(skip(ctx, data, stream_data), fields(data_len = data.as_ref().len()))] pub(crate) async fn decompile_data( ctx: &crate::Context, name: String, data: impl AsRef<[u8]>, - stream_file_name: Option, + stream_data: Option>, ) -> Result> { - let mut r = Cursor::new(data.as_ref()); - let mut stream_r = if let Some(file_name) = stream_file_name { - let stream_data = fs::read(&file_name) - .await - .wrap_err_with(|| format!("Failed to read stream file '{}'", file_name.display()))?; - Some(Cursor::new(stream_data)) - } else { - None - }; + let mut r = Cursor::new(data); + let mut stream_r = stream_data.map(Cursor::new); let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; texture @@ -498,38 +635,47 @@ pub(crate) async fn decompile( name: String, variant: &BundleFileVariant, ) -> Result> { - if !variant.external() { + let data_file = variant.data_file_name().map(|name| match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(name), + None => PathBuf::from("bundle").join(name), + }); + + if variant.external() { + let Some(path) = data_file else { + eyre::bail!("File is marked external but has no data file name"); + }; + + tracing::debug!( + "Decompiling texture from external file '{}'", + path.display() + ); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + + decompile_data(ctx, name, data, None::<&[u8]>).await + } else { tracing::debug!("Decompiling texture from bundle data"); - let stream_file_name = variant.data_file_name().map(|name| match &ctx.game_dir { - Some(dir) => dir.join("bundle").join(name), - None => PathBuf::from("bundle").join(name), - }); + let stream_data = match data_file { + Some(path) => { + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + Some(data) + } + None => None, + }; - return decompile_data(ctx, name, variant.data(), stream_file_name).await; + decompile_data(ctx, name, variant.data(), stream_data).await } - - let Some(file_name) = variant.data_file_name() else { - eyre::bail!("Texture file has no data and no data file"); - }; - - tracing::debug!("Decompiling texture from external file '{}'", file_name); - - let path = match &ctx.game_dir { - Some(dir) => dir.join("bundle").join(file_name), - None => PathBuf::from("bundle").join(file_name), - }; - - tracing::trace!(path = %path.display()); - - let data = fs::read(&path) - .await - .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) - .with_suggestion(|| { - "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." - })?; - - decompile_data(ctx, name, &data, None).await } #[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs index 5ef1b90..ca0b73f 100644 --- a/lib/sdk/src/filetype/texture/dds.rs +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -7,10 +7,10 @@ use color_eyre::Result; use num_derive::{FromPrimitive, ToPrimitive}; use num_traits::{FromPrimitive as _, ToPrimitive as _}; +use crate::binary; use crate::binary::sync::{ReadExt, WriteExt}; const MAGIC_DDS: u32 = 0x20534444; -pub const FOURCC_DX10: u32 = 0x30315844; bitflags! { #[derive(Clone, Copy, Debug)] @@ -75,27 +75,8 @@ bitflags! { } } -fn flags_from_bits(bits: T::Bits) -> T -where - ::Bits: std::fmt::Binary, -{ - if let Some(flags) = T::from_bits(bits) { - flags - } else { - let unknown = bits & !T::all().bits(); - - tracing::warn!( - "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", - std::any::type_name::(), - T::all().bits(), - unknown - ); - - T::from_bits_truncate(bits) - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, FromPrimitive, ToPrimitive)] +#[repr(u32)] pub enum D3D10ResourceDimension { Unknown = 0, Buffer = 1, @@ -107,6 +88,7 @@ pub enum D3D10ResourceDimension { #[allow(clippy::upper_case_acronyms)] #[allow(non_camel_case_types)] #[derive(Clone, Copy, Debug, strum::Display, FromPrimitive, ToPrimitive)] +#[repr(u32)] pub enum DXGIFormat { UNKNOWN = 0, R32G32B32A32_TYPELESS = 1, @@ -243,7 +225,7 @@ pub struct Dx10Header { } impl Dx10Header { - #[tracing::instrument(skip(r))] + #[tracing::instrument("Dx10Header::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { let dxgi_format = r .read_u32() @@ -251,7 +233,7 @@ impl Dx10Header { let resource_dimension = r.read_u32().map(|val| { D3D10ResourceDimension::from_u32(val).unwrap_or(D3D10ResourceDimension::Unknown) })?; - let misc_flag = r.read_u32().map(flags_from_bits)?; + let misc_flag = r.read_u32().map(binary::flags_from_bits)?; let array_size = r.read_u32()? as usize; let misc_flags2 = r.read_u32()?; @@ -264,7 +246,7 @@ impl Dx10Header { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("Dx10Header::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { w.write_u32( self.dxgi_format @@ -284,10 +266,30 @@ impl Dx10Header { } } +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum FourCC { + Empty = u32::MAX, + DXT1 = 0x31545844, + DXT2 = 0x33545844, + DXT5 = 0x35545844, + AXI1 = 0x31495441, + AXI2 = 0x32495441, + DX10 = 0x30315844, + D3D_A16B16G16R16 = 0x24, + D3D_R16F = 0x6F, + D3D_G16R16F = 0x70, + D3D_A16B16G16R16F = 0x71, + D3D_R32F = 0x72, + D3D_G32R32F = 0x73, + D3D_A32B32G32R32F = 0x74, +} + #[derive(Clone, Copy, Debug)] pub struct DDSPixelFormat { pub flags: DDPF, - pub four_cc: u32, + pub four_cc: FourCC, pub rgb_bit_count: u32, pub r_bit_mask: u32, pub g_bit_mask: u32, @@ -296,7 +298,7 @@ pub struct DDSPixelFormat { } impl DDSPixelFormat { - #[tracing::instrument(skip(r))] + #[tracing::instrument("DDSPixelFormat::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { let size = r.read_u32()? as usize; eyre::ensure!( @@ -305,8 +307,17 @@ impl DDSPixelFormat { size ); - let flags = r.read_u32().map(flags_from_bits)?; - let four_cc = r.read_u32()?; + let flags: DDPF = r.read_u32().map(binary::flags_from_bits)?; + + let four_cc = if flags.contains(DDPF::FOURCC) { + r.read_u32().and_then(|bytes| { + FourCC::from_u32(bytes).ok_or_eyre(format!("Unknown FourCC value: {:08X}", bytes)) + })? + } else { + r.skip_u32(0)?; + FourCC::Empty + }; + let rgb_bit_count = r.read_u32()?; let r_bit_mask = r.read_u32()?; let g_bit_mask = r.read_u32()?; @@ -324,13 +335,13 @@ impl DDSPixelFormat { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("DDSPixelFormat::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { // Structure size w.write_u32(32)?; w.write_u32(self.flags.bits())?; - w.write_u32(self.four_cc)?; + w.write_u32(self.four_cc.to_u32().unwrap_or_default())?; w.write_u32(self.rgb_bit_count)?; w.write_u32(self.r_bit_mask)?; w.write_u32(self.g_bit_mask)?; @@ -356,7 +367,7 @@ pub struct DDSHeader { } impl DDSHeader { - #[tracing::instrument(skip(r))] + #[tracing::instrument("DDSHeader::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { r.skip_u32(MAGIC_DDS).wrap_err("Invalid magic bytes")?; @@ -367,7 +378,7 @@ impl DDSHeader { size ); - let flags = r.read_u32().map(flags_from_bits)?; + let flags = r.read_u32().map(binary::flags_from_bits)?; let height = r.read_u32()? as usize; let width = r.read_u32()? as usize; let pitch_or_linear_size = r.read_u32()? as usize; @@ -378,8 +389,8 @@ impl DDSHeader { r.seek(SeekFrom::Current(11 * 4))?; let pixel_format = DDSPixelFormat::from_binary(&mut r)?; - let caps = r.read_u32().map(flags_from_bits)?; - let caps_2 = r.read_u32().map(flags_from_bits)?; + let caps = r.read_u32().map(binary::flags_from_bits)?; + let caps_2 = r.read_u32().map(binary::flags_from_bits)?; // Skip unused and reserved bytes r.seek(SeekFrom::Current(3 * 4))?; @@ -397,7 +408,7 @@ impl DDSHeader { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("DDSHeader::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { w.write_u32(MAGIC_DDS)?; @@ -423,6 +434,7 @@ impl DDSHeader { } #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +#[repr(u32)] pub enum ImageType { Image2D = 0, Image3D = 1, From 51055040c052f3c1d1455d175d192e5cf751c002 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 15:00:58 +0200 Subject: [PATCH 08/35] Refactor code for file injection --- crates/dtmt/src/cmd/bundle/inject.rs | 295 ++++++++++++++++++++------- lib/sdk/src/bundle/file.rs | 7 +- 2 files changed, 221 insertions(+), 81 deletions(-) diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 4e5cbac..3d04fd6 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -1,126 +1,267 @@ use std::path::PathBuf; use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::{self, Context, Result}; +use color_eyre::eyre::{self, Context, OptionExt as _, Result}; use color_eyre::Help; +use path_slash::PathBufExt as _; +use sdk::filetype::texture; use sdk::murmur::IdString64; use sdk::Bundle; -use tokio::fs::{self, File}; -use tokio::io::AsyncReadExt; +use tokio::fs; pub(crate) fn command_definition() -> Command { Command::new("inject") + .subcommand_required(true) .about("Inject a file into a bundle.\n\ Raw binary data can be used to directly replace the file's variant data blob without affecting the metadata.\n\ Alternatively, a compiler format may be specified, and a complete bundle file is created.") - .arg( - Arg::new("replace") - .help("The name of a file in the bundle whose content should be replaced.") - .short('r') - .long("replace"), - ) - .arg( - Arg::new("compile") - .help("Compile the file with the given data format before injecting.") - .long("compile") - .short('c') - ) .arg( Arg::new("output") .help( "The path to write the changed bundle to. \ - If omitted, the input bundle will be overwritten.", + If omitted, the input bundle will be overwritten.\n\ + Remember to add a `.patch_` suffix if you also use '--patch'.", ) .short('o') .long("output") .value_parser(value_parser!(PathBuf)), ) .arg( - Arg::new("bundle") - .help("Path to the bundle to inject the file into.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("patch") + .help("Create a patch bundle. Optionally, a patch NUMBER may be specified as \ + '--patch=123'.\nThe maximum number is 1.") + .short('p') + .long("patch") + .num_args(0..=1) + .require_equals(true) + .default_missing_value("1") + .value_name("NUMBER") + .value_parser(value_parser!(u16)) ) - .arg( - Arg::new("file") - .help("Path to the file to inject.") - .required(true) - .value_parser(value_parser!(PathBuf)), + .subcommand( + Command::new("replace") + .about("Replace an existing file in the bundle") + .arg( + Arg::new("compile") + .help("Compile the file as TYPE before injecting. \ + Metadata will also be overwritten") + .long("compile") + .short('c') + .value_name("TYPE") + .value_parser(["texture"]) + ) + .arg( + Arg::new("variant") + .help("Sepcify the variant index to replace.") + .long("variant") + .default_value("0") + .value_parser(value_parser!(u8)) + ) + .arg( + Arg::new("new-file") + .help("Path to the file to inject.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ) + .arg( + Arg::new("bundle") + .help("Path to the bundle to inject the file into.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ) + .arg( + Arg::new("bundle-file") + .help("The name of a file in the bundle whose content should be replaced.") + .required(true), + ), ) + // .subcommand( + // Command::new("add") + // .about("Add a new file to the bundle") + // .arg( + // Arg::new("new-file") + // .help("Path to the file to inject.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ) + // .arg( + // Arg::new("bundle") + // .help("Path to the bundle to inject the file into.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ), + // ) } -#[tracing::instrument(skip_all)] +#[tracing::instrument( + skip_all, + fields( + bundle_path = tracing::field::Empty, + in_file_path = tracing::field::Empty, + compile = tracing::field::Empty, + output_path = tracing::field::Empty, + file_name = tracing::field::Empty, + ) +)] pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { - let bundle_path = matches + let Some((op, sub_matches)) = matches.subcommand() else { + unreachable!("clap is configured to require a subcommand, and they're all handled above"); + }; + + let compile = matches.get_one::("compile"); + + let bundle_path = sub_matches .get_one::("bundle") .expect("required parameter not found"); - let file_path = matches + let in_file_path = sub_matches .get_one::("file") .expect("required parameter not found"); - tracing::trace!(bundle_path = %bundle_path.display(), file_path = %file_path.display()); + let patch_number = matches + .get_one::("patch") + .map(|num| format!("{:03}", num)); - let mut bundle = { - let binary = fs::read(bundle_path).await?; - let name = Bundle::get_name_from_path(&ctx, bundle_path); - Bundle::from_binary(&ctx, name, binary).wrap_err("Failed to open bundle file")? - }; + let output_path = matches + .get_one::("output") + .cloned() + .unwrap_or_else(|| { + let mut output_path = bundle_path.clone(); - let name = match matches.get_one::("replace") { - Some(name) => match u64::from_str_radix(name, 16) { - Ok(id) => IdString64::from(id), - Err(_) => IdString64::String(name.clone()), - }, - None => eyre::bail!("Currently, only the '--replace' operation is supported."), - }; + if let Some(patch_number) = patch_number.as_ref() { + output_path.set_extension(format!("patch_{:03}", patch_number)); + } - let mut file = File::open(&file_path) - .await - .wrap_err_with(|| format!("Failed to open '{}'", file_path.display()))?; + output_path + }); - if let Some(variant) = bundle - .files_mut() - .filter(|file| file.matches_name(name.clone())) - // TODO: Handle file variants - .find_map(|file| file.variants_mut().next()) - { - let mut data = Vec::new(); - file.read_to_end(&mut data) - .await - .wrap_err("Failed to read input file")?; - variant.set_data(data); + let target_name = if op == "replace" { + sub_matches + .get_one::("bundle-file") + .map(|name| match u64::from_str_radix(name, 16) { + Ok(id) => IdString64::from(id), + Err(_) => IdString64::String(name.clone()), + }) + .expect("argument is required") } else { - let err = - eyre::eyre!("No file '{}' in this bundle.", name.display()).with_suggestion(|| { - format!( - "Run '{} bundle list {}' to list the files in this bundle.", - clap::crate_name!(), - bundle_path.display() - ) - // Not yet supported. - // }) - // .with_suggestion(|| { - // format!( - // "Use '{} bundle inject --add {} {} {}' to add it as a new file", - // clap::crate_name!(), - // name.display(), - // bundle_path.display(), - // file_path.display() - // ) - }); + let mut path = PathBuf::from(in_file_path); + path.set_extension(""); + IdString64::from(path.to_slash_lossy().to_string()) + }; - return Err(err); + { + let span = tracing::Span::current(); + if !span.is_disabled() { + span.record("bundle_path", bundle_path.display().to_string()); + span.record("in_file_path", in_file_path.display().to_string()); + span.record("output_path", output_path.display().to_string()); + span.record("compile", compile); + span.record("file_name", target_name.display().to_string()); + } + } + + let mut bundle = { + let name = Bundle::get_name_from_path(&ctx, bundle_path); + if patch_number.is_some() { + unimplemented!("Create patch bundle"); + } else { + let binary = fs::read(bundle_path).await?; + Bundle::from_binary(&ctx, name, binary) + .wrap_err_with(|| format!("Failed to open bundle '{}'", bundle_path.display()))? + } + }; + + if op == "copy" { + unimplemented!("Implement copying a file from one bundle to the other."); + } + + let file_data = tokio::fs::read(&in_file_path) + .await + .wrap_err_with(|| format!("Failed to read file '{}'", in_file_path.display()))?; + + let data = if let Some(compile_type) = compile { + match compile_type.as_str() { + "texture" => { + let sjson = String::from_utf8(file_data).wrap_err_with(|| { + format!("Invalid UTF8 data in '{}'", in_file_path.display()) + })?; + + let root = in_file_path + .parent() + .ok_or_eyre("File path has no parent")?; + + let texture = texture::compile(target_name.clone(), sjson, root) + .await + .wrap_err_with(|| { + format!( + "Failed to compile file as texture: {}", + in_file_path.display() + ) + })?; + + texture.to_binary()? + } + _ => unreachable!("clap only allows file types that we implement"), + } + } else { + file_data + }; + + match op { + "replace" => { + let Some(file) = bundle + .files_mut() + .find(|file| file.matches_name(&target_name)) + else { + let err = eyre::eyre!( + "No file with name '{}' in bundle '{}'", + target_name.display(), + bundle_path.display() + ); + + return Err(err).with_suggestion(|| { + format!( + "Run '{} bundle list \"{}\"' to list the files in this bundle.", + clap::crate_name!(), + bundle_path.display() + ) + }); + }; + + let variant_index = sub_matches + .get_one::("variant") + .expect("argument with default missing"); + + let Some(variant) = file.variants_mut().nth(*variant_index as usize) else { + let err = eyre::eyre!( + "Variant index '{}' does not exist in '{}'", + variant_index, + target_name.display() + ); + + return Err(err).with_suggestion(|| { + format!( + "See '{} bundle inject add --help' if you want to add it as a new file", + clap::crate_name!(), + ) + }); + }; + + variant.set_data(data); + } + "add" => { + unimplemented!("Implement adding a new file to the bundle."); + } + _ => unreachable!("no other operations exist"), } - let out_path = matches.get_one::("output").unwrap_or(bundle_path); let data = bundle .to_binary() .wrap_err("Failed to write changed bundle to output")?; - fs::write(out_path, &data) + fs::write(&output_path, &data) .await - .wrap_err("Failed to write data to output file")?; + .wrap_err_with(|| format!("Failed to write data to '{}'", output_path.display()))?; Ok(()) } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index aa18184..f2429fa 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -335,14 +335,13 @@ impl BundleFile { s } - pub fn matches_name(&self, name: impl Into) -> bool { - let name = name.into(); - if self.name == name { + pub fn matches_name(&self, name: &IdString64) -> bool { + if self.name == *name { return true; } if let IdString64::String(name) = name { - self.name(false, None) == name || self.name(true, None) == name + self.name(false, None) == *name || self.name(true, None) == *name } else { false } From bb86584262000727c0bbde0cbbe4a516641f3e4a Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 16:03:04 +0200 Subject: [PATCH 09/35] Add cmdline to tracing output Can come in handy when other people report problems and show the error message or full log, but not the command line. Setting that span to `level = "error"` ensures that it won't be disabled by level filters. --- crates/dtmt/src/main.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index a36e58d..04a42fe 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -38,10 +38,21 @@ struct GlobalConfig { } #[tokio::main] -#[tracing::instrument] +#[tracing::instrument(level = "error", fields(cmd_line = tracing::field::Empty))] async fn main() -> Result<()> { color_eyre::install()?; + { + let span = tracing::Span::current(); + if !span.is_disabled() { + let cmdline: String = std::env::args_os().fold(String::new(), |mut s, arg| { + s.push_str(&arg.to_string_lossy()); + s + }); + span.record("cmd_line", cmdline); + } + } + let matches = command!() .subcommand_required(true) .arg( From ba1b3c4323b59809939899681613ff873481d510 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 29 Jul 2024 13:35:40 +0200 Subject: [PATCH 10/35] dtmt: Fix file injection I ended up wrapping the raw data in a `BundleFile` twice. I also made '--compile' the default, as it should be much less often that raw data needs to be inserted. Even files that are essentially raw binary blobs, like `.wwise_event`, still have some custom fields that need to be accounted for. --- crates/dtmt/src/cmd/bundle/inject.rs | 210 ++++++++++++++++----------- lib/sdk/src/bundle/file.rs | 16 ++ lib/sdk/src/bundle/mod.rs | 3 +- lib/sdk/src/lib.rs | 2 +- 4 files changed, 141 insertions(+), 90 deletions(-) diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 3d04fd6..958615a 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -1,12 +1,13 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::str::FromStr as _; -use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::{self, Context, OptionExt as _, Result}; +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::{self, Context, OptionExt, Result}; use color_eyre::Help; use path_slash::PathBufExt as _; use sdk::filetype::texture; use sdk::murmur::IdString64; -use sdk::Bundle; +use sdk::{Bundle, BundleFile, BundleFileType}; use tokio::fs; pub(crate) fn command_definition() -> Command { @@ -29,7 +30,9 @@ pub(crate) fn command_definition() -> Command { .arg( Arg::new("patch") .help("Create a patch bundle. Optionally, a patch NUMBER may be specified as \ - '--patch=123'.\nThe maximum number is 1.") + '--patch=123'.\nThe maximum number is 999, the default is 1.\n\ + If `--output` is not specified, the `.patch_` suffix is added to \ + the given bundle name.") .short('p') .long("patch") .num_args(0..=1) @@ -38,30 +41,28 @@ pub(crate) fn command_definition() -> Command { .value_name("NUMBER") .value_parser(value_parser!(u16)) ) + .arg( + Arg::new("type") + .help("Compile the new file as the given TYPE. If omitted, the file type is \ + is guessed from the file extension.") + .value_name("TYPE") + ) .subcommand( Command::new("replace") .about("Replace an existing file in the bundle") - .arg( - Arg::new("compile") - .help("Compile the file as TYPE before injecting. \ - Metadata will also be overwritten") - .long("compile") - .short('c') - .value_name("TYPE") - .value_parser(["texture"]) - ) .arg( Arg::new("variant") - .help("Sepcify the variant index to replace.") + .help("In combination with '--raw', specify the variant index to replace.") .long("variant") .default_value("0") .value_parser(value_parser!(u8)) ) .arg( - Arg::new("new-file") - .help("Path to the file to inject.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("raw") + .help("Insert the given file as raw binary data.\n\ + Cannot be used with '--patch'.") + .long("raw") + .action(ArgAction::SetTrue) ) .arg( Arg::new("bundle") @@ -73,6 +74,12 @@ pub(crate) fn command_definition() -> Command { Arg::new("bundle-file") .help("The name of a file in the bundle whose content should be replaced.") .required(true), + ) + .arg( + Arg::new("new-file") + .help("Path to the file to inject.") + .required(true) + .value_parser(value_parser!(PathBuf)), ), ) // .subcommand( @@ -93,14 +100,42 @@ pub(crate) fn command_definition() -> Command { // ) } +#[tracing::instrument] +async fn compile_file( + path: impl AsRef + std::fmt::Debug, + name: impl Into + std::fmt::Debug, + file_type: BundleFileType, +) -> Result { + let path = path.as_ref(); + + let file_data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?; + let sjson = String::from_utf8(file_data) + .wrap_err_with(|| format!("Invalid UTF8 data in '{}'", path.display()))?; + + let root = path.parent().ok_or_eyre("File path has no parent")?; + + match file_type { + BundleFileType::Texture => texture::compile(name.into(), sjson, root) + .await + .wrap_err_with(|| format!("Failed to compile file as texture: {}", path.display())), + _ => eyre::bail!( + "Compilation for type '{}' is not implemented, yet", + file_type + ), + } +} + #[tracing::instrument( skip_all, fields( bundle_path = tracing::field::Empty, in_file_path = tracing::field::Empty, - compile = tracing::field::Empty, output_path = tracing::field::Empty, - file_name = tracing::field::Empty, + target_name = tracing::field::Empty, + file_type = tracing::field::Empty, + raw = tracing::field::Empty, ) )] pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { @@ -108,14 +143,12 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { unreachable!("clap is configured to require a subcommand, and they're all handled above"); }; - let compile = matches.get_one::("compile"); - let bundle_path = sub_matches .get_one::("bundle") .expect("required parameter not found"); let in_file_path = sub_matches - .get_one::("file") + .get_one::("new-file") .expect("required parameter not found"); let patch_number = matches @@ -149,69 +182,46 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { IdString64::from(path.to_slash_lossy().to_string()) }; + let file_type = if let Some(forced_type) = matches.get_one::("type") { + BundleFileType::from_str(forced_type.as_str()).wrap_err("Unknown file type")? + } else { + in_file_path + .extension() + .and_then(|s| s.to_str()) + .ok_or_eyre("File extension missing") + .and_then(BundleFileType::from_str) + .wrap_err("Unknown file type") + .with_suggestion(|| "Use '--type TYPE' to specify the file type")? + }; + { let span = tracing::Span::current(); if !span.is_disabled() { span.record("bundle_path", bundle_path.display().to_string()); span.record("in_file_path", in_file_path.display().to_string()); span.record("output_path", output_path.display().to_string()); - span.record("compile", compile); - span.record("file_name", target_name.display().to_string()); + span.record("raw", sub_matches.get_flag("raw")); + span.record("target_name", target_name.display().to_string()); + span.record("file_type", format!("{:?}", file_type)); } } + let bundle_name = Bundle::get_name_from_path(&ctx, bundle_path); let mut bundle = { - let name = Bundle::get_name_from_path(&ctx, bundle_path); - if patch_number.is_some() { - unimplemented!("Create patch bundle"); - } else { - let binary = fs::read(bundle_path).await?; - Bundle::from_binary(&ctx, name, binary) - .wrap_err_with(|| format!("Failed to open bundle '{}'", bundle_path.display()))? - } + let binary = fs::read(bundle_path).await?; + Bundle::from_binary(&ctx, bundle_name.clone(), binary) + .wrap_err_with(|| format!("Failed to open bundle '{}'", bundle_path.display()))? }; if op == "copy" { unimplemented!("Implement copying a file from one bundle to the other."); } - let file_data = tokio::fs::read(&in_file_path) - .await - .wrap_err_with(|| format!("Failed to read file '{}'", in_file_path.display()))?; - - let data = if let Some(compile_type) = compile { - match compile_type.as_str() { - "texture" => { - let sjson = String::from_utf8(file_data).wrap_err_with(|| { - format!("Invalid UTF8 data in '{}'", in_file_path.display()) - })?; - - let root = in_file_path - .parent() - .ok_or_eyre("File path has no parent")?; - - let texture = texture::compile(target_name.clone(), sjson, root) - .await - .wrap_err_with(|| { - format!( - "Failed to compile file as texture: {}", - in_file_path.display() - ) - })?; - - texture.to_binary()? - } - _ => unreachable!("clap only allows file types that we implement"), - } - } else { - file_data - }; - - match op { + let output_bundle = match op { "replace" => { let Some(file) = bundle .files_mut() - .find(|file| file.matches_name(&target_name)) + .find(|file| *file.base_name() == target_name) else { let err = eyre::eyre!( "No file with name '{}' in bundle '{}'", @@ -228,34 +238,58 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { }); }; - let variant_index = sub_matches - .get_one::("variant") - .expect("argument with default missing"); + if sub_matches.get_flag("raw") { + let variant_index = sub_matches + .get_one::("variant") + .expect("argument with default missing"); - let Some(variant) = file.variants_mut().nth(*variant_index as usize) else { - let err = eyre::eyre!( - "Variant index '{}' does not exist in '{}'", - variant_index, - target_name.display() - ); + let Some(variant) = file.variants_mut().nth(*variant_index as usize) else { + let err = eyre::eyre!( + "Variant index '{}' does not exist in '{}'", + variant_index, + target_name.display() + ); - return Err(err).with_suggestion(|| { - format!( - "See '{} bundle inject add --help' if you want to add it as a new file", - clap::crate_name!(), - ) - }); - }; + return Err(err).with_suggestion(|| { + format!( + "See '{} bundle inject add --help' if you want to add it as a new file", + clap::crate_name!(), + ) + }); + }; - variant.set_data(data); + let data = tokio::fs::read(&in_file_path).await.wrap_err_with(|| { + format!("Failed to read file '{}'", in_file_path.display()) + })?; + variant.set_data(data); + file.set_modded(true); + bundle + } else { + let mut bundle_file = compile_file(in_file_path, target_name.clone(), file_type) + .await + .wrap_err("Failed to compile")?; + + bundle_file.set_modded(true); + + if patch_number.is_some() { + let mut output_bundle = Bundle::new(bundle_name); + output_bundle.add_file(bundle_file); + output_bundle + } else { + *file = bundle_file; + + dbg!(&file); + bundle + } + } } "add" => { unimplemented!("Implement adding a new file to the bundle."); } _ => unreachable!("no other operations exist"), - } + }; - let data = bundle + let data = output_bundle .to_binary() .wrap_err("Failed to write changed bundle to output")?; @@ -263,5 +297,7 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { .await .wrap_err_with(|| format!("Failed to write data to '{}'", output_path.display()))?; + tracing::info!("Modified bundle written to '{}'", output_path.display()); + Ok(()) } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index f2429fa..0d0a99f 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -21,6 +21,7 @@ struct BundleFileHeader { len_data_file_name: usize, } +#[derive(Clone)] pub struct BundleFileVariant { property: u32, data: Vec, @@ -140,9 +141,12 @@ bitflags! { #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { const DATA = 0b100; + // A custom flag used by DTMT to signify a file altered by mods. + const MODDED = 1 << 31; } } +#[derive(Clone, Debug)] pub struct BundleFile { file_type: BundleFileType, name: IdString64, @@ -164,6 +168,18 @@ impl BundleFile { self.variants.push(variant) } + pub fn set_variants(&mut self, variants: Vec) { + self.variants = variants; + } + + pub fn set_props(&mut self, props: Properties) { + self.props = props; + } + + pub fn set_modded(&mut self, is_modded: bool) { + self.props.set(Properties::MODDED, is_modded); + } + #[tracing::instrument(name = "File::read", skip(ctx, r))] pub fn from_reader(ctx: &crate::Context, r: &mut R, props: Properties) -> Result where diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index e969f39..03a04c7 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -7,14 +7,13 @@ use color_eyre::{Help, Report, SectionExt}; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE}; use crate::binary::sync::*; -use crate::bundle::file::Properties; use crate::murmur::{HashGroup, IdString64, Murmur64}; pub(crate) mod database; pub(crate) mod file; pub(crate) mod filetype; -pub use file::{BundleFile, BundleFileVariant}; +pub use file::{BundleFile, BundleFileVariant, Properties}; pub use filetype::BundleFileType; #[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index bf10826..41310fc 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -10,5 +10,5 @@ pub mod murmur; pub use binary::{FromBinary, ToBinary}; pub use bundle::database::BundleDatabase; pub use bundle::decompress; -pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant}; +pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant, Properties}; pub use context::{CmdLine, Context}; From 40b041133004b3aba4351732bcff6c8b707740a4 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 29 Jul 2024 15:10:20 +0200 Subject: [PATCH 11/35] sdk: Remove unused function --- lib/sdk/src/binary.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 40f9e9a..675dc43 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -152,7 +152,6 @@ pub mod sync { make_read!(read_u32, read_u32_le, u32); make_read!(read_u64, read_u64_le, u64); - make_skip!(skip_u8, read_u8, u8); make_skip!(skip_u16, read_u16, u16); make_skip!(skip_u32, read_u32, u32); From 6bb938fa97e5aa45a976f99d3c9c96d827958ba4 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 21 Feb 2025 14:51:42 +0100 Subject: [PATCH 12/35] Use macro to generate file type enum and impls Due to the large amount of variants, and the different kind of values connected to each variant (hash, extension name) being scattered across the various `impl` blocks, the file became rather convoluted. While I don't generally like the indirection of macros or meta programming, it's not that bad with Rust, thanks to Rust Analyzer being able to attach diagnostics to the source inside the macro definition, and the ability to generate the macro's output for validation. Therefore, the new macro allows putting all data used for this enum definition into a single block. --- lib/sdk/src/bundle/filetype.rs | 500 +++++++++------------------------ 1 file changed, 132 insertions(+), 368 deletions(-) diff --git a/lib/sdk/src/bundle/filetype.rs b/lib/sdk/src/bundle/filetype.rs index 65209b4..68ff6b5 100644 --- a/lib/sdk/src/bundle/filetype.rs +++ b/lib/sdk/src/bundle/filetype.rs @@ -3,238 +3,147 @@ use serde::Serialize; use crate::murmur::Murmur64; -#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] -pub enum BundleFileType { - Animation, - AnimationCurves, - Apb, - BakedLighting, - Bik, - BlendSet, - Bones, - Chroma, - CommonPackage, - Config, - Crypto, - Data, - Entity, - Flow, - Font, - Ies, - Ini, - Input, - Ivf, - Keys, - Level, - Lua, - Material, - Mod, - MouseCursor, - NavData, - NetworkConfig, - OddleNet, - Package, - Particles, - PhysicsProperties, - RenderConfig, - RtPipeline, - Scene, - Shader, - ShaderLibrary, - ShaderLibraryGroup, - ShadingEnvionmentMapping, - ShadingEnvironment, - Slug, - SlugAlbum, - SoundEnvironment, - SpuJob, - StateMachine, - StaticPVS, - Strings, - SurfaceProperties, - Texture, - TimpaniBank, - TimpaniMaster, - Tome, - Ugg, - Unit, - Upb, - VectorField, - Wav, - WwiseBank, - WwiseDep, - WwiseEvent, - WwiseMetadata, - WwiseStream, - Xml, - Theme, - MissionThemes, +macro_rules! make_enum { + ( + $( $variant:ident, $hash:expr, $ext:expr $(, $decompiled:expr)? ; )+ + ) => { + #[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] + pub enum BundleFileType { + $( + $variant, + )+ + Unknown(Murmur64), + } - Unknown(Murmur64), + impl BundleFileType { + pub fn ext_name(&self) -> String { + match self { + $( + Self::$variant => String::from($ext), + )+ + Self::Unknown(s) => format!("{s:016X}"), + } + } + + pub fn decompiled_ext_name(&self) -> String { + match self { + $( + $( Self::$variant => String::from($decompiled), )? + )+ + _ => self.ext_name(), + } + } + } + + impl std::str::FromStr for BundleFileType { + type Err = color_eyre::Report; + fn from_str(s: &str) -> Result { + match s { + $( + $ext => Ok(Self::$variant), + )+ + s => eyre::bail!("Unknown type string '{}'", s), + } + } + } + + impl From for BundleFileType { + fn from(h: u64) -> Self { + match h { + $( + $hash => Self::$variant, + )+ + hash => Self::Unknown(hash.into()), + } + } + } + + impl From for u64 { + fn from(t: BundleFileType) -> u64 { + match t { + $( + BundleFileType::$variant => $hash, + )+ + BundleFileType::Unknown(hash) => hash.into(), + } + } + } + } +} + +make_enum! { + AnimationCurves, 0xdcfb9e18fff13984, "animation_curves"; + Animation, 0x931e336d7646cc26, "animation"; + Apb, 0x3eed05ba83af5090, "apb"; + BakedLighting, 0x7ffdb779b04e4ed1, "baked_lighting"; + Bik, 0xaa5965f03029fa18, "bik"; + BlendSet, 0xe301e8af94e3b5a3, "blend_set"; + Bones, 0x18dead01056b72e9, "bones"; + Chroma, 0xb7893adf7567506a, "chroma"; + CommonPackage, 0xfe9754bd19814a47, "common_package"; + Config, 0x82645835e6b73232, "config"; + Crypto, 0x69108ded1e3e634b, "crypto"; + Data, 0x8fd0d44d20650b68, "data"; + Entity, 0x9831ca893b0d087d, "entity"; + Flow, 0x92d3ee038eeb610d, "flow"; + Font, 0x9efe0a916aae7880, "font"; + Ies, 0x8f7d5a2c0f967655, "ies"; + Ini, 0xd526a27da14f1dc5, "ini"; + Input, 0x2bbcabe5074ade9e, "input"; + Ivf, 0xfa4a8e091a91201e, "ivf"; + Keys, 0xa62f9297dc969e85, "keys"; + Level, 0x2a690fd348fe9ac5, "level"; + Lua, 0xa14e8dfa2cd117e2, "lua"; + Material, 0xeac0b497876adedf, "material"; + Mod, 0x3fcdd69156a46417, "mod"; + MouseCursor, 0xb277b11fe4a61d37, "mouse_cursor"; + NavData, 0x169de9566953d264, "nav_data"; + NetworkConfig, 0x3b1fa9e8f6bac374, "network_config"; + OddleNet, 0xb0f2c12eb107f4d8, "oodle_net"; + Package, 0xad9c6d9ed1e5e77a, "package"; + Particles, 0xa8193123526fad64, "particles"; + PhysicsProperties, 0xbf21403a3ab0bbb1, "physics_properties"; + RenderConfig, 0x27862fe24795319c, "render_config"; + RtPipeline, 0x9ca183c2d0e76dee, "rt_pipeline"; + Scene, 0x9d0a795bfe818d19, "scene"; + Shader, 0xcce8d5b5f5ae333f, "shader"; + ShaderLibrary, 0xe5ee32a477239a93, "shader_library"; + ShaderLibraryGroup, 0x9e5c3cc74575aeb5, "shader_library_group"; + ShadingEnvionmentMapping, 0x250e0a11ac8e26f8, "shading_envionment_mapping"; + ShadingEnvironment, 0xfe73c7dcff8a7ca5, "shading_environment"; + Slug, 0xa27b4d04a9ba6f9e, "slug"; + SlugAlbum, 0xe9fc9ea7042e5ec0, "slug_album"; + SoundEnvironment, 0xd8b27864a97ffdd7, "sound_environment"; + SpuJob, 0xf97af9983c05b950, "spu_job"; + StateMachine, 0xa486d4045106165c, "state_machine"; + StaticPVS, 0xe3f0baa17d620321, "static_pvs"; + Strings, 0x0d972bab10b40fd3, "strings"; + SurfaceProperties, 0xad2d3fa30d9ab394, "surface_properties"; + Texture, 0xcd4238c6a0c69e32, "texture", "dds"; + TimpaniBank, 0x99736be1fff739a4, "timpani_bank"; + TimpaniMaster, 0x00a3e6c59a2b9c6c, "timpani_master"; + Tome, 0x19c792357c99f49b, "tome"; + Ugg, 0x712d6e3dd1024c9c, "ugg"; + Unit, 0xe0a48d0be9a7453f, "unit"; + Upb, 0xa99510c6e86dd3c2, "upb"; + VectorField, 0xf7505933166d6755, "vector_field"; + Wav, 0x786f65c00a816b19, "wav"; + WwiseBank, 0x535a7bd3e650d799, "wwise_bank", "bnk"; + WwiseDep, 0xaf32095c82f2b070, "wwise_dep"; + WwiseEvent, 0xaabdd317b58dfc8a, "wwise_event"; + WwiseMetadata, 0xd50a8b7e1c82b110, "wwise_metadata"; + WwiseStream, 0x504b55235d21440e, "wwise_stream", "ogg"; + Xml, 0x76015845a6003765, "xml"; + Theme, 0x38BB9442048A7FBD, "theme"; + MissionThemes, 0x80F2DE893657F83A, "mission_themes"; } impl BundleFileType { - pub fn ext_name(&self) -> String { - match self { - BundleFileType::AnimationCurves => String::from("animation_curves"), - BundleFileType::Animation => String::from("animation"), - BundleFileType::Apb => String::from("apb"), - BundleFileType::BakedLighting => String::from("baked_lighting"), - BundleFileType::Bik => String::from("bik"), - BundleFileType::BlendSet => String::from("blend_set"), - BundleFileType::Bones => String::from("bones"), - BundleFileType::Chroma => String::from("chroma"), - BundleFileType::CommonPackage => String::from("common_package"), - BundleFileType::Config => String::from("config"), - BundleFileType::Crypto => String::from("crypto"), - BundleFileType::Data => String::from("data"), - BundleFileType::Entity => String::from("entity"), - BundleFileType::Flow => String::from("flow"), - BundleFileType::Font => String::from("font"), - BundleFileType::Ies => String::from("ies"), - BundleFileType::Ini => String::from("ini"), - BundleFileType::Input => String::from("input"), - BundleFileType::Ivf => String::from("ivf"), - BundleFileType::Keys => String::from("keys"), - BundleFileType::Level => String::from("level"), - BundleFileType::Lua => String::from("lua"), - BundleFileType::Material => String::from("material"), - BundleFileType::Mod => String::from("mod"), - BundleFileType::MouseCursor => String::from("mouse_cursor"), - BundleFileType::NavData => String::from("nav_data"), - BundleFileType::NetworkConfig => String::from("network_config"), - BundleFileType::OddleNet => String::from("oodle_net"), - BundleFileType::Package => String::from("package"), - BundleFileType::Particles => String::from("particles"), - BundleFileType::PhysicsProperties => String::from("physics_properties"), - BundleFileType::RenderConfig => String::from("render_config"), - BundleFileType::RtPipeline => String::from("rt_pipeline"), - BundleFileType::Scene => String::from("scene"), - BundleFileType::ShaderLibraryGroup => String::from("shader_library_group"), - BundleFileType::ShaderLibrary => String::from("shader_library"), - BundleFileType::Shader => String::from("shader"), - BundleFileType::ShadingEnvionmentMapping => String::from("shading_environment_mapping"), - BundleFileType::ShadingEnvironment => String::from("shading_environment"), - BundleFileType::SlugAlbum => String::from("slug_album"), - BundleFileType::Slug => String::from("slug"), - BundleFileType::SoundEnvironment => String::from("sound_environment"), - BundleFileType::SpuJob => String::from("spu_job"), - BundleFileType::StateMachine => String::from("state_machine"), - BundleFileType::StaticPVS => String::from("static_pvs"), - BundleFileType::Strings => String::from("strings"), - BundleFileType::SurfaceProperties => String::from("surface_properties"), - BundleFileType::Texture => String::from("texture"), - BundleFileType::TimpaniBank => String::from("timpani_bank"), - BundleFileType::TimpaniMaster => String::from("timpani_master"), - BundleFileType::Tome => String::from("tome"), - BundleFileType::Ugg => String::from("ugg"), - BundleFileType::Unit => String::from("unit"), - BundleFileType::Upb => String::from("upb"), - BundleFileType::VectorField => String::from("vector_field"), - BundleFileType::Wav => String::from("wav"), - BundleFileType::WwiseBank => String::from("wwise_bank"), - BundleFileType::WwiseDep => String::from("wwise_dep"), - BundleFileType::WwiseEvent => String::from("wwise_event"), - BundleFileType::WwiseMetadata => String::from("wwise_metadata"), - BundleFileType::WwiseStream => String::from("wwise_stream"), - BundleFileType::Xml => String::from("xml"), - BundleFileType::Theme => String::from("theme"), - BundleFileType::MissionThemes => String::from("mission_themes"), - - BundleFileType::Unknown(s) => format!("{s:016X}"), - } - } - - pub fn decompiled_ext_name(&self) -> String { - match self { - BundleFileType::Texture => String::from("dds"), - BundleFileType::WwiseBank => String::from("bnk"), - BundleFileType::WwiseStream => String::from("ogg"), - _ => self.ext_name(), - } - } - pub fn hash(&self) -> Murmur64 { Murmur64::from(*self) } } -impl std::str::FromStr for BundleFileType { - type Err = color_eyre::Report; - - fn from_str(s: &str) -> Result { - let val = match s { - "animation_curves" => BundleFileType::AnimationCurves, - "animation" => BundleFileType::Animation, - "apb" => BundleFileType::Apb, - "baked_lighting" => BundleFileType::BakedLighting, - "bik" => BundleFileType::Bik, - "blend_set" => BundleFileType::BlendSet, - "bones" => BundleFileType::Bones, - "chroma" => BundleFileType::Chroma, - "common_package" => BundleFileType::CommonPackage, - "config" => BundleFileType::Config, - "crypto" => BundleFileType::Crypto, - "data" => BundleFileType::Data, - "entity" => BundleFileType::Entity, - "flow" => BundleFileType::Flow, - "font" => BundleFileType::Font, - "ies" => BundleFileType::Ies, - "ini" => BundleFileType::Ini, - "input" => BundleFileType::Input, - "ivf" => BundleFileType::Ivf, - "keys" => BundleFileType::Keys, - "level" => BundleFileType::Level, - "lua" => BundleFileType::Lua, - "material" => BundleFileType::Material, - "mod" => BundleFileType::Mod, - "mouse_cursor" => BundleFileType::MouseCursor, - "nav_data" => BundleFileType::NavData, - "network_config" => BundleFileType::NetworkConfig, - "oodle_net" => BundleFileType::OddleNet, - "package" => BundleFileType::Package, - "particles" => BundleFileType::Particles, - "physics_properties" => BundleFileType::PhysicsProperties, - "render_config" => BundleFileType::RenderConfig, - "rt_pipeline" => BundleFileType::RtPipeline, - "scene" => BundleFileType::Scene, - "shader_library_group" => BundleFileType::ShaderLibraryGroup, - "shader_library" => BundleFileType::ShaderLibrary, - "shader" => BundleFileType::Shader, - "shading_environment_mapping" => BundleFileType::ShadingEnvionmentMapping, - "shading_environment" => BundleFileType::ShadingEnvironment, - "slug_album" => BundleFileType::SlugAlbum, - "slug" => BundleFileType::Slug, - "sound_environment" => BundleFileType::SoundEnvironment, - "spu_job" => BundleFileType::SpuJob, - "state_machine" => BundleFileType::StateMachine, - "static_pvs" => BundleFileType::StaticPVS, - "strings" => BundleFileType::Strings, - "surface_properties" => BundleFileType::SurfaceProperties, - "texture" => BundleFileType::Texture, - "timpani_bank" => BundleFileType::TimpaniBank, - "timpani_master" => BundleFileType::TimpaniMaster, - "tome" => BundleFileType::Tome, - "ugg" => BundleFileType::Ugg, - "unit" => BundleFileType::Unit, - "upb" => BundleFileType::Upb, - "vector_field" => BundleFileType::VectorField, - "wav" => BundleFileType::Wav, - "wwise_bank" => BundleFileType::WwiseBank, - "wwise_dep" => BundleFileType::WwiseDep, - "wwise_event" => BundleFileType::WwiseEvent, - "wwise_metadata" => BundleFileType::WwiseMetadata, - "wwise_stream" => BundleFileType::WwiseStream, - "xml" => BundleFileType::Xml, - "theme" => BundleFileType::Theme, - "mission_themes" => BundleFileType::MissionThemes, - s => eyre::bail!("Unknown type string '{}'", s), - }; - - Ok(val) - } -} - impl Serialize for BundleFileType { fn serialize(&self, serializer: S) -> Result where @@ -251,151 +160,6 @@ impl From for BundleFileType { } } -impl From for BundleFileType { - fn from(hash: u64) -> BundleFileType { - match hash { - 0x931e336d7646cc26 => BundleFileType::Animation, - 0xdcfb9e18fff13984 => BundleFileType::AnimationCurves, - 0x3eed05ba83af5090 => BundleFileType::Apb, - 0x7ffdb779b04e4ed1 => BundleFileType::BakedLighting, - 0xaa5965f03029fa18 => BundleFileType::Bik, - 0xe301e8af94e3b5a3 => BundleFileType::BlendSet, - 0x18dead01056b72e9 => BundleFileType::Bones, - 0xb7893adf7567506a => BundleFileType::Chroma, - 0xfe9754bd19814a47 => BundleFileType::CommonPackage, - 0x82645835e6b73232 => BundleFileType::Config, - 0x69108ded1e3e634b => BundleFileType::Crypto, - 0x8fd0d44d20650b68 => BundleFileType::Data, - 0x9831ca893b0d087d => BundleFileType::Entity, - 0x92d3ee038eeb610d => BundleFileType::Flow, - 0x9efe0a916aae7880 => BundleFileType::Font, - 0x8f7d5a2c0f967655 => BundleFileType::Ies, - 0xd526a27da14f1dc5 => BundleFileType::Ini, - 0x2bbcabe5074ade9e => BundleFileType::Input, - 0xfa4a8e091a91201e => BundleFileType::Ivf, - 0xa62f9297dc969e85 => BundleFileType::Keys, - 0x2a690fd348fe9ac5 => BundleFileType::Level, - 0xa14e8dfa2cd117e2 => BundleFileType::Lua, - 0xeac0b497876adedf => BundleFileType::Material, - 0x3fcdd69156a46417 => BundleFileType::Mod, - 0xb277b11fe4a61d37 => BundleFileType::MouseCursor, - 0x169de9566953d264 => BundleFileType::NavData, - 0x3b1fa9e8f6bac374 => BundleFileType::NetworkConfig, - 0xb0f2c12eb107f4d8 => BundleFileType::OddleNet, - 0xad9c6d9ed1e5e77a => BundleFileType::Package, - 0xa8193123526fad64 => BundleFileType::Particles, - 0xbf21403a3ab0bbb1 => BundleFileType::PhysicsProperties, - 0x27862fe24795319c => BundleFileType::RenderConfig, - 0x9ca183c2d0e76dee => BundleFileType::RtPipeline, - 0x9d0a795bfe818d19 => BundleFileType::Scene, - 0xcce8d5b5f5ae333f => BundleFileType::Shader, - 0xe5ee32a477239a93 => BundleFileType::ShaderLibrary, - 0x9e5c3cc74575aeb5 => BundleFileType::ShaderLibraryGroup, - 0x250e0a11ac8e26f8 => BundleFileType::ShadingEnvionmentMapping, - 0xfe73c7dcff8a7ca5 => BundleFileType::ShadingEnvironment, - 0xa27b4d04a9ba6f9e => BundleFileType::Slug, - 0xe9fc9ea7042e5ec0 => BundleFileType::SlugAlbum, - 0xd8b27864a97ffdd7 => BundleFileType::SoundEnvironment, - 0xf97af9983c05b950 => BundleFileType::SpuJob, - 0xa486d4045106165c => BundleFileType::StateMachine, - 0xe3f0baa17d620321 => BundleFileType::StaticPVS, - 0x0d972bab10b40fd3 => BundleFileType::Strings, - 0xad2d3fa30d9ab394 => BundleFileType::SurfaceProperties, - 0xcd4238c6a0c69e32 => BundleFileType::Texture, - 0x99736be1fff739a4 => BundleFileType::TimpaniBank, - 0x00a3e6c59a2b9c6c => BundleFileType::TimpaniMaster, - 0x19c792357c99f49b => BundleFileType::Tome, - 0x712d6e3dd1024c9c => BundleFileType::Ugg, - 0xe0a48d0be9a7453f => BundleFileType::Unit, - 0xa99510c6e86dd3c2 => BundleFileType::Upb, - 0xf7505933166d6755 => BundleFileType::VectorField, - 0x786f65c00a816b19 => BundleFileType::Wav, - 0x535a7bd3e650d799 => BundleFileType::WwiseBank, - 0xaf32095c82f2b070 => BundleFileType::WwiseDep, - 0xaabdd317b58dfc8a => BundleFileType::WwiseEvent, - 0xd50a8b7e1c82b110 => BundleFileType::WwiseMetadata, - 0x504b55235d21440e => BundleFileType::WwiseStream, - 0x76015845a6003765 => BundleFileType::Xml, - 0x38BB9442048A7FBD => Self::Theme, - 0x80F2DE893657F83A => Self::MissionThemes, - - _ => BundleFileType::Unknown(Murmur64::from(hash)), - } - } -} - -impl From for u64 { - fn from(t: BundleFileType) -> u64 { - match t { - BundleFileType::Animation => 0x931e336d7646cc26, - BundleFileType::AnimationCurves => 0xdcfb9e18fff13984, - BundleFileType::Apb => 0x3eed05ba83af5090, - BundleFileType::BakedLighting => 0x7ffdb779b04e4ed1, - BundleFileType::Bik => 0xaa5965f03029fa18, - BundleFileType::BlendSet => 0xe301e8af94e3b5a3, - BundleFileType::Bones => 0x18dead01056b72e9, - BundleFileType::Chroma => 0xb7893adf7567506a, - BundleFileType::CommonPackage => 0xfe9754bd19814a47, - BundleFileType::Config => 0x82645835e6b73232, - BundleFileType::Crypto => 0x69108ded1e3e634b, - BundleFileType::Data => 0x8fd0d44d20650b68, - BundleFileType::Entity => 0x9831ca893b0d087d, - BundleFileType::Flow => 0x92d3ee038eeb610d, - BundleFileType::Font => 0x9efe0a916aae7880, - BundleFileType::Ies => 0x8f7d5a2c0f967655, - BundleFileType::Ini => 0xd526a27da14f1dc5, - BundleFileType::Input => 0x2bbcabe5074ade9e, - BundleFileType::Ivf => 0xfa4a8e091a91201e, - BundleFileType::Keys => 0xa62f9297dc969e85, - BundleFileType::Level => 0x2a690fd348fe9ac5, - BundleFileType::Lua => 0xa14e8dfa2cd117e2, - BundleFileType::Material => 0xeac0b497876adedf, - BundleFileType::Mod => 0x3fcdd69156a46417, - BundleFileType::MouseCursor => 0xb277b11fe4a61d37, - BundleFileType::NavData => 0x169de9566953d264, - BundleFileType::NetworkConfig => 0x3b1fa9e8f6bac374, - BundleFileType::OddleNet => 0xb0f2c12eb107f4d8, - BundleFileType::Package => 0xad9c6d9ed1e5e77a, - BundleFileType::Particles => 0xa8193123526fad64, - BundleFileType::PhysicsProperties => 0xbf21403a3ab0bbb1, - BundleFileType::RenderConfig => 0x27862fe24795319c, - BundleFileType::RtPipeline => 0x9ca183c2d0e76dee, - BundleFileType::Scene => 0x9d0a795bfe818d19, - BundleFileType::Shader => 0xcce8d5b5f5ae333f, - BundleFileType::ShaderLibrary => 0xe5ee32a477239a93, - BundleFileType::ShaderLibraryGroup => 0x9e5c3cc74575aeb5, - BundleFileType::ShadingEnvionmentMapping => 0x250e0a11ac8e26f8, - BundleFileType::ShadingEnvironment => 0xfe73c7dcff8a7ca5, - BundleFileType::Slug => 0xa27b4d04a9ba6f9e, - BundleFileType::SlugAlbum => 0xe9fc9ea7042e5ec0, - BundleFileType::SoundEnvironment => 0xd8b27864a97ffdd7, - BundleFileType::SpuJob => 0xf97af9983c05b950, - BundleFileType::StateMachine => 0xa486d4045106165c, - BundleFileType::StaticPVS => 0xe3f0baa17d620321, - BundleFileType::Strings => 0x0d972bab10b40fd3, - BundleFileType::SurfaceProperties => 0xad2d3fa30d9ab394, - BundleFileType::Texture => 0xcd4238c6a0c69e32, - BundleFileType::TimpaniBank => 0x99736be1fff739a4, - BundleFileType::TimpaniMaster => 0x00a3e6c59a2b9c6c, - BundleFileType::Tome => 0x19c792357c99f49b, - BundleFileType::Ugg => 0x712d6e3dd1024c9c, - BundleFileType::Unit => 0xe0a48d0be9a7453f, - BundleFileType::Upb => 0xa99510c6e86dd3c2, - BundleFileType::VectorField => 0xf7505933166d6755, - BundleFileType::Wav => 0x786f65c00a816b19, - BundleFileType::WwiseBank => 0x535a7bd3e650d799, - BundleFileType::WwiseDep => 0xaf32095c82f2b070, - BundleFileType::WwiseEvent => 0xaabdd317b58dfc8a, - BundleFileType::WwiseMetadata => 0xd50a8b7e1c82b110, - BundleFileType::WwiseStream => 0x504b55235d21440e, - BundleFileType::Xml => 0x76015845a6003765, - BundleFileType::Theme => 0x38BB9442048A7FBD, - BundleFileType::MissionThemes => 0x80F2DE893657F83A, - - BundleFileType::Unknown(hash) => hash.into(), - } - } -} impl From for Murmur64 { fn from(t: BundleFileType) -> Murmur64 { let hash: u64 = t.into(); From 805fe53914f551045d846acb42802bf220fe8f36 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 6 Mar 2025 13:32:47 +0100 Subject: [PATCH 13/35] Only use texture files for texture-meta command --- crates/dtmt/src/cmd/experiment/texture_meta.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/dtmt/src/cmd/experiment/texture_meta.rs b/crates/dtmt/src/cmd/experiment/texture_meta.rs index 98d0035..a77e693 100644 --- a/crates/dtmt/src/cmd/experiment/texture_meta.rs +++ b/crates/dtmt/src/cmd/experiment/texture_meta.rs @@ -5,7 +5,7 @@ use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use color_eyre::eyre::Context; use color_eyre::Result; use futures_util::StreamExt; -use sdk::Bundle; +use sdk::{Bundle, BundleFileType}; use tokio::fs; use crate::cmd::util::resolve_bundle_paths; @@ -46,6 +46,10 @@ async fn handle_bundle(ctx: &sdk::Context, path: &PathBuf) -> Result<()> { .unwrap_or_default(); for f in bundle.files().iter() { + if f.file_type() != BundleFileType::Texture { + continue; + } + for (i, v) in f.variants().iter().enumerate() { let data_file_name = v.data_file_name(); From 43e3bf7b60de613258f4da7fc116bebaefc9c1f1 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 16:03:04 +0200 Subject: [PATCH 14/35] Add cmdline to tracing output Can come in handy when other people report problems and show the error message or full log, but not the command line. Setting that span to `level = "error"` ensures that it won't be disabled by level filters. --- crates/dtmt/src/main.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index 2e10b17..e41e802 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -36,10 +36,21 @@ struct GlobalConfig { } #[tokio::main] -#[tracing::instrument] +#[tracing::instrument(level = "error", fields(cmd_line = tracing::field::Empty))] async fn main() -> Result<()> { color_eyre::install()?; + { + let span = tracing::Span::current(); + if !span.is_disabled() { + let cmdline: String = std::env::args_os().fold(String::new(), |mut s, arg| { + s.push_str(&arg.to_string_lossy()); + s + }); + span.record("cmd_line", cmdline); + } + } + let matches = command!() .subcommand_required(true) .arg( From 636279edfe376bd9d748347c7195d4e82af90015 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 21 Feb 2025 14:51:42 +0100 Subject: [PATCH 15/35] Use macro to generate file type enum and impls Due to the large amount of variants, and the different kind of values connected to each variant (hash, extension name) being scattered across the various `impl` blocks, the file became rather convoluted. While I don't generally like the indirection of macros or meta programming, it's not that bad with Rust, thanks to Rust Analyzer being able to attach diagnostics to the source inside the macro definition, and the ability to generate the macro's output for validation. Therefore, the new macro allows putting all data used for this enum definition into a single block. --- lib/sdk/src/bundle/filetype.rs | 490 +++++++++------------------------ 1 file changed, 132 insertions(+), 358 deletions(-) diff --git a/lib/sdk/src/bundle/filetype.rs b/lib/sdk/src/bundle/filetype.rs index 0b4f292..68ff6b5 100644 --- a/lib/sdk/src/bundle/filetype.rs +++ b/lib/sdk/src/bundle/filetype.rs @@ -3,232 +3,147 @@ use serde::Serialize; use crate::murmur::Murmur64; -#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] -pub enum BundleFileType { - Animation, - AnimationCurves, - Apb, - BakedLighting, - Bik, - BlendSet, - Bones, - Chroma, - CommonPackage, - Config, - Crypto, - Data, - Entity, - Flow, - Font, - Ies, - Ini, - Input, - Ivf, - Keys, - Level, - Lua, - Material, - Mod, - MouseCursor, - NavData, - NetworkConfig, - OddleNet, - Package, - Particles, - PhysicsProperties, - RenderConfig, - RtPipeline, - Scene, - Shader, - ShaderLibrary, - ShaderLibraryGroup, - ShadingEnvionmentMapping, - ShadingEnvironment, - Slug, - SlugAlbum, - SoundEnvironment, - SpuJob, - StateMachine, - StaticPVS, - Strings, - SurfaceProperties, - Texture, - TimpaniBank, - TimpaniMaster, - Tome, - Ugg, - Unit, - Upb, - VectorField, - Wav, - WwiseBank, - WwiseDep, - WwiseEvent, - WwiseMetadata, - WwiseStream, - Xml, +macro_rules! make_enum { + ( + $( $variant:ident, $hash:expr, $ext:expr $(, $decompiled:expr)? ; )+ + ) => { + #[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] + pub enum BundleFileType { + $( + $variant, + )+ + Unknown(Murmur64), + } - Unknown(Murmur64), + impl BundleFileType { + pub fn ext_name(&self) -> String { + match self { + $( + Self::$variant => String::from($ext), + )+ + Self::Unknown(s) => format!("{s:016X}"), + } + } + + pub fn decompiled_ext_name(&self) -> String { + match self { + $( + $( Self::$variant => String::from($decompiled), )? + )+ + _ => self.ext_name(), + } + } + } + + impl std::str::FromStr for BundleFileType { + type Err = color_eyre::Report; + fn from_str(s: &str) -> Result { + match s { + $( + $ext => Ok(Self::$variant), + )+ + s => eyre::bail!("Unknown type string '{}'", s), + } + } + } + + impl From for BundleFileType { + fn from(h: u64) -> Self { + match h { + $( + $hash => Self::$variant, + )+ + hash => Self::Unknown(hash.into()), + } + } + } + + impl From for u64 { + fn from(t: BundleFileType) -> u64 { + match t { + $( + BundleFileType::$variant => $hash, + )+ + BundleFileType::Unknown(hash) => hash.into(), + } + } + } + } +} + +make_enum! { + AnimationCurves, 0xdcfb9e18fff13984, "animation_curves"; + Animation, 0x931e336d7646cc26, "animation"; + Apb, 0x3eed05ba83af5090, "apb"; + BakedLighting, 0x7ffdb779b04e4ed1, "baked_lighting"; + Bik, 0xaa5965f03029fa18, "bik"; + BlendSet, 0xe301e8af94e3b5a3, "blend_set"; + Bones, 0x18dead01056b72e9, "bones"; + Chroma, 0xb7893adf7567506a, "chroma"; + CommonPackage, 0xfe9754bd19814a47, "common_package"; + Config, 0x82645835e6b73232, "config"; + Crypto, 0x69108ded1e3e634b, "crypto"; + Data, 0x8fd0d44d20650b68, "data"; + Entity, 0x9831ca893b0d087d, "entity"; + Flow, 0x92d3ee038eeb610d, "flow"; + Font, 0x9efe0a916aae7880, "font"; + Ies, 0x8f7d5a2c0f967655, "ies"; + Ini, 0xd526a27da14f1dc5, "ini"; + Input, 0x2bbcabe5074ade9e, "input"; + Ivf, 0xfa4a8e091a91201e, "ivf"; + Keys, 0xa62f9297dc969e85, "keys"; + Level, 0x2a690fd348fe9ac5, "level"; + Lua, 0xa14e8dfa2cd117e2, "lua"; + Material, 0xeac0b497876adedf, "material"; + Mod, 0x3fcdd69156a46417, "mod"; + MouseCursor, 0xb277b11fe4a61d37, "mouse_cursor"; + NavData, 0x169de9566953d264, "nav_data"; + NetworkConfig, 0x3b1fa9e8f6bac374, "network_config"; + OddleNet, 0xb0f2c12eb107f4d8, "oodle_net"; + Package, 0xad9c6d9ed1e5e77a, "package"; + Particles, 0xa8193123526fad64, "particles"; + PhysicsProperties, 0xbf21403a3ab0bbb1, "physics_properties"; + RenderConfig, 0x27862fe24795319c, "render_config"; + RtPipeline, 0x9ca183c2d0e76dee, "rt_pipeline"; + Scene, 0x9d0a795bfe818d19, "scene"; + Shader, 0xcce8d5b5f5ae333f, "shader"; + ShaderLibrary, 0xe5ee32a477239a93, "shader_library"; + ShaderLibraryGroup, 0x9e5c3cc74575aeb5, "shader_library_group"; + ShadingEnvionmentMapping, 0x250e0a11ac8e26f8, "shading_envionment_mapping"; + ShadingEnvironment, 0xfe73c7dcff8a7ca5, "shading_environment"; + Slug, 0xa27b4d04a9ba6f9e, "slug"; + SlugAlbum, 0xe9fc9ea7042e5ec0, "slug_album"; + SoundEnvironment, 0xd8b27864a97ffdd7, "sound_environment"; + SpuJob, 0xf97af9983c05b950, "spu_job"; + StateMachine, 0xa486d4045106165c, "state_machine"; + StaticPVS, 0xe3f0baa17d620321, "static_pvs"; + Strings, 0x0d972bab10b40fd3, "strings"; + SurfaceProperties, 0xad2d3fa30d9ab394, "surface_properties"; + Texture, 0xcd4238c6a0c69e32, "texture", "dds"; + TimpaniBank, 0x99736be1fff739a4, "timpani_bank"; + TimpaniMaster, 0x00a3e6c59a2b9c6c, "timpani_master"; + Tome, 0x19c792357c99f49b, "tome"; + Ugg, 0x712d6e3dd1024c9c, "ugg"; + Unit, 0xe0a48d0be9a7453f, "unit"; + Upb, 0xa99510c6e86dd3c2, "upb"; + VectorField, 0xf7505933166d6755, "vector_field"; + Wav, 0x786f65c00a816b19, "wav"; + WwiseBank, 0x535a7bd3e650d799, "wwise_bank", "bnk"; + WwiseDep, 0xaf32095c82f2b070, "wwise_dep"; + WwiseEvent, 0xaabdd317b58dfc8a, "wwise_event"; + WwiseMetadata, 0xd50a8b7e1c82b110, "wwise_metadata"; + WwiseStream, 0x504b55235d21440e, "wwise_stream", "ogg"; + Xml, 0x76015845a6003765, "xml"; + Theme, 0x38BB9442048A7FBD, "theme"; + MissionThemes, 0x80F2DE893657F83A, "mission_themes"; } impl BundleFileType { - pub fn ext_name(&self) -> String { - match self { - BundleFileType::AnimationCurves => String::from("animation_curves"), - BundleFileType::Animation => String::from("animation"), - BundleFileType::Apb => String::from("apb"), - BundleFileType::BakedLighting => String::from("baked_lighting"), - BundleFileType::Bik => String::from("bik"), - BundleFileType::BlendSet => String::from("blend_set"), - BundleFileType::Bones => String::from("bones"), - BundleFileType::Chroma => String::from("chroma"), - BundleFileType::CommonPackage => String::from("common_package"), - BundleFileType::Config => String::from("config"), - BundleFileType::Crypto => String::from("crypto"), - BundleFileType::Data => String::from("data"), - BundleFileType::Entity => String::from("entity"), - BundleFileType::Flow => String::from("flow"), - BundleFileType::Font => String::from("font"), - BundleFileType::Ies => String::from("ies"), - BundleFileType::Ini => String::from("ini"), - BundleFileType::Input => String::from("input"), - BundleFileType::Ivf => String::from("ivf"), - BundleFileType::Keys => String::from("keys"), - BundleFileType::Level => String::from("level"), - BundleFileType::Lua => String::from("lua"), - BundleFileType::Material => String::from("material"), - BundleFileType::Mod => String::from("mod"), - BundleFileType::MouseCursor => String::from("mouse_cursor"), - BundleFileType::NavData => String::from("nav_data"), - BundleFileType::NetworkConfig => String::from("network_config"), - BundleFileType::OddleNet => String::from("oodle_net"), - BundleFileType::Package => String::from("package"), - BundleFileType::Particles => String::from("particles"), - BundleFileType::PhysicsProperties => String::from("physics_properties"), - BundleFileType::RenderConfig => String::from("render_config"), - BundleFileType::RtPipeline => String::from("rt_pipeline"), - BundleFileType::Scene => String::from("scene"), - BundleFileType::ShaderLibraryGroup => String::from("shader_library_group"), - BundleFileType::ShaderLibrary => String::from("shader_library"), - BundleFileType::Shader => String::from("shader"), - BundleFileType::ShadingEnvionmentMapping => String::from("shading_environment_mapping"), - BundleFileType::ShadingEnvironment => String::from("shading_environment"), - BundleFileType::SlugAlbum => String::from("slug_album"), - BundleFileType::Slug => String::from("slug"), - BundleFileType::SoundEnvironment => String::from("sound_environment"), - BundleFileType::SpuJob => String::from("spu_job"), - BundleFileType::StateMachine => String::from("state_machine"), - BundleFileType::StaticPVS => String::from("static_pvs"), - BundleFileType::Strings => String::from("strings"), - BundleFileType::SurfaceProperties => String::from("surface_properties"), - BundleFileType::Texture => String::from("texture"), - BundleFileType::TimpaniBank => String::from("timpani_bank"), - BundleFileType::TimpaniMaster => String::from("timpani_master"), - BundleFileType::Tome => String::from("tome"), - BundleFileType::Ugg => String::from("ugg"), - BundleFileType::Unit => String::from("unit"), - BundleFileType::Upb => String::from("upb"), - BundleFileType::VectorField => String::from("vector_field"), - BundleFileType::Wav => String::from("wav"), - BundleFileType::WwiseBank => String::from("wwise_bank"), - BundleFileType::WwiseDep => String::from("wwise_dep"), - BundleFileType::WwiseEvent => String::from("wwise_event"), - BundleFileType::WwiseMetadata => String::from("wwise_metadata"), - BundleFileType::WwiseStream => String::from("wwise_stream"), - BundleFileType::Xml => String::from("xml"), - - BundleFileType::Unknown(s) => format!("{s:016X}"), - } - } - - pub fn decompiled_ext_name(&self) -> String { - match self { - BundleFileType::Texture => String::from("dds"), - BundleFileType::WwiseBank => String::from("bnk"), - BundleFileType::WwiseStream => String::from("ogg"), - _ => self.ext_name(), - } - } - pub fn hash(&self) -> Murmur64 { Murmur64::from(*self) } } -impl std::str::FromStr for BundleFileType { - type Err = color_eyre::Report; - - fn from_str(s: &str) -> Result { - let val = match s { - "animation_curves" => BundleFileType::AnimationCurves, - "animation" => BundleFileType::Animation, - "apb" => BundleFileType::Apb, - "baked_lighting" => BundleFileType::BakedLighting, - "bik" => BundleFileType::Bik, - "blend_set" => BundleFileType::BlendSet, - "bones" => BundleFileType::Bones, - "chroma" => BundleFileType::Chroma, - "common_package" => BundleFileType::CommonPackage, - "config" => BundleFileType::Config, - "crypto" => BundleFileType::Crypto, - "data" => BundleFileType::Data, - "entity" => BundleFileType::Entity, - "flow" => BundleFileType::Flow, - "font" => BundleFileType::Font, - "ies" => BundleFileType::Ies, - "ini" => BundleFileType::Ini, - "input" => BundleFileType::Input, - "ivf" => BundleFileType::Ivf, - "keys" => BundleFileType::Keys, - "level" => BundleFileType::Level, - "lua" => BundleFileType::Lua, - "material" => BundleFileType::Material, - "mod" => BundleFileType::Mod, - "mouse_cursor" => BundleFileType::MouseCursor, - "nav_data" => BundleFileType::NavData, - "network_config" => BundleFileType::NetworkConfig, - "oodle_net" => BundleFileType::OddleNet, - "package" => BundleFileType::Package, - "particles" => BundleFileType::Particles, - "physics_properties" => BundleFileType::PhysicsProperties, - "render_config" => BundleFileType::RenderConfig, - "rt_pipeline" => BundleFileType::RtPipeline, - "scene" => BundleFileType::Scene, - "shader_library_group" => BundleFileType::ShaderLibraryGroup, - "shader_library" => BundleFileType::ShaderLibrary, - "shader" => BundleFileType::Shader, - "shading_environment_mapping" => BundleFileType::ShadingEnvionmentMapping, - "shading_environment" => BundleFileType::ShadingEnvironment, - "slug_album" => BundleFileType::SlugAlbum, - "slug" => BundleFileType::Slug, - "sound_environment" => BundleFileType::SoundEnvironment, - "spu_job" => BundleFileType::SpuJob, - "state_machine" => BundleFileType::StateMachine, - "static_pvs" => BundleFileType::StaticPVS, - "strings" => BundleFileType::Strings, - "surface_properties" => BundleFileType::SurfaceProperties, - "texture" => BundleFileType::Texture, - "timpani_bank" => BundleFileType::TimpaniBank, - "timpani_master" => BundleFileType::TimpaniMaster, - "tome" => BundleFileType::Tome, - "ugg" => BundleFileType::Ugg, - "unit" => BundleFileType::Unit, - "upb" => BundleFileType::Upb, - "vector_field" => BundleFileType::VectorField, - "wav" => BundleFileType::Wav, - "wwise_bank" => BundleFileType::WwiseBank, - "wwise_dep" => BundleFileType::WwiseDep, - "wwise_event" => BundleFileType::WwiseEvent, - "wwise_metadata" => BundleFileType::WwiseMetadata, - "wwise_stream" => BundleFileType::WwiseStream, - "xml" => BundleFileType::Xml, - s => eyre::bail!("Unknown type string '{}'", s), - }; - - Ok(val) - } -} - impl Serialize for BundleFileType { fn serialize(&self, serializer: S) -> Result where @@ -245,147 +160,6 @@ impl From for BundleFileType { } } -impl From for BundleFileType { - fn from(hash: u64) -> BundleFileType { - match hash { - 0x931e336d7646cc26 => BundleFileType::Animation, - 0xdcfb9e18fff13984 => BundleFileType::AnimationCurves, - 0x3eed05ba83af5090 => BundleFileType::Apb, - 0x7ffdb779b04e4ed1 => BundleFileType::BakedLighting, - 0xaa5965f03029fa18 => BundleFileType::Bik, - 0xe301e8af94e3b5a3 => BundleFileType::BlendSet, - 0x18dead01056b72e9 => BundleFileType::Bones, - 0xb7893adf7567506a => BundleFileType::Chroma, - 0xfe9754bd19814a47 => BundleFileType::CommonPackage, - 0x82645835e6b73232 => BundleFileType::Config, - 0x69108ded1e3e634b => BundleFileType::Crypto, - 0x8fd0d44d20650b68 => BundleFileType::Data, - 0x9831ca893b0d087d => BundleFileType::Entity, - 0x92d3ee038eeb610d => BundleFileType::Flow, - 0x9efe0a916aae7880 => BundleFileType::Font, - 0x8f7d5a2c0f967655 => BundleFileType::Ies, - 0xd526a27da14f1dc5 => BundleFileType::Ini, - 0x2bbcabe5074ade9e => BundleFileType::Input, - 0xfa4a8e091a91201e => BundleFileType::Ivf, - 0xa62f9297dc969e85 => BundleFileType::Keys, - 0x2a690fd348fe9ac5 => BundleFileType::Level, - 0xa14e8dfa2cd117e2 => BundleFileType::Lua, - 0xeac0b497876adedf => BundleFileType::Material, - 0x3fcdd69156a46417 => BundleFileType::Mod, - 0xb277b11fe4a61d37 => BundleFileType::MouseCursor, - 0x169de9566953d264 => BundleFileType::NavData, - 0x3b1fa9e8f6bac374 => BundleFileType::NetworkConfig, - 0xb0f2c12eb107f4d8 => BundleFileType::OddleNet, - 0xad9c6d9ed1e5e77a => BundleFileType::Package, - 0xa8193123526fad64 => BundleFileType::Particles, - 0xbf21403a3ab0bbb1 => BundleFileType::PhysicsProperties, - 0x27862fe24795319c => BundleFileType::RenderConfig, - 0x9ca183c2d0e76dee => BundleFileType::RtPipeline, - 0x9d0a795bfe818d19 => BundleFileType::Scene, - 0xcce8d5b5f5ae333f => BundleFileType::Shader, - 0xe5ee32a477239a93 => BundleFileType::ShaderLibrary, - 0x9e5c3cc74575aeb5 => BundleFileType::ShaderLibraryGroup, - 0x250e0a11ac8e26f8 => BundleFileType::ShadingEnvionmentMapping, - 0xfe73c7dcff8a7ca5 => BundleFileType::ShadingEnvironment, - 0xa27b4d04a9ba6f9e => BundleFileType::Slug, - 0xe9fc9ea7042e5ec0 => BundleFileType::SlugAlbum, - 0xd8b27864a97ffdd7 => BundleFileType::SoundEnvironment, - 0xf97af9983c05b950 => BundleFileType::SpuJob, - 0xa486d4045106165c => BundleFileType::StateMachine, - 0xe3f0baa17d620321 => BundleFileType::StaticPVS, - 0x0d972bab10b40fd3 => BundleFileType::Strings, - 0xad2d3fa30d9ab394 => BundleFileType::SurfaceProperties, - 0xcd4238c6a0c69e32 => BundleFileType::Texture, - 0x99736be1fff739a4 => BundleFileType::TimpaniBank, - 0x00a3e6c59a2b9c6c => BundleFileType::TimpaniMaster, - 0x19c792357c99f49b => BundleFileType::Tome, - 0x712d6e3dd1024c9c => BundleFileType::Ugg, - 0xe0a48d0be9a7453f => BundleFileType::Unit, - 0xa99510c6e86dd3c2 => BundleFileType::Upb, - 0xf7505933166d6755 => BundleFileType::VectorField, - 0x786f65c00a816b19 => BundleFileType::Wav, - 0x535a7bd3e650d799 => BundleFileType::WwiseBank, - 0xaf32095c82f2b070 => BundleFileType::WwiseDep, - 0xaabdd317b58dfc8a => BundleFileType::WwiseEvent, - 0xd50a8b7e1c82b110 => BundleFileType::WwiseMetadata, - 0x504b55235d21440e => BundleFileType::WwiseStream, - 0x76015845a6003765 => BundleFileType::Xml, - - _ => BundleFileType::Unknown(Murmur64::from(hash)), - } - } -} - -impl From for u64 { - fn from(t: BundleFileType) -> u64 { - match t { - BundleFileType::Animation => 0x931e336d7646cc26, - BundleFileType::AnimationCurves => 0xdcfb9e18fff13984, - BundleFileType::Apb => 0x3eed05ba83af5090, - BundleFileType::BakedLighting => 0x7ffdb779b04e4ed1, - BundleFileType::Bik => 0xaa5965f03029fa18, - BundleFileType::BlendSet => 0xe301e8af94e3b5a3, - BundleFileType::Bones => 0x18dead01056b72e9, - BundleFileType::Chroma => 0xb7893adf7567506a, - BundleFileType::CommonPackage => 0xfe9754bd19814a47, - BundleFileType::Config => 0x82645835e6b73232, - BundleFileType::Crypto => 0x69108ded1e3e634b, - BundleFileType::Data => 0x8fd0d44d20650b68, - BundleFileType::Entity => 0x9831ca893b0d087d, - BundleFileType::Flow => 0x92d3ee038eeb610d, - BundleFileType::Font => 0x9efe0a916aae7880, - BundleFileType::Ies => 0x8f7d5a2c0f967655, - BundleFileType::Ini => 0xd526a27da14f1dc5, - BundleFileType::Input => 0x2bbcabe5074ade9e, - BundleFileType::Ivf => 0xfa4a8e091a91201e, - BundleFileType::Keys => 0xa62f9297dc969e85, - BundleFileType::Level => 0x2a690fd348fe9ac5, - BundleFileType::Lua => 0xa14e8dfa2cd117e2, - BundleFileType::Material => 0xeac0b497876adedf, - BundleFileType::Mod => 0x3fcdd69156a46417, - BundleFileType::MouseCursor => 0xb277b11fe4a61d37, - BundleFileType::NavData => 0x169de9566953d264, - BundleFileType::NetworkConfig => 0x3b1fa9e8f6bac374, - BundleFileType::OddleNet => 0xb0f2c12eb107f4d8, - BundleFileType::Package => 0xad9c6d9ed1e5e77a, - BundleFileType::Particles => 0xa8193123526fad64, - BundleFileType::PhysicsProperties => 0xbf21403a3ab0bbb1, - BundleFileType::RenderConfig => 0x27862fe24795319c, - BundleFileType::RtPipeline => 0x9ca183c2d0e76dee, - BundleFileType::Scene => 0x9d0a795bfe818d19, - BundleFileType::Shader => 0xcce8d5b5f5ae333f, - BundleFileType::ShaderLibrary => 0xe5ee32a477239a93, - BundleFileType::ShaderLibraryGroup => 0x9e5c3cc74575aeb5, - BundleFileType::ShadingEnvionmentMapping => 0x250e0a11ac8e26f8, - BundleFileType::ShadingEnvironment => 0xfe73c7dcff8a7ca5, - BundleFileType::Slug => 0xa27b4d04a9ba6f9e, - BundleFileType::SlugAlbum => 0xe9fc9ea7042e5ec0, - BundleFileType::SoundEnvironment => 0xd8b27864a97ffdd7, - BundleFileType::SpuJob => 0xf97af9983c05b950, - BundleFileType::StateMachine => 0xa486d4045106165c, - BundleFileType::StaticPVS => 0xe3f0baa17d620321, - BundleFileType::Strings => 0x0d972bab10b40fd3, - BundleFileType::SurfaceProperties => 0xad2d3fa30d9ab394, - BundleFileType::Texture => 0xcd4238c6a0c69e32, - BundleFileType::TimpaniBank => 0x99736be1fff739a4, - BundleFileType::TimpaniMaster => 0x00a3e6c59a2b9c6c, - BundleFileType::Tome => 0x19c792357c99f49b, - BundleFileType::Ugg => 0x712d6e3dd1024c9c, - BundleFileType::Unit => 0xe0a48d0be9a7453f, - BundleFileType::Upb => 0xa99510c6e86dd3c2, - BundleFileType::VectorField => 0xf7505933166d6755, - BundleFileType::Wav => 0x786f65c00a816b19, - BundleFileType::WwiseBank => 0x535a7bd3e650d799, - BundleFileType::WwiseDep => 0xaf32095c82f2b070, - BundleFileType::WwiseEvent => 0xaabdd317b58dfc8a, - BundleFileType::WwiseMetadata => 0xd50a8b7e1c82b110, - BundleFileType::WwiseStream => 0x504b55235d21440e, - BundleFileType::Xml => 0x76015845a6003765, - - BundleFileType::Unknown(hash) => hash.into(), - } - } -} impl From for Murmur64 { fn from(t: BundleFileType) -> Murmur64 { let hash: u64 = t.into(); From 7b95918000430088139af81b9aea2d540e25f98b Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 22 Sep 2023 15:42:16 +0200 Subject: [PATCH 16/35] Refactor code for file injection I ended up wrapping the raw data in a `BundleFile` twice. I also made '--compile' the default, as it should be much less often that raw data needs to be inserted. Even files that are essentially raw binary blobs, like `.wwise_event`, still have some custom fields that need to be accounted for. --- crates/dtmt/src/cmd/bundle/inject.rs | 327 +++++++++++++++++++++------ lib/sdk/src/bundle/file.rs | 23 +- lib/sdk/src/bundle/mod.rs | 3 +- lib/sdk/src/lib.rs | 2 +- 4 files changed, 277 insertions(+), 78 deletions(-) diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 2b86691..21f4a91 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -1,112 +1,297 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::str::FromStr as _; -use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::{self, Context, Result}; +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::{self, Context, OptionExt, Result}; use color_eyre::Help; -use sdk::Bundle; -use tokio::fs::{self, File}; -use tokio::io::AsyncReadExt; +use path_slash::PathBufExt as _; +use sdk::murmur::IdString64; +use sdk::{Bundle, BundleFile, BundleFileType}; +use tokio::fs; pub(crate) fn command_definition() -> Command { Command::new("inject") - .about("Inject a file into a bundle.") - .arg( - Arg::new("replace") - .help("The name of a file in the bundle whos content should be replaced.") - .short('r') - .long("replace"), - ) + .subcommand_required(true) + .about("Inject a file into a bundle.\n\ + Raw binary data can be used to directly replace the file's variant data blob without affecting the metadata.\n\ + Alternatively, a compiler format may be specified, and a complete bundle file is created.") .arg( Arg::new("output") .help( "The path to write the changed bundle to. \ - If omitted, the input bundle will be overwritten.", + If omitted, the input bundle will be overwritten.\n\ + Remember to add a `.patch_` suffix if you also use '--patch'.", ) .short('o') .long("output") .value_parser(value_parser!(PathBuf)), ) .arg( - Arg::new("bundle") - .help("Path to the bundle to inject the file into.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("patch") + .help("Create a patch bundle. Optionally, a patch NUMBER may be specified as \ + '--patch=123'.\nThe maximum number is 999, the default is 1.\n\ + If `--output` is not specified, the `.patch_` suffix is added to \ + the given bundle name.") + .short('p') + .long("patch") + .num_args(0..=1) + .require_equals(true) + .default_missing_value("1") + .value_name("NUMBER") + .value_parser(value_parser!(u16)) ) .arg( - Arg::new("file") - .help("Path to the file to inject.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("type") + .help("Compile the new file as the given TYPE. If omitted, the file type is \ + is guessed from the file extension.") + .value_name("TYPE") ) + .subcommand( + Command::new("replace") + .about("Replace an existing file in the bundle") + .arg( + Arg::new("variant") + .help("In combination with '--raw', specify the variant index to replace.") + .long("variant") + .default_value("0") + .value_parser(value_parser!(u8)) + ) + .arg( + Arg::new("raw") + .help("Insert the given file as raw binary data.\n\ + Cannot be used with '--patch'.") + .long("raw") + .action(ArgAction::SetTrue) + ) + .arg( + Arg::new("bundle") + .help("Path to the bundle to inject the file into.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ) + .arg( + Arg::new("bundle-file") + .help("The name of a file in the bundle whose content should be replaced.") + .required(true), + ) + .arg( + Arg::new("new-file") + .help("Path to the file to inject.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ), + ) + // .subcommand( + // Command::new("add") + // .about("Add a new file to the bundle") + // .arg( + // Arg::new("new-file") + // .help("Path to the file to inject.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ) + // .arg( + // Arg::new("bundle") + // .help("Path to the bundle to inject the file into.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ), + // ) } -#[tracing::instrument(skip_all)] +#[tracing::instrument] +async fn compile_file( + path: impl AsRef + std::fmt::Debug, + name: impl Into + std::fmt::Debug, + file_type: BundleFileType, +) -> Result { + let path = path.as_ref(); + + let file_data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?; + let _sjson = String::from_utf8(file_data) + .wrap_err_with(|| format!("Invalid UTF8 data in '{}'", path.display()))?; + + let _root = path.parent().ok_or_eyre("File path has no parent")?; + + eyre::bail!( + "Compilation for type '{}' is not implemented, yet", + file_type + ) +} + +#[tracing::instrument( + skip_all, + fields( + bundle_path = tracing::field::Empty, + in_file_path = tracing::field::Empty, + output_path = tracing::field::Empty, + target_name = tracing::field::Empty, + file_type = tracing::field::Empty, + raw = tracing::field::Empty, + ) +)] pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { - let bundle_path = matches + let Some((op, sub_matches)) = matches.subcommand() else { + unreachable!("clap is configured to require a subcommand, and they're all handled above"); + }; + + let bundle_path = sub_matches .get_one::("bundle") .expect("required parameter not found"); - let file_path = matches - .get_one::("file") + let in_file_path = sub_matches + .get_one::("new-file") .expect("required parameter not found"); - tracing::trace!(bundle_path = %bundle_path.display(), file_path = %file_path.display()); + let patch_number = matches + .get_one::("patch") + .map(|num| format!("{:03}", num)); - let mut bundle = { - let binary = fs::read(bundle_path).await?; - let name = Bundle::get_name_from_path(&ctx, bundle_path); - Bundle::from_binary(&ctx, name, binary).wrap_err("Failed to open bundle file")? + let output_path = matches + .get_one::("output") + .cloned() + .unwrap_or_else(|| { + let mut output_path = bundle_path.clone(); + + if let Some(patch_number) = patch_number.as_ref() { + output_path.set_extension(format!("patch_{:03}", patch_number)); + } + + output_path + }); + + let target_name = if op == "replace" { + sub_matches + .get_one::("bundle-file") + .map(|name| match u64::from_str_radix(name, 16) { + Ok(id) => IdString64::from(id), + Err(_) => IdString64::String(name.clone()), + }) + .expect("argument is required") + } else { + let mut path = PathBuf::from(in_file_path); + path.set_extension(""); + IdString64::from(path.to_slash_lossy().to_string()) }; - if let Some(name) = matches.get_one::("replace") { - let mut file = File::open(&file_path) - .await - .wrap_err_with(|| format!("Failed to open '{}'", file_path.display()))?; + let file_type = if let Some(forced_type) = matches.get_one::("type") { + BundleFileType::from_str(forced_type.as_str()).wrap_err("Unknown file type")? + } else { + in_file_path + .extension() + .and_then(|s| s.to_str()) + .ok_or_eyre("File extension missing") + .and_then(BundleFileType::from_str) + .wrap_err("Unknown file type") + .with_suggestion(|| "Use '--type TYPE' to specify the file type")? + }; - if let Some(variant) = bundle - .files_mut() - .filter(|file| file.matches_name(name.clone())) - // TODO: Handle file variants - .find_map(|file| file.variants_mut().next()) - { - let mut data = Vec::new(); - file.read_to_end(&mut data) - .await - .wrap_err("Failed to read input file")?; - variant.set_data(data); - } else { - let err = eyre::eyre!("No file '{}' in this bundle.", name) - .with_suggestion(|| { + { + let span = tracing::Span::current(); + if !span.is_disabled() { + span.record("bundle_path", bundle_path.display().to_string()); + span.record("in_file_path", in_file_path.display().to_string()); + span.record("output_path", output_path.display().to_string()); + span.record("raw", sub_matches.get_flag("raw")); + span.record("target_name", target_name.display().to_string()); + span.record("file_type", format!("{:?}", file_type)); + } + } + + let bundle_name = Bundle::get_name_from_path(&ctx, bundle_path); + let mut bundle = { + let binary = fs::read(bundle_path).await?; + Bundle::from_binary(&ctx, bundle_name.clone(), binary) + .wrap_err_with(|| format!("Failed to open bundle '{}'", bundle_path.display()))? + }; + + if op == "copy" { + unimplemented!("Implement copying a file from one bundle to the other."); + } + + let output_bundle = match op { + "replace" => { + let Some(file) = bundle + .files_mut() + .find(|file| *file.base_name() == target_name) + else { + let err = eyre::eyre!( + "No file with name '{}' in bundle '{}'", + target_name.display(), + bundle_path.display() + ); + + return Err(err).with_suggestion(|| { format!( - "Run '{} bundle list {}' to list the files in this bundle.", + "Run '{} bundle list \"{}\"' to list the files in this bundle.", clap::crate_name!(), bundle_path.display() ) - }) - .with_suggestion(|| { - format!( - "Use '{} bundle inject --add {} {} {}' to add it as a new file", - clap::crate_name!(), - name, - bundle_path.display(), - file_path.display() - ) }); + }; - return Err(err); + if sub_matches.get_flag("raw") { + let variant_index = sub_matches + .get_one::("variant") + .expect("argument with default missing"); + + let Some(variant) = file.variants_mut().nth(*variant_index as usize) else { + let err = eyre::eyre!( + "Variant index '{}' does not exist in '{}'", + variant_index, + target_name.display() + ); + + return Err(err).with_suggestion(|| { + format!( + "See '{} bundle inject add --help' if you want to add it as a new file", + clap::crate_name!(), + ) + }); + }; + + let data = tokio::fs::read(&in_file_path).await.wrap_err_with(|| { + format!("Failed to read file '{}'", in_file_path.display()) + })?; + variant.set_data(data); + file.set_modded(true); + bundle + } else { + let mut bundle_file = compile_file(in_file_path, target_name.clone(), file_type) + .await + .wrap_err("Failed to compile")?; + + bundle_file.set_modded(true); + + if patch_number.is_some() { + let mut output_bundle = Bundle::new(bundle_name); + output_bundle.add_file(bundle_file); + output_bundle + } else { + *file = bundle_file; + + dbg!(&file); + bundle + } + } } + "add" => { + unimplemented!("Implement adding a new file to the bundle."); + } + _ => unreachable!("no other operations exist"), + }; - let out_path = matches.get_one::("output").unwrap_or(bundle_path); - let data = bundle - .to_binary() - .wrap_err("Failed to write changed bundle to output")?; + let data = output_bundle + .to_binary() + .wrap_err("Failed to write changed bundle to output")?; - fs::write(out_path, &data) - .await - .wrap_err("Failed to write data to output file")?; + fs::write(&output_path, &data) + .await + .wrap_err_with(|| format!("Failed to write data to '{}'", output_path.display()))?; - Ok(()) - } else { - eyre::bail!("Currently, only the '--replace' operation is supported."); - } + tracing::info!("Modified bundle written to '{}'", output_path.display()); + + Ok(()) } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index f387409..6d49821 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -20,6 +20,7 @@ struct BundleFileHeader { len_data_file_name: usize, } +#[derive(Clone, Debug)] pub struct BundleFileVariant { property: u32, data: Vec, @@ -109,9 +110,12 @@ bitflags! { #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { const DATA = 0b100; + // A custom flag used by DTMT to signify a file altered by mods. + const MODDED = 1 << 31; } } +#[derive(Clone, Debug)] pub struct BundleFile { file_type: BundleFileType, name: IdString64, @@ -133,6 +137,18 @@ impl BundleFile { self.variants.push(variant) } + pub fn set_variants(&mut self, variants: Vec) { + self.variants = variants; + } + + pub fn set_props(&mut self, props: Properties) { + self.props = props; + } + + pub fn set_modded(&mut self, is_modded: bool) { + self.props.set(Properties::MODDED, is_modded); + } + #[tracing::instrument(name = "File::read", skip(ctx, r))] pub fn from_reader(ctx: &crate::Context, r: &mut R, props: Properties) -> Result where @@ -299,14 +315,13 @@ impl BundleFile { s } - pub fn matches_name(&self, name: impl Into) -> bool { - let name = name.into(); - if self.name == name { + pub fn matches_name(&self, name: &IdString64) -> bool { + if self.name == *name { return true; } if let IdString64::String(name) = name { - self.name(false, None) == name || self.name(true, None) == name + self.name(false, None) == *name || self.name(true, None) == *name } else { false } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 075f4d2..edb71bb 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -7,14 +7,13 @@ use color_eyre::{Help, Report, SectionExt}; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE}; use crate::binary::sync::*; -use crate::bundle::file::Properties; use crate::murmur::{HashGroup, IdString64, Murmur64}; pub(crate) mod database; pub(crate) mod file; pub(crate) mod filetype; -pub use file::{BundleFile, BundleFileVariant}; +pub use file::{BundleFile, BundleFileVariant, Properties}; pub use filetype::BundleFileType; #[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index a24b3bd..9b1806b 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -9,5 +9,5 @@ pub mod murmur; pub use binary::{FromBinary, ToBinary}; pub use bundle::database::BundleDatabase; pub use bundle::decompress; -pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant}; +pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant, Properties}; pub use context::{CmdLine, Context}; From f521e20f2b61aef35315081c7f6c78746acda679 Mon Sep 17 00:00:00 2001 From: Renovate Date: Thu, 15 May 2025 22:16:23 +0000 Subject: [PATCH 17/35] chore(deps): update rust crate csv-async to v1.3.1 --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c4ec109..88a9bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -704,9 +704,9 @@ dependencies = [ [[package]] name = "csv-async" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d37fe5b0d07f4a8260ce1e9a81413e88f459af0f2dfc55c15e96868a2f99c0f0" +checksum = "888dbb0f640d2c4c04e50f933885c7e9c95995d93cec90aba8735b4c610f26f1" dependencies = [ "cfg-if", "csv-core", @@ -2173,7 +2173,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] From 164cb7bc131199a5089e9a2cb8dffe89fabbc9ff Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 20 May 2025 12:46:31 +0000 Subject: [PATCH 18/35] chore(deps): update rust crate tempfile to v3.20.0 --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88a9bb9..bc37db3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1045,7 +1045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3195,7 +3195,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3710,15 +3710,15 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", "rustix 1.0.5", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4543,7 +4543,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] From ca677606fa25fd62466963e3e65efc7db82bb5f9 Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 20 May 2025 13:16:25 +0000 Subject: [PATCH 19/35] chore(deps): update rust crate bitflags to v2.9.1 --- Cargo.lock | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc37db3..aa01c87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,7 +233,7 @@ version = "0.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools", @@ -253,7 +253,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools", @@ -275,9 +275,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bitmaps" @@ -917,7 +917,7 @@ dependencies = [ "ansi-parser", "async-recursion", "bincode", - "bitflags 2.9.0", + "bitflags 2.9.1", "clap", "color-eyre", "colors-transform", @@ -1955,7 +1955,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "inotify-sys", "libc", ] @@ -2182,7 +2182,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "libc", "redox_syscall", ] @@ -2445,7 +2445,7 @@ version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "filetime", "fsevent-sys", "inotify", @@ -2548,7 +2548,7 @@ version = "0.10.66" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "foreign-types", "libc", @@ -2986,7 +2986,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -3178,7 +3178,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.4.14", @@ -3191,7 +3191,7 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", @@ -3325,7 +3325,7 @@ name = "sdk" version = "0.3.0" dependencies = [ "async-recursion", - "bitflags 2.9.0", + "bitflags 2.9.1", "byteorder", "color-eyre", "csv-async", @@ -3352,7 +3352,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation", "core-foundation-sys", "libc", @@ -3674,7 +3674,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation", "system-configuration-sys", ] @@ -4851,7 +4851,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] From 1975435805f5aded7a9eaf3cb9c0a6eefd1c9ff0 Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 20 May 2025 13:16:29 +0000 Subject: [PATCH 20/35] chore(deps): update rust crate clap to v4.5.38 --- Cargo.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc37db3..ed45902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", "clap_derive", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstream", "anstyle", @@ -1045,7 +1045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3195,7 +3195,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3718,7 +3718,7 @@ dependencies = [ "getrandom 0.3.2", "once_cell", "rustix 1.0.5", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From 1e9738c953051cd678000ab78e2cd01b2428e30c Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 20 May 2025 13:16:37 +0000 Subject: [PATCH 21/35] chore(deps): update rust crate tokio to v1.45.0 --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc37db3..29df0e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1045,7 +1045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3195,7 +3195,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3718,7 +3718,7 @@ dependencies = [ "getrandom 0.3.2", "once_cell", "rustix 1.0.5", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3850,9 +3850,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", From 5e1581b428f028ef637c05b2d10598d6bbb148fc Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 21 May 2025 08:31:36 +0000 Subject: [PATCH 22/35] chore(deps): update rust crate minijinja to v2.10.2 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eb31bc..7fb7b22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2302,9 +2302,9 @@ dependencies = [ [[package]] name = "minijinja" -version = "2.9.0" +version = "2.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98642a6dfca91122779a307b77cd07a4aa951fbe32232aaf5bad9febc66be754" +checksum = "dd72e8b4e42274540edabec853f607c015c73436159b06c39c7af85a20433155" dependencies = [ "serde", ] From 14eded5b7e1d3b43bcbd6b5d04860fe69ee4ea42 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 21 May 2025 08:31:45 +0000 Subject: [PATCH 23/35] chore(deps): update rust crate zip to v3 --- Cargo.lock | 41 +++++++++++++++++++++++++---------------- Cargo.toml | 2 +- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eb31bc..212dc5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -686,12 +686,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - [[package]] name = "crypto-common" version = "0.1.6" @@ -1118,12 +1112,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.32" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "libz-rs-sys", + "miniz_oxide 0.8.8", ] [[package]] @@ -2173,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2187,6 +2182,15 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -2327,9 +2331,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -4543,7 +4547,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4965,14 +4969,13 @@ dependencies = [ [[package]] name = "zip" -version = "2.6.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dcb24d0152526ae49b9b96c1dcf71850ca1e0b882e4e28ed898a93c41334744" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" dependencies = [ "arbitrary", "bzip2", "crc32fast", - "crossbeam-utils", "flate2", "indexmap", "memchr", @@ -4981,6 +4984,12 @@ dependencies = [ "zstd", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zopfli" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 87d9ea6..4b083a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ tracing = { version = "0.1.37", features = ["async-await"] } tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } usvg = "0.25.0" -zip = { version = "2.1.3", default-features = false, features = ["deflate", "bzip2", "zstd", "time"] } +zip = { version = "3.0.0", default-features = false, features = ["deflate", "bzip2", "zstd", "time"] } [profile.dev.package.backtrace] opt-level = 3 From 220f37c7288c16e84c7a24a0c985b0d73cf99dc1 Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 24 May 2025 15:01:28 +0000 Subject: [PATCH 24/35] chore(deps): update rust crate tokio to v1.45.1 --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee85157..555a11a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3854,9 +3854,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", From 3901355f9d89104d5c9f461609852702ba4462f7 Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 27 May 2025 18:16:25 +0000 Subject: [PATCH 25/35] chore(deps): update rust crate clap to v4.5.39 --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee85157..09c09c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,9 +426,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", "clap_derive", @@ -436,9 +436,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.38" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", @@ -2168,7 +2168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] From d66dcb5cfd9a83f376c389add3eca4d71a575fd6 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 28 May 2025 16:31:22 +0000 Subject: [PATCH 26/35] fix(deps): update rust crate reqwest to v0.12.18 --- Cargo.lock | 73 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee85157..e9263f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1742,22 +1742,28 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +checksum = "b1c293b6b3d21eca78250dc7dbebd6b9210ec5530e038cbfe0661b5c47ab06e8" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -2014,6 +2020,16 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -2168,7 +2184,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3050,15 +3066,14 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "e98ff6b0dbbe4d5a37318f433d4fc82babd21631f194d370409ceb2e40b2f0b5" dependencies = [ "base64 0.22.1", "bytes", "encoding_rs", "futures-core", - "futures-util", "h2", "http", "http-body", @@ -3075,21 +3090,20 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows-registry", ] [[package]] @@ -3215,21 +3229,14 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-pemfile" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" -dependencies = [ - "base64 0.22.1", - "rustls-pki-types", -] - [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] [[package]] name = "rustls-webpki" @@ -3674,9 +3681,9 @@ dependencies = [ [[package]] name = "system-configuration" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ "bitflags 2.9.1", "core-foundation", @@ -3995,6 +4002,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +dependencies = [ + "bitflags 2.9.1", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -4547,7 +4572,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] From 138cb79ff6285b6ea504d287e2c4509275cdeaad Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 30 May 2025 11:52:44 +0200 Subject: [PATCH 27/35] Disable excessive rebase for Renovate --- .renovaterc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.renovaterc b/.renovaterc index 4a2fbf4..7ad59cd 100644 --- a/.renovaterc +++ b/.renovaterc @@ -3,8 +3,7 @@ "extends": [ "config:recommended", ":combinePatchMinorReleases", - ":enableVulnerabilityAlerts", - ":rebaseStalePrs" + ":enableVulnerabilityAlerts" ], "prConcurrentLimit": 10, "branchPrefix": "renovate/", From db27dd9f395e3a654ce2cfeeab690e46f73c8ee0 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 3 Mar 2023 16:55:22 +0100 Subject: [PATCH 28/35] feat(sdk): Implement partial texture decompilation --- crates/dtmt/src/cmd/bundle/extract.rs | 28 ++ crates/dtmt/src/cmd/dictionary.rs | 11 +- crates/dtmt/src/cmd/experiment/mod.rs | 21 + .../dtmt/src/cmd/experiment/texture_meta.rs | 117 ++++++ crates/dtmt/src/main.rs | 9 +- lib/oodle/src/lib.rs | 3 +- lib/sdk/src/binary.rs | 26 +- lib/sdk/src/bundle/file.rs | 69 +++- lib/sdk/src/bundle/filetype.rs | 3 +- lib/sdk/src/bundle/mod.rs | 2 + lib/sdk/src/context.rs | 9 +- lib/sdk/src/filetype/mod.rs | 1 + lib/sdk/src/filetype/texture.rs | 387 ++++++++++++++++++ lib/sdk/src/murmur/dictionary.rs | 12 +- 14 files changed, 660 insertions(+), 38 deletions(-) create mode 100644 crates/dtmt/src/cmd/experiment/mod.rs create mode 100644 crates/dtmt/src/cmd/experiment/texture_meta.rs create mode 100644 lib/sdk/src/filetype/texture.rs diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index 75f1360..b595dba 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -287,6 +287,34 @@ where P1: AsRef + std::fmt::Debug, P2: AsRef + std::fmt::Debug, { + let ctx = if ctx.game_dir.is_some() { + tracing::debug!( + "Got game directory from config: {}", + ctx.game_dir.as_ref().unwrap().display() + ); + + ctx + } else { + let game_dir = path + .as_ref() + .parent() + .and_then(|parent| parent.parent()) + .map(|p| p.to_path_buf()); + + tracing::info!( + "No game directory configured, guessing from bundle path: {:?}", + game_dir + ); + + Arc::new(sdk::Context { + game_dir, + lookup: Arc::clone(&ctx.lookup), + ljd: ctx.ljd.clone(), + revorb: ctx.revorb.clone(), + ww2ogg: ctx.ww2ogg.clone(), + }) + }; + let bundle = { let data = fs::read(path.as_ref()).await?; let name = Bundle::get_name_from_path(&ctx, path.as_ref()); diff --git a/crates/dtmt/src/cmd/dictionary.rs b/crates/dtmt/src/cmd/dictionary.rs index 4c54c34..8f0d32c 100644 --- a/crates/dtmt/src/cmd/dictionary.rs +++ b/crates/dtmt/src/cmd/dictionary.rs @@ -1,4 +1,5 @@ use std::path::PathBuf; +use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command, ValueEnum}; use cli_table::{print_stdout, WithTitle}; @@ -156,6 +157,8 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( BufReader::new(Box::new(f)) }; + let lookup = Arc::make_mut(&mut ctx.lookup); + let group = sdk::murmur::HashGroup::from(*group); let mut added = 0; @@ -165,15 +168,15 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( let total = { for line in lines.into_iter() { let value = line?; - if ctx.lookup.find(&value, group).is_some() { + if lookup.find(&value, group).is_some() { skipped += 1; } else { - ctx.lookup.add(value, group); + lookup.add(value, group); added += 1; } } - ctx.lookup.len() + lookup.len() }; let out_path = matches @@ -190,7 +193,7 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( }) .with_section(|| out_path.display().to_string().header("Path:"))?; - ctx.lookup + lookup .to_csv(f) .await .wrap_err("Failed to write dictionary to disk")?; diff --git a/crates/dtmt/src/cmd/experiment/mod.rs b/crates/dtmt/src/cmd/experiment/mod.rs new file mode 100644 index 0000000..50ba706 --- /dev/null +++ b/crates/dtmt/src/cmd/experiment/mod.rs @@ -0,0 +1,21 @@ +use clap::{ArgMatches, Command}; +use color_eyre::Result; + +mod texture_meta; + +pub(crate) fn command_definition() -> Command { + Command::new("experiment") + .subcommand_required(true) + .about("A collection of utilities and experiments.") + .subcommand(texture_meta::command_definition()) +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + match matches.subcommand() { + Some(("texture-meta", sub_matches)) => texture_meta::run(ctx, sub_matches).await, + _ => unreachable!( + "clap is configured to require a subcommand, and they're all handled above" + ), + } +} diff --git a/crates/dtmt/src/cmd/experiment/texture_meta.rs b/crates/dtmt/src/cmd/experiment/texture_meta.rs new file mode 100644 index 0000000..98d0035 --- /dev/null +++ b/crates/dtmt/src/cmd/experiment/texture_meta.rs @@ -0,0 +1,117 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::Context; +use color_eyre::Result; +use futures_util::StreamExt; +use sdk::Bundle; +use tokio::fs; + +use crate::cmd::util::resolve_bundle_paths; + +pub(crate) fn command_definition() -> Command { + Command::new("texture-meta") + .about( + "Iterates over the provided bundles and lists certain meta data. + Primarily intended to help spot patterns between dependend data fields and values.", + ) + .arg( + Arg::new("bundle") + .required(true) + .action(ArgAction::Append) + .value_parser(value_parser!(PathBuf)) + .help( + "Path to the bundle(s) to read. If this points to a directory instead \ + of a file, all files in that directory will be checked.", + ), + ) + // TODO: Maybe provide JSON and CSV + // TODO: Maybe allow toggling certain fields +} + +#[tracing::instrument(skip(ctx))] +async fn handle_bundle(ctx: &sdk::Context, path: &PathBuf) -> Result<()> { + let bundle = { + let binary = fs::read(path).await?; + let name = Bundle::get_name_from_path(ctx, path); + Bundle::from_binary(ctx, name, binary)? + }; + + let bundle_dir = ctx + .game_dir + .as_deref() + .map(|dir| dir.join("bundle")) + .or_else(|| path.parent().map(|p| p.to_path_buf())) + .unwrap_or_default(); + + for f in bundle.files().iter() { + for (i, v) in f.variants().iter().enumerate() { + let data_file_name = v.data_file_name(); + + let data_file_length = if let Some(file_name) = data_file_name { + let path = bundle_dir.join(file_name); + + match fs::metadata(&path).await { + Ok(meta) => meta.len(), + Err(err) => { + return Err(err).wrap_err_with(|| { + format!("Failed to open data file {}", path.display()) + }) + } + } + } else { + 0 + }; + + println!( + "{},{},{},{},{:b},{},{},{:?},{},{:#010b}", + bundle.name().display(), + f.name(false, None), + f.file_type().ext_name(), + i, + v.property(), + v.data().len(), + v.external(), + data_file_name, + data_file_length, + v.unknown_1(), + ); + } + } + + Ok(()) +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + let bundles = matches + .get_many::("bundle") + .unwrap_or_default() + .cloned(); + + let paths = resolve_bundle_paths(bundles); + + let ctx = Arc::new(ctx); + + println!( + "Bundle Name,File Name,File Type,Variant,Property,Bundle Data Length,External,Data File,Data File Length,Unknown 1" + ); + + paths + .for_each_concurrent(10, |p| async { + let ctx = ctx.clone(); + async move { + if let Err(err) = handle_bundle(&ctx, &p) + .await + .wrap_err_with(|| format!("Failed to list contents of bundle {}", p.display())) + { + tracing::error!("Failed to handle bundle: {}", format!("{:#}", err)); + } + } + .await; + }) + .await; + + Ok(()) +} diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index e41e802..04a42fe 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -12,6 +12,7 @@ use clap::value_parser; use clap::{command, Arg}; use color_eyre::eyre; use color_eyre::eyre::{Context, Result}; +use sdk::murmur::Dictionary; use serde::{Deserialize, Serialize}; use tokio::fs::File; use tokio::io::BufReader; @@ -21,6 +22,7 @@ mod cmd { pub mod build; pub mod bundle; pub mod dictionary; + pub mod experiment; pub mod migrate; pub mod murmur; pub mod new; @@ -67,6 +69,7 @@ async fn main() -> Result<()> { .subcommand(cmd::build::command_definition()) .subcommand(cmd::bundle::command_definition()) .subcommand(cmd::dictionary::command_definition()) + .subcommand(cmd::experiment::command_definition()) .subcommand(cmd::migrate::command_definition()) .subcommand(cmd::murmur::command_definition()) .subcommand(cmd::new::command_definition()) @@ -107,8 +110,9 @@ async fn main() -> Result<()> { let r = BufReader::new(f); let mut ctx = ctx.write().await; - if let Err(err) = ctx.lookup.from_csv(r).await { - tracing::error!("{:#}", err); + match Dictionary::from_csv(r).await { + Ok(lookup) => ctx.lookup = Arc::new(lookup), + Err(err) => tracing::error!("{:#}", err), } }) }; @@ -144,6 +148,7 @@ async fn main() -> Result<()> { Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?, Some(("bundle", sub_matches)) => cmd::bundle::run(ctx, sub_matches).await?, Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?, + Some(("experiment", sub_matches)) => cmd::experiment::run(ctx, sub_matches).await?, Some(("migrate", sub_matches)) => cmd::migrate::run(ctx, sub_matches).await?, Some(("murmur", sub_matches)) => cmd::murmur::run(ctx, sub_matches).await?, Some(("new", sub_matches)) => cmd::new::run(ctx, sub_matches).await?, diff --git a/lib/oodle/src/lib.rs b/lib/oodle/src/lib.rs index 871daab..7edadee 100644 --- a/lib/oodle/src/lib.rs +++ b/lib/oodle/src/lib.rs @@ -52,6 +52,7 @@ impl From for bindings::OodleLZ_CheckCRC { #[tracing::instrument(skip(data))] pub fn decompress( data: I, + out_size: usize, fuzz_safe: OodleLZ_FuzzSafe, check_crc: OodleLZ_CheckCRC, ) -> Result> @@ -59,7 +60,7 @@ where I: AsRef<[u8]>, { let data = data.as_ref(); - let mut out = vec![0; CHUNK_SIZE]; + let mut out = vec![0; out_size]; let verbosity = if tracing::enabled!(tracing::Level::INFO) { bindings::OodleLZ_Verbosity_OodleLZ_Verbosity_Minimal diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 9348e1b..83ccca0 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -44,10 +44,10 @@ impl FromBinary for Vec { pub mod sync { use std::ffi::CStr; - use std::io::{self, Read, Seek, SeekFrom}; + use std::io::{self, Read, Seek, SeekFrom, Write}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; - use color_eyre::eyre::WrapErr; + use color_eyre::eyre::{self, WrapErr}; use color_eyre::{Help, Report, Result, SectionExt}; macro_rules! make_read { @@ -123,15 +123,17 @@ pub mod sync { }; } - pub trait ReadExt: ReadBytesExt + Seek { + pub trait ReadExt: Read + Seek { fn read_u8(&mut self) -> io::Result { ReadBytesExt::read_u8(self) } + make_read!(read_u16, read_u16_le, u16); make_read!(read_u32, read_u32_le, u32); make_read!(read_u64, read_u64_le, u64); make_skip!(skip_u8, read_u8, u8); + make_skip!(skip_u16, read_u16, u16); make_skip!(skip_u32, read_u32, u32); // Implementation based on https://en.wikipedia.com/wiki/LEB128 @@ -181,9 +183,17 @@ pub mod sync { res } } + + fn read_bool(&mut self) -> Result { + match ReadExt::read_u8(self)? { + 0 => Ok(false), + 1 => Ok(true), + v => eyre::bail!("Invalid value for boolean '{}'", v), + } + } } - pub trait WriteExt: WriteBytesExt + Seek { + pub trait WriteExt: Write + Seek { fn write_u8(&mut self, val: u8) -> io::Result<()> { WriteBytesExt::write_u8(self, val) } @@ -191,6 +201,10 @@ pub mod sync { make_write!(write_u32, write_u32_le, u32); make_write!(write_u64, write_u64_le, u64); + fn write_bool(&mut self, val: bool) -> io::Result<()> { + WriteBytesExt::write_u8(self, if val { 1 } else { 0 }) + } + fn write_padding(&mut self) -> io::Result { let pos = self.stream_position()?; let size = 16 - (pos % 16) as usize; @@ -207,8 +221,8 @@ pub mod sync { } } - impl ReadExt for R {} - impl WriteExt for W {} + impl ReadExt for R {} + impl WriteExt for W {} pub(crate) fn _read_up_to(r: &mut R, buf: &mut Vec) -> Result where diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index 6d49821..0d0a99f 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -15,17 +15,18 @@ use super::filetype::BundleFileType; #[derive(Debug)] struct BundleFileHeader { variant: u32, - unknown_1: u8, + external: bool, size: usize, + unknown_1: u8, len_data_file_name: usize, } -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct BundleFileVariant { property: u32, data: Vec, data_file_name: Option, - // Seems to be related to whether there is a data path. + external: bool, unknown_1: u8, } @@ -39,6 +40,7 @@ impl BundleFileVariant { property: 0, data: Vec::new(), data_file_name: None, + external: false, unknown_1: 0, } } @@ -63,21 +65,30 @@ impl BundleFileVariant { self.data_file_name.as_ref() } + pub fn external(&self) -> bool { + self.external + } + + pub fn unknown_1(&self) -> u8 { + self.unknown_1 + } + #[tracing::instrument(skip_all)] fn read_header(r: &mut R) -> Result where R: Read + Seek, { let variant = r.read_u32()?; - let unknown_1 = r.read_u8()?; + let external = r.read_bool()?; let size = r.read_u32()? as usize; - r.skip_u8(1)?; + let unknown_1 = r.read_u8()?; let len_data_file_name = r.read_u32()? as usize; Ok(BundleFileHeader { size, - unknown_1, + external, variant, + unknown_1, len_data_file_name, }) } @@ -88,7 +99,7 @@ impl BundleFileVariant { W: Write + Seek, { w.write_u32(self.property)?; - w.write_u8(self.unknown_1)?; + w.write_bool(self.external)?; let len_data_file_name = self.data_file_name.as_ref().map(|s| s.len()).unwrap_or(0); @@ -106,6 +117,26 @@ impl BundleFileVariant { } } +impl std::fmt::Debug for BundleFileVariant { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut out = f.debug_struct("BundleFileVariant"); + out.field("property", &self.property); + + if self.data.len() <= 5 { + out.field("data", &format!("{:x?}", &self.data)); + } else { + out.field( + "data", + &format!("{:x?}.. ({} bytes)", &self.data[..5], &self.data.len()), + ); + } + + out.field("data_file_name", &self.data_file_name) + .field("external", &self.external) + .finish() + } +} + bitflags! { #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { @@ -204,6 +235,7 @@ impl BundleFile { let s = r .read_string_len(header.len_data_file_name) .wrap_err("Failed to read data file name")?; + Some(s) } else { None @@ -216,6 +248,7 @@ impl BundleFile { property: header.variant, data, data_file_name, + external: header.external, unknown_1: header.unknown_1, }; @@ -243,7 +276,7 @@ impl BundleFile { for variant in self.variants.iter() { w.write_u32(variant.property())?; - w.write_u8(variant.unknown_1)?; + w.write_bool(variant.external)?; let len_data_file_name = variant.data_file_name().map(|s| s.len()).unwrap_or(0); @@ -277,6 +310,9 @@ impl BundleFile { ) -> Result { match file_type { BundleFileType::Lua => lua::compile(name, sjson).wrap_err("Failed to compile Lua file"), + BundleFileType::Texture => texture::compile(name, sjson, root) + .await + .wrap_err("Failed to compile Texture file"), BundleFileType::Unknown(_) => { eyre::bail!("Unknown file type. Cannot compile from SJSON"); } @@ -359,18 +395,16 @@ impl BundleFile { Ok(files) } - #[tracing::instrument(name = "File::decompiled", skip_all)] + #[tracing::instrument( + name = "File::decompiled", + skip_all, + fields(file = self.name(false, None), file_type = self.file_type().ext_name(), variants = self.variants.len()) + )] pub async fn decompiled(&self, ctx: &crate::Context) -> Result> { let file_type = self.file_type(); - if tracing::enabled!(tracing::Level::DEBUG) { - tracing::debug!( - name = self.name(true, None), - variants = self.variants.len(), - "Attempting to decompile" - ); - } - + // The `Strings` type handles all variants combined. + // For the other ones, each variant will be its own file. if file_type == BundleFileType::Strings { return strings::decompile(ctx, &self.variants); } @@ -386,6 +420,7 @@ impl BundleFile { let res = match file_type { BundleFileType::Lua => lua::decompile(ctx, data).await, BundleFileType::Package => package::decompile(ctx, name.clone(), data), + BundleFileType::Texture => texture::decompile(ctx, name.clone(), variant).await, _ => { tracing::debug!("Can't decompile, unknown file type"); Ok(vec![UserFile::with_name(data.to_vec(), name.clone())]) diff --git a/lib/sdk/src/bundle/filetype.rs b/lib/sdk/src/bundle/filetype.rs index 68ff6b5..a7a25dc 100644 --- a/lib/sdk/src/bundle/filetype.rs +++ b/lib/sdk/src/bundle/filetype.rs @@ -1,4 +1,5 @@ -use color_eyre::{eyre, Result}; +use color_eyre::eyre; +use color_eyre::Result; use serde::Serialize; use crate::murmur::Murmur64; diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index edb71bb..03a04c7 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -162,6 +162,7 @@ impl Bundle { // TODO: Optimize to not reallocate? let mut raw_buffer = oodle::decompress( &compressed_buffer, + oodle::CHUNK_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, ) @@ -359,6 +360,7 @@ where // TODO: Optimize to not reallocate? let mut raw_buffer = oodle::decompress( &compressed_buffer, + oodle::CHUNK_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, )?; diff --git a/lib/sdk/src/context.rs b/lib/sdk/src/context.rs index 1500290..8c10b3c 100644 --- a/lib/sdk/src/context.rs +++ b/lib/sdk/src/context.rs @@ -1,8 +1,11 @@ +use std::ffi::OsString; +use std::path::PathBuf; use std::process::Command; -use std::{ffi::OsString, path::PathBuf}; +use std::sync::Arc; use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64}; +#[derive(Clone)] pub struct CmdLine { cmd: OsString, args: Vec, @@ -52,7 +55,7 @@ impl From<&CmdLine> for Command { } pub struct Context { - pub lookup: Dictionary, + pub lookup: Arc, pub ljd: Option, pub revorb: Option, pub ww2ogg: Option, @@ -62,7 +65,7 @@ pub struct Context { impl Context { pub fn new() -> Self { Self { - lookup: Dictionary::new(), + lookup: Arc::new(Dictionary::new()), ljd: None, revorb: None, ww2ogg: None, diff --git a/lib/sdk/src/filetype/mod.rs b/lib/sdk/src/filetype/mod.rs index c62c503..b837993 100644 --- a/lib/sdk/src/filetype/mod.rs +++ b/lib/sdk/src/filetype/mod.rs @@ -1,3 +1,4 @@ pub mod lua; pub mod package; pub mod strings; +pub mod texture; diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs new file mode 100644 index 0000000..a8c85cd --- /dev/null +++ b/lib/sdk/src/filetype/texture.rs @@ -0,0 +1,387 @@ +use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use bitflags::bitflags; +use color_eyre::eyre::Context; +use color_eyre::{eyre, SectionExt}; +use color_eyre::{Help, Result}; +use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; +use serde::Deserialize; +use tokio::fs; + +use crate::binary::sync::{ReadExt, WriteExt}; +use crate::bundle::file::UserFile; +use crate::murmur::{IdString32, IdString64}; +use crate::{BundleFile, BundleFileType, BundleFileVariant}; + +bitflags! { + #[derive(Clone, Copy, Debug)] + struct TextureFlags: u32 { + const STREAMABLE = 0b0000_0001; + const UNKNOWN = 1 << 1; + const SRGB = 1 << 8; + } +} + +#[derive(Clone, Debug)] +struct TextureHeader { + flags: TextureFlags, + n_streamable_mipmaps: u32, + width: u32, + height: u32, +} + +impl TextureHeader { + #[tracing::instrument(skip(r))] + fn from_binary(mut r: impl ReadExt) -> Result { + let flags = r.read_u32().and_then(|bits| { + TextureFlags::from_bits(bits) + .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) + })?; + let n_streamable_mipmaps = r.read_u32()?; + let width = r.read_u32()?; + let height = r.read_u32()?; + + // Don't quite know yet what this is, only that it is related to mipmaps. + // The reference to "streamable mipmaps" comes from VT2, so far. + // As such, it might be related to the stream file, but since all texture files have it, + // The engine calculates some offset and then moves 68 bytes at that offset to the beginning. + // Hence the split between `68` and `60` in the length. + r.seek(SeekFrom::Current(68 + 60))?; + + Ok(Self { + flags, + n_streamable_mipmaps, + width, + height, + }) + } + + #[tracing::instrument(skip(w))] + fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + eyre::ensure!( + self.flags.is_empty() && self.n_streamable_mipmaps == 0, + "Only textures are supported where `flags == 0` and `n_streamable_mipmaps == 0`." + ); + + w.write_u32(self.flags.bits())?; + w.write_u32(self.n_streamable_mipmaps)?; + w.write_u32(self.width)?; + w.write_u32(self.height)?; + + // See `from_binary` about this unknown section. + let buf = [0; 148]; + w.write_all(&buf)?; + + Ok(()) + } +} + +#[derive(Clone, Debug)] +struct Texture { + header: TextureHeader, + data: Vec, + stream: Option>, + category: IdString32, +} + +impl Texture { + #[tracing::instrument(skip(r, stream_r))] + fn from_binary(mut r: impl Read + Seek, mut stream_r: Option) -> Result { + // Looking at the executable in IDA, there is one other valid value: `2`. + // If this ever comes up in the game data, I'll have to reverse engineer the + // (de)compression algorithm through IDA. + let compression_type = r.read_u32()?; + eyre::ensure!( + compression_type == 1, + "Unknown compression type for texture '{}'", + compression_type + ); + + let compressed_size = r.read_u32()? as usize; + let uncompressed_size = r.read_u32()? as usize; + + let out_buf = { + let mut comp_buf = vec![0; compressed_size]; + r.read_exact(&mut comp_buf)?; + + oodle::decompress( + comp_buf, + uncompressed_size, + OodleLZ_FuzzSafe::No, + OodleLZ_CheckCRC::No, + )? + }; + + eyre::ensure!( + out_buf.len() == uncompressed_size, + "Length of decompressed buffer did not match expected value. Expected {}, got {}", + uncompressed_size, + out_buf.len() + ); + + // No idea what this number is supposed to mean. + // Even the game engine just skips this one. + r.skip_u32(0x43)?; + + let header = TextureHeader::from_binary(&mut r)?; + + let meta_size = r.read_u32()?; + + eyre::ensure!( + meta_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. meta_size = {}, stream = {}", + meta_size, + stream_r.is_some() + ); + + let stream = if let Some(stream_r) = stream_r.as_mut() { + // Number of compression chunks in the stream file + let num_chunks = r.read_u32()?; + r.skip_u16(0)?; + + { + let num_chunks_1 = r.read_u16()? as u32; + + eyre::ensure!( + num_chunks == num_chunks_1, + "Chunk numbers don't match. first = {}, second = {}", + num_chunks, + num_chunks_1 + ); + } + + const RAW_SIZE: usize = 0x10000; + let mut stream_raw = Vec::new(); + let mut last = 0; + + for i in 0..num_chunks { + let offset_next = r.read_u32()? as usize; + let size = offset_next - last; + + let span = tracing::info_span!( + "read stream chunk", + num_chunks, + i, + chunk_size = size, + offset = last + ); + let _enter = span.enter(); + + let mut buf = vec![0; size]; + stream_r + .read_exact(&mut buf) + .wrap_err("Failed to read chunk from stream file")?; + + let raw = + oodle::decompress(&buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) + .wrap_err("Failed to decompress stream chunk")?; + + stream_raw.extend_from_slice(&raw); + + last = offset_next; + } + + Some(stream_raw) + } else { + None + }; + + let category = r.read_u32().map(IdString32::from)?; + + Ok(Self { + category, + header, + data: out_buf, + stream, + }) + } + + #[tracing::instrument(skip(w))] + fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + let compression_type = 1; + w.write_u32(compression_type)?; + + let comp_buf = oodle::compress(&self.data).wrap_err("Failed to compress DDS data")?; + + w.write_u32(comp_buf.len() as u32)?; + w.write_u32(self.data.len() as u32)?; + w.write_all(&comp_buf)?; + + // Unknown field, which the engine seems to ignore. + // All game files have the same value here, so we just mirror that. + w.write_u32(0x43)?; + + self.header.to_binary(&mut w)?; + + // More data not fully figured out, yet. + let meta_size = 0; + w.write_u32(meta_size)?; + + w.write_u32(self.category.to_murmur32().into())?; + Ok(()) + } + + #[tracing::instrument] + fn to_user_files(&self, name: String) -> Vec { + let mut files = Vec::with_capacity(2); + + // TODO: Don't clone. + + if let Some(stream) = &self.stream { + let stream_name = PathBuf::from(&name).with_extension("stream"); + files.push(UserFile::with_name( + stream.clone(), + stream_name.display().to_string(), + )); + } + + files.push(UserFile::with_name(self.data.clone(), name)); + files + } +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinition { + common: TextureDefinitionPlatform, + // Stingray supports per-platform sections here, where you can create overrides with the same + // values as in `common`. But since we only support PC, we don't need to implement + // that. +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionPlatform { + input: TextureDefinitionInput, + output: TextureDefinitionOutput, +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionInput { + filename: String, +} + +#[derive(Clone, Debug, Deserialize)] +struct TextureDefinitionOutput { + category: String, +} + +#[tracing::instrument(skip(data), fields(buf_len = data.as_ref().len()))] +pub(crate) async fn decompile_data( + name: String, + data: impl AsRef<[u8]>, + stream_file_name: Option, +) -> Result> { + let mut r = Cursor::new(data.as_ref()); + let mut stream_r = if let Some(file_name) = stream_file_name { + let stream_data = fs::read(&file_name) + .await + .wrap_err_with(|| format!("Failed to read stream file '{}'", file_name.display()))?; + Some(Cursor::new(stream_data)) + } else { + None + }; + + let texture = Texture::from_binary(&mut r, stream_r.as_mut())?; + let files = texture.to_user_files(name); + Ok(files) +} + +#[tracing::instrument(skip(ctx))] +pub(crate) async fn decompile( + ctx: &crate::Context, + name: String, + variant: &BundleFileVariant, +) -> Result> { + if !variant.external() { + tracing::debug!("Decompiling texture from bundle data"); + + let stream_file_name = variant.data_file_name().map(|name| match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(name), + None => PathBuf::from("bundle").join(name), + }); + + return decompile_data(name, variant.data(), stream_file_name).await; + } + + let Some(file_name) = variant.data_file_name() else { + eyre::bail!("Texture file has no data and no data file"); + }; + + tracing::debug!("Decompiling texture from external file '{}'", file_name); + + let path = match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(file_name), + None => PathBuf::from("bundle").join(file_name), + }; + + tracing::trace!(path = %path.display()); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + + decompile_data(name, &data, None).await +} + +#[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] +pub async fn compile( + name: IdString64, + sjson: impl AsRef, + root: impl AsRef + std::fmt::Debug, +) -> Result { + let definitions: TextureDefinition = serde_sjson::from_str(sjson.as_ref()) + .wrap_err("Failed to deserialize SJSON") + .with_section(|| sjson.as_ref().to_string().header("SJSON:"))?; + + let dds = { + let path = root.as_ref().join(definitions.common.input.filename); + fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read DDS file '{}'", path.display()))? + }; + + let (width, height) = { + let mut r = Cursor::new(&dds); + + let magic = r.read_u32()?; + eyre::ensure!( + magic == 0x20534444, + "Invalid magic bytes for DDS. Expected 0x20534444, got {:08x}", + magic + ); + + r.seek(SeekFrom::Current(5))?; + + let width = r.read_u16()? as u32; + let height = r.read_u16()? as u32; + + (width, height) + }; + + let mut w = Cursor::new(Vec::new()); + + let texture = Texture { + header: TextureHeader { + // As long as we can't handle mipmaps, these two need be `0` + flags: TextureFlags::empty(), + n_streamable_mipmaps: 0, + width, + height, + }, + data: dds, + stream: None, + category: IdString32::String(definitions.common.output.category), + }; + texture.to_binary(&mut w)?; + + let mut variant = BundleFileVariant::new(); + variant.set_data(w.into_inner()); + + let mut file = BundleFile::new(name, BundleFileType::Texture); + file.add_variant(variant); + + Ok(file) +} diff --git a/lib/sdk/src/murmur/dictionary.rs b/lib/sdk/src/murmur/dictionary.rs index 267f0a4..c1b5636 100644 --- a/lib/sdk/src/murmur/dictionary.rs +++ b/lib/sdk/src/murmur/dictionary.rs @@ -48,6 +48,7 @@ struct Row { group: HashGroup, } +#[derive(Clone)] pub struct Entry { value: String, long: Murmur64, @@ -73,6 +74,7 @@ impl Entry { } } +#[derive(Clone)] pub struct Dictionary { entries: Vec, } @@ -88,10 +90,12 @@ impl Dictionary { Self { entries: vec![] } } - pub async fn from_csv(&mut self, r: R) -> Result<()> + pub async fn from_csv(r: R) -> Result where R: AsyncRead + std::marker::Unpin + std::marker::Send, { + let mut entries = vec![]; + let r = AsyncDeserializer::from_reader(r); let mut records = r.into_deserialize::(); @@ -112,10 +116,10 @@ impl Dictionary { group: record.group, }; - self.entries.push(entry); + entries.push(entry); } - Ok(()) + Ok(Self { entries }) } pub async fn to_csv(&self, w: W) -> Result<()> @@ -161,7 +165,7 @@ impl Dictionary { self.entries.push(entry); } - pub fn find(&mut self, value: &String, group: HashGroup) -> Option<&Entry> { + pub fn find(&self, value: &String, group: HashGroup) -> Option<&Entry> { self.entries .iter() .find(|e| e.value == *value && e.group == group) From 67f313107e851aa5571885b21049316f4f0fb7f5 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 6 Oct 2023 10:01:18 +0200 Subject: [PATCH 29/35] sdk: Add dictionary group for texture categories --- lib/sdk/src/context.rs | 8 ++++---- lib/sdk/src/filetype/strings.rs | 6 +++--- lib/sdk/src/filetype/texture.rs | 21 +++++++++++++-------- lib/sdk/src/murmur/dictionary.rs | 12 ++++++++++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/sdk/src/context.rs b/lib/sdk/src/context.rs index 8c10b3c..c565429 100644 --- a/lib/sdk/src/context.rs +++ b/lib/sdk/src/context.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::process::Command; use std::sync::Arc; -use crate::murmur::{Dictionary, HashGroup, IdString64, Murmur32, Murmur64}; +use crate::murmur::{Dictionary, HashGroup, IdString32, IdString64, Murmur32, Murmur64}; #[derive(Clone)] pub struct CmdLine { @@ -87,17 +87,17 @@ impl Context { } } - pub fn lookup_hash_short(&self, hash: M, group: HashGroup) -> String + pub fn lookup_hash_short(&self, hash: M, group: HashGroup) -> IdString32 where M: Into, { let hash = hash.into(); if let Some(s) = self.lookup.lookup_short(hash, group) { tracing::debug!(%hash, string = s, "Murmur32 lookup successful"); - s.to_owned() + s.to_string().into() } else { tracing::debug!(%hash, "Murmur32 lookup failed"); - format!("{hash:08X}") + hash.into() } } } diff --git a/lib/sdk/src/filetype/strings.rs b/lib/sdk/src/filetype/strings.rs index 8643266..c7ad6f9 100644 --- a/lib/sdk/src/filetype/strings.rs +++ b/lib/sdk/src/filetype/strings.rs @@ -5,7 +5,7 @@ use color_eyre::{Report, Result}; use crate::binary::sync::ReadExt; use crate::bundle::file::{BundleFileVariant, UserFile}; -use crate::murmur::HashGroup; +use crate::murmur::{HashGroup, IdString32}; #[derive(Copy, Clone, PartialEq, Eq, Hash, serde::Serialize)] #[serde(untagged)] @@ -26,7 +26,7 @@ impl Language { } #[derive(serde::Serialize)] -pub struct Strings(HashMap>); +pub struct Strings(HashMap>); #[inline(always)] fn read_string(r: R) -> Result @@ -46,7 +46,7 @@ where impl Strings { #[tracing::instrument(skip_all, fields(languages = variants.len()))] pub fn from_variants(ctx: &crate::Context, variants: &[BundleFileVariant]) -> Result { - let mut map: HashMap> = HashMap::new(); + let mut map: HashMap> = HashMap::new(); for (i, variant) in variants.iter().enumerate() { let _span = tracing::trace_span!("variant {}", i); diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index a8c85cd..7486fb8 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -11,7 +11,7 @@ use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; -use crate::murmur::{IdString32, IdString64}; +use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; bitflags! { @@ -86,8 +86,12 @@ struct Texture { } impl Texture { - #[tracing::instrument(skip(r, stream_r))] - fn from_binary(mut r: impl Read + Seek, mut stream_r: Option) -> Result { + #[tracing::instrument(skip(ctx, r, stream_r))] + fn from_binary( + ctx: &crate::Context, + mut r: impl Read + Seek, + mut stream_r: Option, + ) -> Result { // Looking at the executable in IDA, there is one other valid value: `2`. // If this ever comes up in the game data, I'll have to reverse engineer the // (de)compression algorithm through IDA. @@ -187,7 +191,7 @@ impl Texture { None }; - let category = r.read_u32().map(IdString32::from)?; + let category = ctx.lookup_hash_short(r.read_u32()?, HashGroup::TextureCategory); Ok(Self { category, @@ -265,8 +269,9 @@ struct TextureDefinitionOutput { category: String, } -#[tracing::instrument(skip(data), fields(buf_len = data.as_ref().len()))] +#[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] pub(crate) async fn decompile_data( + ctx: &crate::Context, name: String, data: impl AsRef<[u8]>, stream_file_name: Option, @@ -281,7 +286,7 @@ pub(crate) async fn decompile_data( None }; - let texture = Texture::from_binary(&mut r, stream_r.as_mut())?; + let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; let files = texture.to_user_files(name); Ok(files) } @@ -300,7 +305,7 @@ pub(crate) async fn decompile( None => PathBuf::from("bundle").join(name), }); - return decompile_data(name, variant.data(), stream_file_name).await; + return decompile_data(ctx, name, variant.data(), stream_file_name).await; } let Some(file_name) = variant.data_file_name() else { @@ -323,7 +328,7 @@ pub(crate) async fn decompile( "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." })?; - decompile_data(name, &data, None).await + decompile_data(ctx, name, &data, None).await } #[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] diff --git a/lib/sdk/src/murmur/dictionary.rs b/lib/sdk/src/murmur/dictionary.rs index c1b5636..55e0966 100644 --- a/lib/sdk/src/murmur/dictionary.rs +++ b/lib/sdk/src/murmur/dictionary.rs @@ -12,12 +12,19 @@ pub enum HashGroup { Filename, Filetype, Strings, + TextureCategory, Other, } impl HashGroup { - pub fn all() -> [Self; 3] { - [Self::Filename, Self::Filetype, Self::Other] + pub fn all() -> [Self; 5] { + [ + Self::Filename, + Self::Filetype, + Self::Strings, + Self::TextureCategory, + Self::Other, + ] } } @@ -27,6 +34,7 @@ impl std::fmt::Display for HashGroup { HashGroup::Filename => write!(f, "filename"), HashGroup::Filetype => write!(f, "filetype"), HashGroup::Strings => write!(f, "strings"), + HashGroup::TextureCategory => write!(f, "texture-category"), HashGroup::Other => write!(f, "other"), } } From 58071958d260a19a54a570d1921126d5acdc678c Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 6 Oct 2023 11:01:15 +0200 Subject: [PATCH 30/35] sdk: Add decompiled SJSON texture file In addition to the actual image file, also write a `.texture` engine file. --- lib/sdk/src/filetype/texture.rs | 83 +++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 30 deletions(-) diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 7486fb8..7912e45 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -6,7 +6,7 @@ use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; @@ -14,6 +14,30 @@ use crate::bundle::file::UserFile; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinition { + common: TextureDefinitionPlatform, + // Stingray supports per-platform sections here, where you can create overrides with the same + // values as in `common`. But since we only support PC, we don't need to implement + // that. +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionPlatform { + input: TextureDefinitionInput, + output: TextureDefinitionOutput, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionInput { + filename: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct TextureDefinitionOutput { + category: String, +} + bitflags! { #[derive(Clone, Copy, Debug)] struct TextureFlags: u32 { @@ -227,8 +251,21 @@ impl Texture { } #[tracing::instrument] - fn to_user_files(&self, name: String) -> Vec { - let mut files = Vec::with_capacity(2); + fn to_sjson(&self, filename: String) -> Result { + let texture = TextureDefinition { + common: TextureDefinitionPlatform { + input: TextureDefinitionInput { filename }, + output: TextureDefinitionOutput { + category: self.category.display().to_string(), + }, + }, + }; + serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") + } + + #[tracing::instrument] + fn to_user_files(&self, name: String) -> Result> { + let mut files = Vec::with_capacity(3); // TODO: Don't clone. @@ -240,35 +277,20 @@ impl Texture { )); } + { + let data = self.to_sjson(name.clone())?.as_bytes().to_vec(); + let name = PathBuf::from(&name) + .with_extension("texture") + .display() + .to_string(); + files.push(UserFile::with_name(data, name)); + } + files.push(UserFile::with_name(self.data.clone(), name)); - files + Ok(files) } } -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinition { - common: TextureDefinitionPlatform, - // Stingray supports per-platform sections here, where you can create overrides with the same - // values as in `common`. But since we only support PC, we don't need to implement - // that. -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionPlatform { - input: TextureDefinitionInput, - output: TextureDefinitionOutput, -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionInput { - filename: String, -} - -#[derive(Clone, Debug, Deserialize)] -struct TextureDefinitionOutput { - category: String, -} - #[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] pub(crate) async fn decompile_data( ctx: &crate::Context, @@ -287,8 +309,9 @@ pub(crate) async fn decompile_data( }; let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; - let files = texture.to_user_files(name); - Ok(files) + texture + .to_user_files(name) + .wrap_err("Failed to build user files") } #[tracing::instrument(skip(ctx))] From 63fb0a1c0871564ed0e08da0e5fe801ab028b54e Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 22 Jul 2024 11:28:36 +0200 Subject: [PATCH 31/35] Reverse DDSImage::load Decompiling the game binary shows a rather elaborate algorithm to load DDS images from binary. Though comparing it to Microsoft's documentation on DDS, most of it seems to be pretty standard handling. However, we don't actually need all of it. The part about calculating pitch and reading blocks only accesses a subset of the `ImageFormat` struct, so we can strip our implementation to just that. --- lib/sdk/src/filetype/texture.rs | 29 ++-- lib/sdk/src/filetype/texture/dds.rs | 203 ++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 12 deletions(-) create mode 100644 lib/sdk/src/filetype/texture/dds.rs diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 7912e45..58c0dbf 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -14,6 +14,8 @@ use crate::bundle::file::UserFile; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; +mod dds; + #[derive(Clone, Debug, Deserialize, Serialize)] struct TextureDefinition { common: TextureDefinitionPlatform, @@ -53,6 +55,7 @@ struct TextureHeader { n_streamable_mipmaps: u32, width: u32, height: u32, + mip_info_size: u32, } impl TextureHeader { @@ -66,18 +69,19 @@ impl TextureHeader { let width = r.read_u32()?; let height = r.read_u32()?; - // Don't quite know yet what this is, only that it is related to mipmaps. - // The reference to "streamable mipmaps" comes from VT2, so far. - // As such, it might be related to the stream file, but since all texture files have it, - // The engine calculates some offset and then moves 68 bytes at that offset to the beginning. - // Hence the split between `68` and `60` in the length. - r.seek(SeekFrom::Current(68 + 60))?; + r.skip_u32(0)?; + + // A section of 15 pairs of two u32 + r.seek(SeekFrom::Current(2 * 4 * 15))?; + + let mip_info_size = r.read_u32()?; Ok(Self { flags, n_streamable_mipmaps, width, height, + mip_info_size, }) } @@ -94,9 +98,12 @@ impl TextureHeader { w.write_u32(self.height)?; // See `from_binary` about this unknown section. - let buf = [0; 148]; + let buf = [0; (2 * 4 * 15) + 4]; w.write_all(&buf)?; + // TODO: For now we write `0` here, until the mipmap section is figured out + w.write_u32(0)?; + Ok(()) } } @@ -154,12 +161,10 @@ impl Texture { let header = TextureHeader::from_binary(&mut r)?; - let meta_size = r.read_u32()?; - eyre::ensure!( - meta_size == 0 || stream_r.is_some(), - "Compression chunks and stream file don't match up. meta_size = {}, stream = {}", - meta_size, + header.mip_info_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. mip_info_size = {}, has_stream = {}", + header.mip_info_size, stream_r.is_some() ); diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs new file mode 100644 index 0000000..135d24c --- /dev/null +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -0,0 +1,203 @@ +use bitflags::bitflags; +use color_eyre::eyre; +use color_eyre::Result; + +bitflags! { + #[derive(Clone, Copy, Debug)] + pub struct DDSD: u32 { + /// Required + const CAPS = 0x1; + /// Required + const HEIGHT = 0x2; + /// Required + const WIDTH = 0x4; + /// Pitch for an uncompressed texture + const PITCH = 0x8; + /// Required + const PIXELFORMAT = 0x1000; + /// Required in a mipmapped texture + const MIPMAPCOUNT = 0x20000; + /// Pitch for a compressed texture + const LINEARSIZE = 0x80000; + /// Required in a depth texture + const DEPTH = 0x800000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DDSCAPS: u32 { + const COMPLEX = 0x8; + const MIPMAP = 0x400000; + const TEXTURE = 0x1000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DDSCAPS2: u32 { + const CUBEMAP = 0x200; + const CUBEMAP_POSITIVEX = 0x400; + const CUBEMAP_NEGATIVEX = 0x800; + const CUBEMAP_POSITIVEY = 0x1000; + const CUBEMAP_NEGATIVEY = 0x2000; + const CUBEMAP_POSITIVEZ = 0x4000; + const CUBEMAP_NEGATIVEZ = 0x8000; + const VOLUME = 0x200000; + + const CUBEMAP_ALLFACES = Self::CUBEMAP_POSITIVEX.bits() + | Self::CUBEMAP_NEGATIVEX.bits() + | Self::CUBEMAP_POSITIVEY.bits() + | Self::CUBEMAP_NEGATIVEY.bits() + | Self::CUBEMAP_POSITIVEZ.bits() + | Self::CUBEMAP_NEGATIVEZ.bits(); + } + + #[derive(Clone, Copy, Debug)] + pub struct DDPF: u32 { + const ALPHAPIXELS = 0x1; + const ALPHA = 0x2; + const FOURCC = 0x4; + const RGB = 0x40; + const YUV = 0x200; + const LUMINANCE = 0x20000; + } + + #[derive(Clone, Copy, Debug)] + pub struct DdsResourceMiscFlags: u32 { + const TEXTURECUBE = 0x4; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum D3D10ResourceDimension { + Unknown = 0, + Buffer = 1, + Texture1D = 2, + Texture2D = 3, + Texture3D = 4, +} + +pub struct Dx10Header { + /// Resource data formats, including fully-typed and typeless formats. + /// See https://learn.microsoft.com/en-us/windows/win32/api/dxgiformat/ne-dxgiformat-dxgi_format + dxgi_format: u32, + resource_dimension: D3D10ResourceDimension, + misc_flag: DdsResourceMiscFlags, + array_size: u32, + misc_flags2: u32, +} + +pub struct DDSPixelFormat { + /// Structure size. Must be `32`. + size: u32, + flags: DDPF, + four_cc: u32, + rgb_bit_count: u32, + r_bit_mask: u32, + g_bit_mask: u32, + b_bit_mask: u32, + a_bit_mask: u32, +} + +pub struct DDSHeader { + /// Size of this structure. Must be `124`. + size: u32, + /// Flags to indicate which members contain valid data. + flags: DDSD, + height: u32, + width: u32, + pitch_or_linear_size: u32, + depth: u32, + mipmap_count: u32, + reserved_1: [u8; 11], + pixel_format: DDSPixelFormat, + caps: DDSCAPS, + caps_2: DDSCAPS2, + reserved_2: [u8; 3], +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImageType { + Image2D = 0, + Image3D = 1, + ImageCube = 2, + Unknown = 3, + Image2dArray = 4, + ImagecubeArray = 5, +} + +/// A stripped version of `ImageType` that only contains just the data needed +/// to read a DDS image stream. +pub struct StrippedImageFormat { + pub image_type: ImageType, + pub width: u32, + pub height: u32, + pub layers: u32, + pub mip_levels: u32, +} + +// This is a stripped down version of the logic that the engine implements to fill +// `stingray::ImageFormat`. With the `type` field we need to distinguish between `IMAGE3D` +// and everything else, and we need the various dimensions filled to calculate the chunks. +pub fn stripped_format_from_header( + dds_header: &DDSHeader, + dx10_header: &Dx10Header, +) -> Result { + let mut image_format = StrippedImageFormat { + image_type: ImageType::Unknown, + width: dds_header.width, + height: dds_header.height, + layers: 0, + mip_levels: 0, + }; + + if dds_header.mipmap_count > 0 { + image_format.mip_levels = dds_header.mipmap_count; + } else { + image_format.mip_levels = 1; + } + + // INFO: These next two sections are conditional in the engine code, + // based on a lot of stuff in "fourcc" and other fields. But it might + // actually be fine to just do it like this, as this seems universal + // to DDS. + // Will have to check how it plays out with actual assets. + + if dds_header.caps_2.contains(DDSCAPS2::CUBEMAP) { + image_format.image_type = ImageType::ImageCube; + image_format.layers = 6; + } else if dds_header.caps_2.contains(DDSCAPS2::VOLUME) { + image_format.image_type = ImageType::Image3D; + image_format.layers = dds_header.depth; + } else { + image_format.image_type = ImageType::Image2D; + image_format.layers = 1; + } + + if dx10_header.resource_dimension == D3D10ResourceDimension::Texture2D { + if dx10_header.misc_flag == DdsResourceMiscFlags::TextureCube { + image_format.image_type = ImageType::ImageCube; + if dx10_header.array_size > 1 { + image_format.layers = dx10_header.array_size; + } else { + image_format.layers = 6; + } + } else { + image_format.image_type = ImageType::Image2D; + image_format.layers = dx10_header.array_size; + } + } else if dx10_header.resource_dimension == D3D10ResourceDimension::Texture3D { + image_format.image_type = ImageType::Image3D; + image_format.layers = dds_header.depth; + } + + if dx10_header.array_size > 1 { + match image_format.image_type { + ImageType::Image2D => image_format.image_type = ImageType::Image2dArray, + ImageType::ImageCube => image_format.image_type = ImageType::ImagecubeArray, + ImageType::Image3D => { + eyre::bail!("3D-Arrays are not a supported image format") + } + _ => {} + } + } + + Ok(image_format) +} From 9f849ab3ecbbb0808b8c5f44e6d42375a78eee43 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 24 Jul 2024 14:15:51 +0200 Subject: [PATCH 32/35] sdk: Implement decompiling streamed mipmaps For now, we only extract the largest mipmap. --- Cargo.lock | 36 +++ Cargo.toml | 3 + lib/sdk/Cargo.toml | 3 + lib/sdk/src/filetype/texture.rs | 319 +++++++++++++++++------ lib/sdk/src/filetype/texture/dds.rs | 384 +++++++++++++++++++++++++--- lib/sdk/src/lib.rs | 1 + 6 files changed, 639 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65ac0dd..68fb0e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2500,6 +2500,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3346,11 +3357,14 @@ dependencies = [ "glob", "luajit2-sys", "nanorand", + "num-derive", + "num-traits", "oodle", "path-slash", "pin-project-lite", "serde", "serde_sjson", + "strum", "tokio", "tokio-stream", "tracing", @@ -3612,6 +3626,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.100", +] + [[package]] name = "subtle" version = "2.6.1" diff --git a/Cargo.toml b/Cargo.toml index 4b083a9..c6243c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,8 @@ luajit2-sys = { path = "lib/luajit2-sys" } minijinja = { version = "2.0.1", default-features = false, features = ["serde"] } nanorand = "0.7.0" nexusmods = { path = "lib/nexusmods" } +num-derive = "0.4.2" +num-traits = "0.2.19" notify = "8.0.0" oodle = { path = "lib/oodle" } open = "5.0.1" @@ -49,6 +51,7 @@ serde = { version = "1.0.152", features = ["derive", "rc"] } serde_sjson = "1.2.1" steamlocate = "2.0.0-beta.2" strip-ansi-escapes = "0.2.0" +strum = { version = "0.26.3", features = ["derive", "strum_macros"] } time = { version = "0.3.20", features = ["serde", "serde-well-known", "local-offset", "formatting", "macros"] } tokio = { version = "1.23.0", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] } tokio-stream = { version = "0.1.12", features = ["fs", "io-util"] } diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 4667a1c..0cd0e4e 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -15,11 +15,14 @@ futures-util = { workspace = true } glob = { workspace = true } luajit2-sys = { workspace = true } nanorand = { workspace = true } +num-derive = { workspace = true } +num-traits = { workspace = true } oodle = { workspace = true } path-slash = { workspace = true } pin-project-lite = { workspace = true } serde = { workspace = true } serde_sjson = { workspace = true } +strum = { workspace = true } tokio = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 58c0dbf..1565b8c 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -1,16 +1,18 @@ -use std::io::{Cursor, Read, Seek, SeekFrom}; +use std::io::{Cursor, Read, Seek, SeekFrom, Write as _}; use std::path::{Path, PathBuf}; use bitflags::bitflags; use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; +use num_traits::ToPrimitive as _; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; +use crate::filetype::texture::dds::{DXGIFormat, ImageType}; use crate::murmur::{HashGroup, IdString32, IdString64}; use crate::{BundleFile, BundleFileType, BundleFileVariant}; @@ -49,13 +51,20 @@ bitflags! { } } +#[derive(Copy, Clone, Debug, Default)] +struct TextureHeaderMipInfo { + offset: usize, + size: usize, +} + #[derive(Clone, Debug)] struct TextureHeader { flags: TextureFlags, - n_streamable_mipmaps: u32, - width: u32, - height: u32, - mip_info_size: u32, + n_streamable_mipmaps: usize, + width: usize, + height: usize, + mip_infos: [TextureHeaderMipInfo; 16], + meta_size: usize, } impl TextureHeader { @@ -65,23 +74,26 @@ impl TextureHeader { TextureFlags::from_bits(bits) .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) })?; - let n_streamable_mipmaps = r.read_u32()?; - let width = r.read_u32()?; - let height = r.read_u32()?; + let n_streamable_mipmaps = r.read_u32()? as usize; + let width = r.read_u32()? as usize; + let height = r.read_u32()? as usize; - r.skip_u32(0)?; + let mut mip_infos = [TextureHeaderMipInfo::default(); 16]; - // A section of 15 pairs of two u32 - r.seek(SeekFrom::Current(2 * 4 * 15))?; + for info in mip_infos.iter_mut() { + info.offset = r.read_u32()? as usize; + info.size = r.read_u32()? as usize; + } - let mip_info_size = r.read_u32()?; + let meta_size = r.read_u32()? as usize; Ok(Self { flags, n_streamable_mipmaps, width, height, - mip_info_size, + mip_infos, + meta_size, }) } @@ -93,15 +105,16 @@ impl TextureHeader { ); w.write_u32(self.flags.bits())?; - w.write_u32(self.n_streamable_mipmaps)?; - w.write_u32(self.width)?; - w.write_u32(self.height)?; + w.write_u32(self.n_streamable_mipmaps as u32)?; + w.write_u32(self.width as u32)?; + w.write_u32(self.height as u32)?; - // See `from_binary` about this unknown section. - let buf = [0; (2 * 4 * 15) + 4]; - w.write_all(&buf)?; + for info in self.mip_infos { + w.write_u32(info.offset as u32)?; + w.write_u32(info.size as u32)?; + } - // TODO: For now we write `0` here, until the mipmap section is figured out + // TODO: For now we write `0` here, until the meta section is figured out w.write_u32(0)?; Ok(()) @@ -117,6 +130,90 @@ struct Texture { } impl Texture { + #[tracing::instrument(skip(data, chunks))] + fn decompress_stream_data(mut data: impl Read, chunks: impl AsRef<[usize]>) -> Result> { + const RAW_SIZE: usize = 0x10000; + + let chunks = chunks.as_ref(); + + let max_size = chunks.iter().max().copied().unwrap_or(RAW_SIZE); + let mut read_buf = vec![0; max_size]; + + let mut stream_raw = Vec::with_capacity(chunks.iter().sum()); + let mut last = 0; + + for offset_next in chunks { + let size = offset_next - last; + + let span = tracing::info_span!( + "stream chunk", + num_chunks = chunks.len(), + chunk_size_comp = size, + offset = last + ); + let _enter = span.enter(); + + let buf = &mut read_buf[0..size]; + data.read_exact(buf) + .wrap_err("Failed to read chunk from stream file")?; + + let raw = oodle::decompress(buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) + .wrap_err("Failed to decompress stream chunk")?; + eyre::ensure!( + raw.len() == RAW_SIZE, + "Invalid chunk length after decompression" + ); + + stream_raw.extend_from_slice(&raw); + + last = *offset_next; + } + Ok(stream_raw) + } + + #[tracing::instrument(skip(data), fields(data_len = data.as_ref().len()))] + fn reorder_stream_mipmap( + data: impl AsRef<[u8]>, + bits_per_block: usize, + bytes_per_block: usize, + block_size: usize, + pitch: usize, + ) -> Result> { + const CHUNK_SIZE: usize = 0x10000; + let data = data.as_ref(); + + let mut out = Vec::with_capacity(data.len()); + let mut window = vec![0u8; pitch * 64]; + + let row_size = bits_per_block * block_size; + tracing::Span::current().record("row_size", row_size); + + eyre::ensure!( + data.len() % CHUNK_SIZE == 0, + "Stream data does not divide evenly into chunks" + ); + + for (i, chunk) in data.chunks_exact(CHUNK_SIZE).enumerate() { + let chunk_x = (i % bytes_per_block) * row_size; + + let span = tracing::trace_span!("chunk", i, chunk_x = chunk_x); + let _guard = span.enter(); + + if i > 0 && i % bytes_per_block == 0 { + out.extend_from_slice(&window); + } + + for (j, row) in chunk.chunks_exact(row_size).enumerate() { + let start = chunk_x + j * pitch; + let end = start + row_size; + tracing::trace!("{i}/{j} at {}:{}", start, end); + window[start..end].copy_from_slice(row); + } + } + + Ok(out) + } + #[tracing::instrument(skip(ctx, r, stream_r))] fn from_binary( ctx: &crate::Context, @@ -162,19 +259,19 @@ impl Texture { let header = TextureHeader::from_binary(&mut r)?; eyre::ensure!( - header.mip_info_size == 0 || stream_r.is_some(), - "Compression chunks and stream file don't match up. mip_info_size = {}, has_stream = {}", - header.mip_info_size, + header.meta_size == 0 || stream_r.is_some(), + "Compression chunks and stream file don't match up. meta_size = {}, has_stream = {}", + header.meta_size, stream_r.is_some() ); let stream = if let Some(stream_r) = stream_r.as_mut() { // Number of compression chunks in the stream file - let num_chunks = r.read_u32()?; + let num_chunks = r.read_u32()? as usize; r.skip_u16(0)?; { - let num_chunks_1 = r.read_u16()? as u32; + let num_chunks_1 = r.read_u16()? as usize; eyre::ensure!( num_chunks == num_chunks_1, @@ -184,37 +281,15 @@ impl Texture { ); } - const RAW_SIZE: usize = 0x10000; - let mut stream_raw = Vec::new(); - let mut last = 0; + let mut chunks = Vec::with_capacity(num_chunks); - for i in 0..num_chunks { - let offset_next = r.read_u32()? as usize; - let size = offset_next - last; - - let span = tracing::info_span!( - "read stream chunk", - num_chunks, - i, - chunk_size = size, - offset = last - ); - let _enter = span.enter(); - - let mut buf = vec![0; size]; - stream_r - .read_exact(&mut buf) - .wrap_err("Failed to read chunk from stream file")?; - - let raw = - oodle::decompress(&buf, RAW_SIZE, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No) - .wrap_err("Failed to decompress stream chunk")?; - - stream_raw.extend_from_slice(&raw); - - last = offset_next; + for _ in 0..num_chunks { + chunks.push(r.read_u32()? as usize); } + let stream_raw = Self::decompress_stream_data(stream_r, chunks) + .wrap_err("Failed to decompress stream data")?; + Some(stream_raw) } else { None @@ -247,10 +322,6 @@ impl Texture { self.header.to_binary(&mut w)?; - // More data not fully figured out, yet. - let meta_size = 0; - w.write_u32(meta_size)?; - w.write_u32(self.category.to_murmur32().into())?; Ok(()) } @@ -268,19 +339,9 @@ impl Texture { serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") } - #[tracing::instrument] + #[tracing::instrument(skip(self))] fn to_user_files(&self, name: String) -> Result> { - let mut files = Vec::with_capacity(3); - - // TODO: Don't clone. - - if let Some(stream) = &self.stream { - let stream_name = PathBuf::from(&name).with_extension("stream"); - files.push(UserFile::with_name( - stream.clone(), - stream_name.display().to_string(), - )); - } + let mut files = Vec::with_capacity(2); { let data = self.to_sjson(name.clone())?.as_bytes().to_vec(); @@ -291,7 +352,119 @@ impl Texture { files.push(UserFile::with_name(data, name)); } - files.push(UserFile::with_name(self.data.clone(), name)); + // For debugging purposes, also extract the raw files + if cfg!(debug_assertions) { + if let Some(stream) = &self.stream { + let stream_name = PathBuf::from(&name).with_extension("stream"); + files.push(UserFile::with_name( + stream.clone(), + stream_name.display().to_string(), + )); + } + + let name = PathBuf::from(&name) + .with_extension("raw.dds") + .display() + .to_string(); + files.push(UserFile::with_name(self.data.clone(), name)); + } + + { + let mut data = Cursor::new(&self.data); + let mut dds_header = + dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; + + eyre::ensure!( + dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) + && dds_header.pixel_format.four_cc == dds::FOURCC_DX10, + "Only DX10 textures are currently supported." + ); + + let dx10_header = + dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; + + match dx10_header.dxgi_format { + DXGIFormat::BC1_UNORM + | DXGIFormat::BC3_UNORM + | DXGIFormat::BC4_UNORM + | DXGIFormat::BC5_UNORM + | DXGIFormat::BC6H_UF16 + | DXGIFormat::BC7_UNORM => {} + _ => { + eyre::bail!( + "Unsupported DXGI format: {} (0x{:0X})", + dx10_header.dxgi_format, + dx10_header.dxgi_format.to_u32().unwrap_or_default() + ); + } + } + + let stingray_image_format = + dds::stripped_format_from_header(&dds_header, &dx10_header)?; + eyre::ensure!( + stingray_image_format.image_type == ImageType::Image2D, + "Unsupported image type: {}", + stingray_image_format.image_type, + ); + + let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; + let bits_per_block: usize = match block_size { + 8 => 128, + 16 => 64, + block_size => eyre::bail!("Unsupported block size {}", block_size), + }; + + let pitch = self.header.width / 4 * block_size; + let bytes_per_block = self.header.width / bits_per_block / 4; + + tracing::debug!( + "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", + block_size, + pitch, + bits_per_block, + bytes_per_block + ); + + let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); + + // Currently, we only extract the largest mipmap, + // so we need to set the dimensions accordingly, and remove the + // flag. + dds_header.width = self.header.width; + dds_header.height = self.header.height; + dds_header.mipmap_count = 0; + dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; + + dds_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DDS header")?; + + dx10_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DX10 header")?; + + if let Some(stream) = &self.stream { + let data = Self::reorder_stream_mipmap( + stream, + bits_per_block, + bytes_per_block, + block_size, + pitch, + ) + .wrap_err("Failed to reorder stream chunks")?; + + out_data + .write_all(&data) + .wrap_err("Failed to write streamed mipmap data")?; + } else { + out_data + .write_all(data.split().1) + .wrap_err("Failed to write texture data")?; + }; + + files.push(UserFile::with_name(out_data.into_inner(), name)); + } + Ok(files) } } @@ -388,8 +561,8 @@ pub async fn compile( r.seek(SeekFrom::Current(5))?; - let width = r.read_u16()? as u32; - let height = r.read_u16()? as u32; + let width = r.read_u32()? as usize; + let height = r.read_u32()? as usize; (width, height) }; @@ -403,6 +576,8 @@ pub async fn compile( n_streamable_mipmaps: 0, width, height, + mip_infos: [TextureHeaderMipInfo::default(); 16], + meta_size: 0, }, data: dds, stream: None, diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs index 135d24c..5ef1b90 100644 --- a/lib/sdk/src/filetype/texture/dds.rs +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -1,6 +1,16 @@ +use std::io::SeekFrom; + use bitflags::bitflags; -use color_eyre::eyre; +use color_eyre::eyre::Context as _; +use color_eyre::eyre::{self, OptionExt as _}; use color_eyre::Result; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::{FromPrimitive as _, ToPrimitive as _}; + +use crate::binary::sync::{ReadExt, WriteExt}; + +const MAGIC_DDS: u32 = 0x20534444; +pub const FOURCC_DX10: u32 = 0x30315844; bitflags! { #[derive(Clone, Copy, Debug)] @@ -65,7 +75,27 @@ bitflags! { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +fn flags_from_bits(bits: T::Bits) -> T +where + ::Bits: std::fmt::Binary, +{ + if let Some(flags) = T::from_bits(bits) { + flags + } else { + let unknown = bits & !T::all().bits(); + + tracing::warn!( + "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", + std::any::type_name::(), + T::all().bits(), + unknown + ); + + T::from_bits_truncate(bits) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, FromPrimitive, ToPrimitive)] pub enum D3D10ResourceDimension { Unknown = 0, Buffer = 1, @@ -74,46 +104,325 @@ pub enum D3D10ResourceDimension { Texture3D = 4, } +#[allow(clippy::upper_case_acronyms)] +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, strum::Display, FromPrimitive, ToPrimitive)] +pub enum DXGIFormat { + UNKNOWN = 0, + R32G32B32A32_TYPELESS = 1, + R32G32B32A32_FLOAT = 2, + R32G32B32A32_UINT = 3, + R32G32B32A32_SINT = 4, + R32G32B32_TYPELESS = 5, + R32G32B32_FLOAT = 6, + R32G32B32_UINT = 7, + R32G32B32_SINT = 8, + R16G16B16A16_TYPELESS = 9, + R16G16B16A16_FLOAT = 10, + R16G16B16A16_UNORM = 11, + R16G16B16A16_UINT = 12, + R16G16B16A16_SNORM = 13, + R16G16B16A16_SINT = 14, + R32G32_TYPELESS = 15, + R32G32_FLOAT = 16, + R32G32_UINT = 17, + R32G32_SINT = 18, + R32G8X24_TYPELESS = 19, + D32_FLOAT_S8X24_UINT = 20, + R32_FLOAT_X8X24_TYPELESS = 21, + X32_TYPELESS_G8X24_UINT = 22, + R10G10B10A2_TYPELESS = 23, + R10G10B10A2_UNORM = 24, + R10G10B10A2_UINT = 25, + R11G11B10_FLOAT = 26, + R8G8B8A8_TYPELESS = 27, + R8G8B8A8_UNORM = 28, + R8G8B8A8_UNORM_SRGB = 29, + R8G8B8A8_UINT = 30, + R8G8B8A8_SNORM = 31, + R8G8B8A8_SINT = 32, + R16G16_TYPELESS = 33, + R16G16_FLOAT = 34, + R16G16_UNORM = 35, + R16G16_UINT = 36, + R16G16_SNORM = 37, + R16G16_SINT = 38, + R32_TYPELESS = 39, + D32_FLOAT = 40, + R32_FLOAT = 41, + R32_UINT = 42, + R32_SINT = 43, + R24G8_TYPELESS = 44, + D24_UNORM_S8_UINT = 45, + R24_UNORM_X8_TYPELESS = 46, + X24_TYPELESS_G8_UINT = 47, + R8G8_TYPELESS = 48, + R8G8_UNORM = 49, + R8G8_UINT = 50, + R8G8_SNORM = 51, + R8G8_SINT = 52, + R16_TYPELESS = 53, + R16_FLOAT = 54, + D16_UNORM = 55, + R16_UNORM = 56, + R16_UINT = 57, + R16_SNORM = 58, + R16_SINT = 59, + R8_TYPELESS = 60, + R8_UNORM = 61, + R8_UINT = 62, + R8_SNORM = 63, + R8_SINT = 64, + A8_UNORM = 65, + R1_UNORM = 66, + R9G9B9E5_SHAREDEXP = 67, + R8G8_B8G8_UNORM = 68, + G8R8_G8B8_UNORM = 69, + BC1_TYPELESS = 70, + BC1_UNORM = 71, + BC1_UNORM_SRGB = 72, + BC2_TYPELESS = 73, + BC2_UNORM = 74, + BC2_UNORM_SRGB = 75, + BC3_TYPELESS = 76, + BC3_UNORM = 77, + BC3_UNORM_SRGB = 78, + BC4_TYPELESS = 79, + BC4_UNORM = 80, + BC4_SNORM = 81, + BC5_TYPELESS = 82, + BC5_UNORM = 83, + BC5_SNORM = 84, + B5G6R5_UNORM = 85, + B5G5R5A1_UNORM = 86, + B8G8R8A8_UNORM = 87, + B8G8R8X8_UNORM = 88, + R10G10B10_XR_BIAS_A2_UNORM = 89, + B8G8R8A8_TYPELESS = 90, + B8G8R8A8_UNORM_SRGB = 91, + B8G8R8X8_TYPELESS = 92, + B8G8R8X8_UNORM_SRGB = 93, + BC6H_TYPELESS = 94, + BC6H_UF16 = 95, + BC6H_SF16 = 96, + BC7_TYPELESS = 97, + BC7_UNORM = 98, + BC7_UNORM_SRGB = 99, + AYUV = 100, + Y410 = 101, + Y416 = 102, + NV12 = 103, + P010 = 104, + P016 = 105, + OPAQUE = 106, + YUY2 = 107, + Y210 = 108, + Y216 = 109, + NV11 = 110, + AI44 = 111, + IA44 = 112, + P8 = 113, + A8P8 = 114, + B4G4R4A4_UNORM = 115, + P208 = 130, + V208 = 131, + V408 = 132, + SAMPLER_FEEDBACK_MIN_MIP_OPAQUE, + SAMPLER_FEEDBACK_MIP_REGION_USED_OPAQUE, +} + +#[derive(Clone, Copy, Debug)] pub struct Dx10Header { /// Resource data formats, including fully-typed and typeless formats. /// See https://learn.microsoft.com/en-us/windows/win32/api/dxgiformat/ne-dxgiformat-dxgi_format - dxgi_format: u32, - resource_dimension: D3D10ResourceDimension, - misc_flag: DdsResourceMiscFlags, - array_size: u32, - misc_flags2: u32, + pub dxgi_format: DXGIFormat, + pub resource_dimension: D3D10ResourceDimension, + pub misc_flag: DdsResourceMiscFlags, + pub array_size: usize, + pub misc_flags2: u32, } +impl Dx10Header { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + let dxgi_format = r + .read_u32() + .map(|val| DXGIFormat::from_u32(val).unwrap_or(DXGIFormat::UNKNOWN))?; + let resource_dimension = r.read_u32().map(|val| { + D3D10ResourceDimension::from_u32(val).unwrap_or(D3D10ResourceDimension::Unknown) + })?; + let misc_flag = r.read_u32().map(flags_from_bits)?; + let array_size = r.read_u32()? as usize; + let misc_flags2 = r.read_u32()?; + + Ok(Self { + dxgi_format, + resource_dimension, + misc_flag, + array_size, + misc_flags2, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + w.write_u32( + self.dxgi_format + .to_u32() + .ok_or_eyre("DXGIFormat should fit in a u32")?, + )?; + w.write_u32( + self.resource_dimension + .to_u32() + .ok_or_eyre("DXGIFormat should fit in a u32")?, + )?; + w.write_u32(self.misc_flag.bits())?; + w.write_u32(self.array_size as u32)?; + w.write_u32(self.misc_flags2)?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] pub struct DDSPixelFormat { - /// Structure size. Must be `32`. - size: u32, - flags: DDPF, - four_cc: u32, - rgb_bit_count: u32, - r_bit_mask: u32, - g_bit_mask: u32, - b_bit_mask: u32, - a_bit_mask: u32, + pub flags: DDPF, + pub four_cc: u32, + pub rgb_bit_count: u32, + pub r_bit_mask: u32, + pub g_bit_mask: u32, + pub b_bit_mask: u32, + pub a_bit_mask: u32, } +impl DDSPixelFormat { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + let size = r.read_u32()? as usize; + eyre::ensure!( + size == 32, + "Invalid structure size. Got 0X{:0X}, expected 0x20", + size + ); + + let flags = r.read_u32().map(flags_from_bits)?; + let four_cc = r.read_u32()?; + let rgb_bit_count = r.read_u32()?; + let r_bit_mask = r.read_u32()?; + let g_bit_mask = r.read_u32()?; + let b_bit_mask = r.read_u32()?; + let a_bit_mask = r.read_u32()?; + + Ok(Self { + flags, + four_cc, + rgb_bit_count, + r_bit_mask, + g_bit_mask, + b_bit_mask, + a_bit_mask, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + // Structure size + w.write_u32(32)?; + + w.write_u32(self.flags.bits())?; + w.write_u32(self.four_cc)?; + w.write_u32(self.rgb_bit_count)?; + w.write_u32(self.r_bit_mask)?; + w.write_u32(self.g_bit_mask)?; + w.write_u32(self.b_bit_mask)?; + w.write_u32(self.a_bit_mask)?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug)] pub struct DDSHeader { - /// Size of this structure. Must be `124`. - size: u32, /// Flags to indicate which members contain valid data. - flags: DDSD, - height: u32, - width: u32, - pitch_or_linear_size: u32, - depth: u32, - mipmap_count: u32, - reserved_1: [u8; 11], - pixel_format: DDSPixelFormat, - caps: DDSCAPS, - caps_2: DDSCAPS2, - reserved_2: [u8; 3], + pub flags: DDSD, + pub height: usize, + pub width: usize, + pub pitch_or_linear_size: usize, + pub depth: usize, + pub mipmap_count: usize, + pub pixel_format: DDSPixelFormat, + pub caps: DDSCAPS, + pub caps_2: DDSCAPS2, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +impl DDSHeader { + #[tracing::instrument(skip(r))] + pub fn from_binary(mut r: impl ReadExt) -> Result { + r.skip_u32(MAGIC_DDS).wrap_err("Invalid magic bytes")?; + + let size = r.read_u32()?; + eyre::ensure!( + size == 124, + "Invalid structure size. Got 0x{:0X}, expected 0x7C", + size + ); + + let flags = r.read_u32().map(flags_from_bits)?; + let height = r.read_u32()? as usize; + let width = r.read_u32()? as usize; + let pitch_or_linear_size = r.read_u32()? as usize; + let depth = r.read_u32()? as usize; + let mipmap_count = r.read_u32()? as usize; + + // Skip reserved bytes + r.seek(SeekFrom::Current(11 * 4))?; + + let pixel_format = DDSPixelFormat::from_binary(&mut r)?; + let caps = r.read_u32().map(flags_from_bits)?; + let caps_2 = r.read_u32().map(flags_from_bits)?; + + // Skip unused and reserved bytes + r.seek(SeekFrom::Current(3 * 4))?; + + Ok(Self { + flags, + height, + width, + pitch_or_linear_size, + depth, + mipmap_count, + pixel_format, + caps, + caps_2, + }) + } + + #[tracing::instrument(skip(w))] + pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { + w.write_u32(MAGIC_DDS)?; + + // Structure size in bytes + w.write_u32(124)?; + w.write_u32(self.flags.bits())?; + w.write_u32(self.height as u32)?; + w.write_u32(self.width as u32)?; + w.write_u32(self.pitch_or_linear_size as u32)?; + w.write_u32(self.depth as u32)?; + w.write_u32(self.mipmap_count as u32)?; + + w.write_all(&[0u8; 11 * 4])?; + + self.pixel_format.to_binary(&mut w)?; + w.write_u32(self.caps.bits())?; + w.write_u32(self.caps_2.bits())?; + + w.write_all(&[0u8; 3 * 4])?; + + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] pub enum ImageType { Image2D = 0, Image3D = 1, @@ -125,12 +434,14 @@ pub enum ImageType { /// A stripped version of `ImageType` that only contains just the data needed /// to read a DDS image stream. +#[allow(dead_code)] +#[derive(Clone, Copy, Debug)] pub struct StrippedImageFormat { pub image_type: ImageType, - pub width: u32, - pub height: u32, - pub layers: u32, - pub mip_levels: u32, + pub width: usize, + pub height: usize, + pub layers: usize, + pub mip_levels: usize, } // This is a stripped down version of the logic that the engine implements to fill @@ -172,7 +483,10 @@ pub fn stripped_format_from_header( } if dx10_header.resource_dimension == D3D10ResourceDimension::Texture2D { - if dx10_header.misc_flag == DdsResourceMiscFlags::TextureCube { + if dx10_header + .misc_flag + .contains(DdsResourceMiscFlags::TEXTURECUBE) + { image_format.image_type = ImageType::ImageCube; if dx10_header.array_size > 1 { image_format.layers = dx10_header.array_size; diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 9b1806b..41310fc 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(cursor_split)] #![feature(test)] mod binary; From 94af8862e80c8f7077896bc8183f25a9bb645cae Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 10:43:42 +0200 Subject: [PATCH 33/35] Implement more texture formats The second compression method found in the game's code seems to be Zlib, but it doesn't seem to be used in the game files. What does get used is a compression type of `0`, which appears to be uncompressed data. For DDS formats, all the ones that are currently used by in the game files can be emitted as is. Though for some of them, other tools might not be able to display them. --- Cargo.lock | 13 + Cargo.toml | 1 + crates/dtmt/src/cmd/bundle/extract.rs | 13 +- lib/sdk/Cargo.toml | 1 + lib/sdk/src/binary.rs | 20 ++ lib/sdk/src/filetype/texture.rs | 456 +++++++++++++++++--------- lib/sdk/src/filetype/texture/dds.rs | 82 +++-- 7 files changed, 395 insertions(+), 191 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68fb0e5..eca8515 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1118,6 +1118,7 @@ checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", "libz-rs-sys", + "libz-sys", "miniz_oxide 0.8.8", ] @@ -2207,6 +2208,17 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "libz-sys" +version = "1.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.4.14" @@ -3352,6 +3364,7 @@ dependencies = [ "color-eyre", "csv-async", "fastrand", + "flate2", "futures", "futures-util", "glob", diff --git a/Cargo.toml b/Cargo.toml index c6243c8..fcb4ab8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ druid = { version = "0.8", features = ["im", "serde", "image", "png", "jpeg", "b druid-widget-nursery = "0.1" dtmt-shared = { path = "lib/dtmt-shared" } fastrand = "2.1.0" +flate2 = { version = "1.0.30", features = ["zlib"] } futures = "0.3.25" futures-util = "0.3.24" glob = "0.3.0" diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index b595dba..3790181 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -275,7 +275,13 @@ struct ExtractOptions<'a> { #[tracing::instrument( skip(ctx, options), - fields(decompile = options.decompile, flatten = options.flatten, dry_run = options.dry_run) + fields( + bundle_name = tracing::field::Empty, + bundle_hash = tracing::field::Empty, + decompile = options.decompile, + flatten = options.flatten, + dry_run = options.dry_run, + ) )] async fn extract_bundle( ctx: Arc, @@ -318,6 +324,11 @@ where let bundle = { let data = fs::read(path.as_ref()).await?; let name = Bundle::get_name_from_path(&ctx, path.as_ref()); + { + let span = tracing::span::Span::current(); + span.record("bundle_hash", format!("{:X}", name)); + span.record("bundle_name", name.display().to_string()); + } Bundle::from_binary(&ctx, name, data)? }; diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 0cd0e4e..9abbb23 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -10,6 +10,7 @@ byteorder = { workspace = true } color-eyre = { workspace = true } csv-async = { workspace = true } fastrand = { workspace = true } +flate2 = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } glob = { workspace = true } diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 83ccca0..40f9e9a 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -42,6 +42,26 @@ impl FromBinary for Vec { } } +pub fn flags_from_bits(bits: T::Bits) -> T +where + ::Bits: std::fmt::Binary, +{ + if let Some(flags) = T::from_bits(bits) { + flags + } else { + let unknown = bits & !T::all().bits(); + + tracing::warn!( + "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", + std::any::type_name::(), + T::all().bits(), + unknown + ); + + T::from_bits_truncate(bits) + } +} + pub mod sync { use std::ffi::CStr; use std::io::{self, Read, Seek, SeekFrom, Write}; diff --git a/lib/sdk/src/filetype/texture.rs b/lib/sdk/src/filetype/texture.rs index 1565b8c..88314ea 100644 --- a/lib/sdk/src/filetype/texture.rs +++ b/lib/sdk/src/filetype/texture.rs @@ -5,16 +5,15 @@ use bitflags::bitflags; use color_eyre::eyre::Context; use color_eyre::{eyre, SectionExt}; use color_eyre::{Help, Result}; -use num_traits::ToPrimitive as _; +use flate2::read::ZlibDecoder; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; use serde::{Deserialize, Serialize}; use tokio::fs; use crate::binary::sync::{ReadExt, WriteExt}; use crate::bundle::file::UserFile; -use crate::filetype::texture::dds::{DXGIFormat, ImageType}; use crate::murmur::{HashGroup, IdString32, IdString64}; -use crate::{BundleFile, BundleFileType, BundleFileVariant}; +use crate::{binary, BundleFile, BundleFileType, BundleFileVariant}; mod dds; @@ -43,7 +42,7 @@ struct TextureDefinitionOutput { } bitflags! { - #[derive(Clone, Copy, Debug)] + #[derive(Clone, Copy, Debug, Default)] struct TextureFlags: u32 { const STREAMABLE = 0b0000_0001; const UNKNOWN = 1 << 1; @@ -57,7 +56,7 @@ struct TextureHeaderMipInfo { size: usize, } -#[derive(Clone, Debug)] +#[derive(Clone, Default)] struct TextureHeader { flags: TextureFlags, n_streamable_mipmaps: usize, @@ -67,13 +66,33 @@ struct TextureHeader { meta_size: usize, } +impl std::fmt::Debug for TextureHeader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TextureHeader") + .field("flags", &self.flags) + .field("n_streamable_mipmaps", &self.n_streamable_mipmaps) + .field("width", &self.width) + .field("height", &self.height) + .field("mip_infos", &{ + let mut s = self + .mip_infos + .iter() + .fold(String::from("["), |mut s, info| { + s.push_str(&format!("{}/{}, ", info.offset, info.size)); + s + }); + s.push(']'); + s + }) + .field("meta_size", &self.meta_size) + .finish() + } +} + impl TextureHeader { #[tracing::instrument(skip(r))] fn from_binary(mut r: impl ReadExt) -> Result { - let flags = r.read_u32().and_then(|bits| { - TextureFlags::from_bits(bits) - .ok_or_else(|| eyre::eyre!("Unknown bits set in TextureFlags: {:032b}", bits)) - })?; + let flags = r.read_u32().map(binary::flags_from_bits)?; let n_streamable_mipmaps = r.read_u32()? as usize; let width = r.read_u32()? as usize; let height = r.read_u32()? as usize; @@ -121,7 +140,7 @@ impl TextureHeader { } } -#[derive(Clone, Debug)] +#[derive(Clone)] struct Texture { header: TextureHeader, data: Vec, @@ -129,6 +148,37 @@ struct Texture { category: IdString32, } +impl std::fmt::Debug for Texture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut out = f.debug_struct("Texture"); + out.field("header", &self.header); + + if self.data.len() <= 5 { + out.field("data", &format!("{:x?}", &self.data)); + } else { + out.field( + "data", + &format!("{:x?}.. ({} bytes)", &self.data[..5], &self.data.len()), + ); + } + + if let Some(stream) = self.stream.as_ref() { + if stream.len() <= 5 { + out.field("stream", &format!("{:x?}", &stream)); + } else { + out.field( + "stream", + &format!("{:x?}.. ({} bytes)", &stream[..5], &stream.len()), + ); + } + } else { + out.field("stream", &"None"); + } + + out.field("category", &self.category).finish() + } +} + impl Texture { #[tracing::instrument(skip(data, chunks))] fn decompress_stream_data(mut data: impl Read, chunks: impl AsRef<[usize]>) -> Result> { @@ -214,35 +264,83 @@ impl Texture { Ok(out) } - #[tracing::instrument(skip(ctx, r, stream_r))] + #[tracing::instrument( + "Texture::from_binary", + skip(ctx, r, stream_r), + fields( + compression_type = tracing::field::Empty, + compressed_size = tracing::field::Empty, + uncompressed_size = tracing::field::Empty, + ) + )] fn from_binary( ctx: &crate::Context, mut r: impl Read + Seek, mut stream_r: Option, ) -> Result { - // Looking at the executable in IDA, there is one other valid value: `2`. - // If this ever comes up in the game data, I'll have to reverse engineer the - // (de)compression algorithm through IDA. let compression_type = r.read_u32()?; - eyre::ensure!( - compression_type == 1, - "Unknown compression type for texture '{}'", - compression_type - ); - let compressed_size = r.read_u32()? as usize; let uncompressed_size = r.read_u32()? as usize; - let out_buf = { - let mut comp_buf = vec![0; compressed_size]; - r.read_exact(&mut comp_buf)?; + { + let span = tracing::Span::current(); + span.record("compression_type", compression_type); + span.record("compressed_size", compressed_size); + span.record("uncompressed_size", uncompressed_size); + } - oodle::decompress( + let mut comp_buf = vec![0; compressed_size]; + r.read_exact(&mut comp_buf)?; + + let out_buf = match compression_type { + // Uncompressed + // This one never seems to contain the additional `TextureHeader` metadata, + // so we return early in this branch. + 0 => { + eyre::ensure!( + compressed_size == 0 && uncompressed_size == 0, + "Cannot handle texture with compression_type == 0, but buffer sizes > 0" + ); + tracing::trace!("Found raw texture"); + + let pos = r.stream_position()?; + let end = { + r.seek(SeekFrom::End(0))?; + let end = r.stream_position()?; + r.seek(SeekFrom::Start(pos))?; + end + }; + + // Reads until the last u32. + let mut data = vec![0u8; (end - pos - 4) as usize]; + r.read_exact(&mut data)?; + + let category = r.read_u32().map(IdString32::from)?; + + return Ok(Self { + header: TextureHeader::default(), + data, + stream: None, + category, + }); + } + 1 => oodle::decompress( comp_buf, uncompressed_size, OodleLZ_FuzzSafe::No, OodleLZ_CheckCRC::No, - )? + )?, + 2 => { + let mut decoder = ZlibDecoder::new(comp_buf.as_slice()); + let mut buf = Vec::with_capacity(uncompressed_size); + + decoder.read_to_end(&mut buf)?; + buf + } + _ => eyre::bail!( + "Unknown compression type for texture '{}'", + compression_type + ), }; eyre::ensure!( @@ -339,6 +437,129 @@ impl Texture { serde_sjson::to_string(&texture).wrap_err("Failed to serialize texture definition") } + #[tracing::instrument(fields( + dds_header = tracing::field::Empty, + dx10_header = tracing::field::Empty, + image_format = tracing::field::Empty, + ))] + fn create_dds_user_file(&self, name: String) -> Result { + let mut data = Cursor::new(&self.data); + let mut dds_header = + dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; + + { + let span = tracing::Span::current(); + span.record("dds_header", format!("{:?}", dds_header)); + } + + if !dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) { + tracing::debug!("Found DDS without FourCC. Dumping raw data"); + return Ok(UserFile::with_name(self.data.clone(), name)); + } + + // eyre::ensure!( + // dds_header.pixel_format.four_cc == dds::FourCC::DX10, + // "Only DX10 textures are currently supported. FourCC == {}", + // dds_header.pixel_format.four_cc, + // ); + + let dx10_header = + dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; + + { + let span = tracing::Span::current(); + span.record("dx10_header", format!("{:?}", dx10_header)); + } + + // match dx10_header.dxgi_format { + // DXGIFormat::BC1_UNORM + // | DXGIFormat::BC3_UNORM + // | DXGIFormat::BC4_UNORM + // | DXGIFormat::BC5_UNORM + // | DXGIFormat::BC6H_UF16 + // | DXGIFormat::BC7_UNORM => {} + // _ => { + // eyre::bail!( + // "Unsupported DXGI format: {} (0x{:0X})", + // dx10_header.dxgi_format, + // dx10_header.dxgi_format.to_u32().unwrap_or_default() + // ); + // } + // } + + let stingray_image_format = dds::stripped_format_from_header(&dds_header, &dx10_header)?; + { + let span = tracing::Span::current(); + span.record("image_format", format!("{:?}", stingray_image_format)); + } + + // eyre::ensure!( + // stingray_image_format.image_type == ImageType::Image2D, + // "Unsupported image type: {}", + // stingray_image_format.image_type, + // ); + + let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; + let bits_per_block: usize = match block_size { + 8 => 128, + 16 => 64, + block_size => eyre::bail!("Unsupported block size {}", block_size), + }; + + let pitch = self.header.width / 4 * block_size; + let bytes_per_block = self.header.width / bits_per_block / 4; + + tracing::debug!( + "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", + block_size, + pitch, + bits_per_block, + bytes_per_block + ); + + let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); + + // Currently, we only extract the largest mipmap, + // so we need to set the dimensions accordingly, and remove the + // flag. + dds_header.width = self.header.width; + dds_header.height = self.header.height; + dds_header.mipmap_count = 0; + dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; + + dds_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DDS header")?; + + dx10_header + .to_binary(&mut out_data) + .wrap_err("Failed to write DX10 header")?; + + // If there is stream data, we build the mipmap data from it. + // If not, we take whatever is left in the bundle file. + if let Some(stream) = &self.stream { + let data = Self::reorder_stream_mipmap( + stream, + bits_per_block, + bytes_per_block, + block_size, + pitch, + ) + .wrap_err("Failed to reorder stream chunks")?; + + out_data + .write_all(&data) + .wrap_err("Failed to write streamed mipmap data")?; + } else { + let (_, remaining) = data.split(); + out_data + .write_all(remaining) + .wrap_err("Failed to write texture data")?; + }; + + Ok(UserFile::with_name(out_data.into_inner(), name)) + } + #[tracing::instrument(skip(self))] fn to_user_files(&self, name: String) -> Result> { let mut files = Vec::with_capacity(2); @@ -369,122 +590,38 @@ impl Texture { files.push(UserFile::with_name(self.data.clone(), name)); } + match self + .create_dds_user_file(name) + .wrap_err("Failed to create DDS file") { - let mut data = Cursor::new(&self.data); - let mut dds_header = - dds::DDSHeader::from_binary(&mut data).wrap_err("Failed to read DDS header")?; - - eyre::ensure!( - dds_header.pixel_format.flags.contains(dds::DDPF::FOURCC) - && dds_header.pixel_format.four_cc == dds::FOURCC_DX10, - "Only DX10 textures are currently supported." - ); - - let dx10_header = - dds::Dx10Header::from_binary(&mut data).wrap_err("Failed to read DX10 header")?; - - match dx10_header.dxgi_format { - DXGIFormat::BC1_UNORM - | DXGIFormat::BC3_UNORM - | DXGIFormat::BC4_UNORM - | DXGIFormat::BC5_UNORM - | DXGIFormat::BC6H_UF16 - | DXGIFormat::BC7_UNORM => {} - _ => { - eyre::bail!( - "Unsupported DXGI format: {} (0x{:0X})", - dx10_header.dxgi_format, - dx10_header.dxgi_format.to_u32().unwrap_or_default() + Ok(dds) => files.push(dds), + Err(err) => { + if cfg!(debug_assertions) { + tracing::error!( + "{:?}", + err.with_section(|| { + "Running in debug mode, continuing to produce raw files".header("Note:") + }) ); + } else { + return Err(err); } } - - let stingray_image_format = - dds::stripped_format_from_header(&dds_header, &dx10_header)?; - eyre::ensure!( - stingray_image_format.image_type == ImageType::Image2D, - "Unsupported image type: {}", - stingray_image_format.image_type, - ); - - let block_size = 4 * dds_header.pitch_or_linear_size / dds_header.width; - let bits_per_block: usize = match block_size { - 8 => 128, - 16 => 64, - block_size => eyre::bail!("Unsupported block size {}", block_size), - }; - - let pitch = self.header.width / 4 * block_size; - let bytes_per_block = self.header.width / bits_per_block / 4; - - tracing::debug!( - "block_size = {} | pitch = {} | bits_per_block = {} | bytes_per_block = {}", - block_size, - pitch, - bits_per_block, - bytes_per_block - ); - - let mut out_data = Cursor::new(Vec::with_capacity(self.data.len())); - - // Currently, we only extract the largest mipmap, - // so we need to set the dimensions accordingly, and remove the - // flag. - dds_header.width = self.header.width; - dds_header.height = self.header.height; - dds_header.mipmap_count = 0; - dds_header.flags &= !dds::DDSD::MIPMAPCOUNT; - - dds_header - .to_binary(&mut out_data) - .wrap_err("Failed to write DDS header")?; - - dx10_header - .to_binary(&mut out_data) - .wrap_err("Failed to write DX10 header")?; - - if let Some(stream) = &self.stream { - let data = Self::reorder_stream_mipmap( - stream, - bits_per_block, - bytes_per_block, - block_size, - pitch, - ) - .wrap_err("Failed to reorder stream chunks")?; - - out_data - .write_all(&data) - .wrap_err("Failed to write streamed mipmap data")?; - } else { - out_data - .write_all(data.split().1) - .wrap_err("Failed to write texture data")?; - }; - - files.push(UserFile::with_name(out_data.into_inner(), name)); - } + }; Ok(files) } } -#[tracing::instrument(skip(ctx, data), fields(buf_len = data.as_ref().len()))] +#[tracing::instrument(skip(ctx, data, stream_data), fields(data_len = data.as_ref().len()))] pub(crate) async fn decompile_data( ctx: &crate::Context, name: String, data: impl AsRef<[u8]>, - stream_file_name: Option, + stream_data: Option>, ) -> Result> { - let mut r = Cursor::new(data.as_ref()); - let mut stream_r = if let Some(file_name) = stream_file_name { - let stream_data = fs::read(&file_name) - .await - .wrap_err_with(|| format!("Failed to read stream file '{}'", file_name.display()))?; - Some(Cursor::new(stream_data)) - } else { - None - }; + let mut r = Cursor::new(data); + let mut stream_r = stream_data.map(Cursor::new); let texture = Texture::from_binary(ctx, &mut r, stream_r.as_mut())?; texture @@ -498,38 +635,47 @@ pub(crate) async fn decompile( name: String, variant: &BundleFileVariant, ) -> Result> { - if !variant.external() { + let data_file = variant.data_file_name().map(|name| match &ctx.game_dir { + Some(dir) => dir.join("bundle").join(name), + None => PathBuf::from("bundle").join(name), + }); + + if variant.external() { + let Some(path) = data_file else { + eyre::bail!("File is marked external but has no data file name"); + }; + + tracing::debug!( + "Decompiling texture from external file '{}'", + path.display() + ); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + + decompile_data(ctx, name, data, None::<&[u8]>).await + } else { tracing::debug!("Decompiling texture from bundle data"); - let stream_file_name = variant.data_file_name().map(|name| match &ctx.game_dir { - Some(dir) => dir.join("bundle").join(name), - None => PathBuf::from("bundle").join(name), - }); + let stream_data = match data_file { + Some(path) => { + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) + .with_suggestion(|| { + "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." + })?; + Some(data) + } + None => None, + }; - return decompile_data(ctx, name, variant.data(), stream_file_name).await; + decompile_data(ctx, name, variant.data(), stream_data).await } - - let Some(file_name) = variant.data_file_name() else { - eyre::bail!("Texture file has no data and no data file"); - }; - - tracing::debug!("Decompiling texture from external file '{}'", file_name); - - let path = match &ctx.game_dir { - Some(dir) => dir.join("bundle").join(file_name), - None => PathBuf::from("bundle").join(file_name), - }; - - tracing::trace!(path = %path.display()); - - let data = fs::read(&path) - .await - .wrap_err_with(|| format!("Failed to read data file '{}'", path.display())) - .with_suggestion(|| { - "Provide a game directory in the config file or make sure the `data` directory is next to the provided bundle." - })?; - - decompile_data(ctx, name, &data, None).await } #[tracing::instrument(skip(sjson, name), fields(sjson_len = sjson.as_ref().len(), name = %name.display()))] diff --git a/lib/sdk/src/filetype/texture/dds.rs b/lib/sdk/src/filetype/texture/dds.rs index 5ef1b90..ca0b73f 100644 --- a/lib/sdk/src/filetype/texture/dds.rs +++ b/lib/sdk/src/filetype/texture/dds.rs @@ -7,10 +7,10 @@ use color_eyre::Result; use num_derive::{FromPrimitive, ToPrimitive}; use num_traits::{FromPrimitive as _, ToPrimitive as _}; +use crate::binary; use crate::binary::sync::{ReadExt, WriteExt}; const MAGIC_DDS: u32 = 0x20534444; -pub const FOURCC_DX10: u32 = 0x30315844; bitflags! { #[derive(Clone, Copy, Debug)] @@ -75,27 +75,8 @@ bitflags! { } } -fn flags_from_bits(bits: T::Bits) -> T -where - ::Bits: std::fmt::Binary, -{ - if let Some(flags) = T::from_bits(bits) { - flags - } else { - let unknown = bits & !T::all().bits(); - - tracing::warn!( - "Unknown bits found for '{}': known = {:0b}, unknown = {:0b}", - std::any::type_name::(), - T::all().bits(), - unknown - ); - - T::from_bits_truncate(bits) - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, FromPrimitive, ToPrimitive)] +#[repr(u32)] pub enum D3D10ResourceDimension { Unknown = 0, Buffer = 1, @@ -107,6 +88,7 @@ pub enum D3D10ResourceDimension { #[allow(clippy::upper_case_acronyms)] #[allow(non_camel_case_types)] #[derive(Clone, Copy, Debug, strum::Display, FromPrimitive, ToPrimitive)] +#[repr(u32)] pub enum DXGIFormat { UNKNOWN = 0, R32G32B32A32_TYPELESS = 1, @@ -243,7 +225,7 @@ pub struct Dx10Header { } impl Dx10Header { - #[tracing::instrument(skip(r))] + #[tracing::instrument("Dx10Header::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { let dxgi_format = r .read_u32() @@ -251,7 +233,7 @@ impl Dx10Header { let resource_dimension = r.read_u32().map(|val| { D3D10ResourceDimension::from_u32(val).unwrap_or(D3D10ResourceDimension::Unknown) })?; - let misc_flag = r.read_u32().map(flags_from_bits)?; + let misc_flag = r.read_u32().map(binary::flags_from_bits)?; let array_size = r.read_u32()? as usize; let misc_flags2 = r.read_u32()?; @@ -264,7 +246,7 @@ impl Dx10Header { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("Dx10Header::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { w.write_u32( self.dxgi_format @@ -284,10 +266,30 @@ impl Dx10Header { } } +#[allow(non_camel_case_types)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum FourCC { + Empty = u32::MAX, + DXT1 = 0x31545844, + DXT2 = 0x33545844, + DXT5 = 0x35545844, + AXI1 = 0x31495441, + AXI2 = 0x32495441, + DX10 = 0x30315844, + D3D_A16B16G16R16 = 0x24, + D3D_R16F = 0x6F, + D3D_G16R16F = 0x70, + D3D_A16B16G16R16F = 0x71, + D3D_R32F = 0x72, + D3D_G32R32F = 0x73, + D3D_A32B32G32R32F = 0x74, +} + #[derive(Clone, Copy, Debug)] pub struct DDSPixelFormat { pub flags: DDPF, - pub four_cc: u32, + pub four_cc: FourCC, pub rgb_bit_count: u32, pub r_bit_mask: u32, pub g_bit_mask: u32, @@ -296,7 +298,7 @@ pub struct DDSPixelFormat { } impl DDSPixelFormat { - #[tracing::instrument(skip(r))] + #[tracing::instrument("DDSPixelFormat::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { let size = r.read_u32()? as usize; eyre::ensure!( @@ -305,8 +307,17 @@ impl DDSPixelFormat { size ); - let flags = r.read_u32().map(flags_from_bits)?; - let four_cc = r.read_u32()?; + let flags: DDPF = r.read_u32().map(binary::flags_from_bits)?; + + let four_cc = if flags.contains(DDPF::FOURCC) { + r.read_u32().and_then(|bytes| { + FourCC::from_u32(bytes).ok_or_eyre(format!("Unknown FourCC value: {:08X}", bytes)) + })? + } else { + r.skip_u32(0)?; + FourCC::Empty + }; + let rgb_bit_count = r.read_u32()?; let r_bit_mask = r.read_u32()?; let g_bit_mask = r.read_u32()?; @@ -324,13 +335,13 @@ impl DDSPixelFormat { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("DDSPixelFormat::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { // Structure size w.write_u32(32)?; w.write_u32(self.flags.bits())?; - w.write_u32(self.four_cc)?; + w.write_u32(self.four_cc.to_u32().unwrap_or_default())?; w.write_u32(self.rgb_bit_count)?; w.write_u32(self.r_bit_mask)?; w.write_u32(self.g_bit_mask)?; @@ -356,7 +367,7 @@ pub struct DDSHeader { } impl DDSHeader { - #[tracing::instrument(skip(r))] + #[tracing::instrument("DDSHeader::from_binary", skip(r))] pub fn from_binary(mut r: impl ReadExt) -> Result { r.skip_u32(MAGIC_DDS).wrap_err("Invalid magic bytes")?; @@ -367,7 +378,7 @@ impl DDSHeader { size ); - let flags = r.read_u32().map(flags_from_bits)?; + let flags = r.read_u32().map(binary::flags_from_bits)?; let height = r.read_u32()? as usize; let width = r.read_u32()? as usize; let pitch_or_linear_size = r.read_u32()? as usize; @@ -378,8 +389,8 @@ impl DDSHeader { r.seek(SeekFrom::Current(11 * 4))?; let pixel_format = DDSPixelFormat::from_binary(&mut r)?; - let caps = r.read_u32().map(flags_from_bits)?; - let caps_2 = r.read_u32().map(flags_from_bits)?; + let caps = r.read_u32().map(binary::flags_from_bits)?; + let caps_2 = r.read_u32().map(binary::flags_from_bits)?; // Skip unused and reserved bytes r.seek(SeekFrom::Current(3 * 4))?; @@ -397,7 +408,7 @@ impl DDSHeader { }) } - #[tracing::instrument(skip(w))] + #[tracing::instrument("DDSHeader::to_binary", skip(w))] pub fn to_binary(&self, mut w: impl WriteExt) -> Result<()> { w.write_u32(MAGIC_DDS)?; @@ -423,6 +434,7 @@ impl DDSHeader { } #[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display)] +#[repr(u32)] pub enum ImageType { Image2D = 0, Image3D = 1, From cbb3709c89194bf7ba2c504aea5aad41253449dd Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 29 Jul 2024 15:10:20 +0200 Subject: [PATCH 34/35] sdk: Remove unused function --- lib/sdk/src/binary.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 40f9e9a..675dc43 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -152,7 +152,6 @@ pub mod sync { make_read!(read_u32, read_u32_le, u32); make_read!(read_u64, read_u64_le, u64); - make_skip!(skip_u8, read_u8, u8); make_skip!(skip_u16, read_u16, u16); make_skip!(skip_u32, read_u32, u32); From 04b6a43f9a78dc717acc96ba500e7d8197b84978 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 6 Mar 2025 13:32:47 +0100 Subject: [PATCH 35/35] Only use texture files for texture-meta command --- crates/dtmt/src/cmd/experiment/texture_meta.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/dtmt/src/cmd/experiment/texture_meta.rs b/crates/dtmt/src/cmd/experiment/texture_meta.rs index 98d0035..a77e693 100644 --- a/crates/dtmt/src/cmd/experiment/texture_meta.rs +++ b/crates/dtmt/src/cmd/experiment/texture_meta.rs @@ -5,7 +5,7 @@ use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use color_eyre::eyre::Context; use color_eyre::Result; use futures_util::StreamExt; -use sdk::Bundle; +use sdk::{Bundle, BundleFileType}; use tokio::fs; use crate::cmd::util::resolve_bundle_paths; @@ -46,6 +46,10 @@ async fn handle_bundle(ctx: &sdk::Context, path: &PathBuf) -> Result<()> { .unwrap_or_default(); for f in bundle.files().iter() { + if f.file_type() != BundleFileType::Texture { + continue; + } + for (i, v) in f.variants().iter().enumerate() { let data_file_name = v.data_file_name();