From 43e3bf7b60de613258f4da7fc116bebaefc9c1f1 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 26 Jul 2024 16:03:04 +0200 Subject: [PATCH 1/3] Add cmdline to tracing output Can come in handy when other people report problems and show the error message or full log, but not the command line. Setting that span to `level = "error"` ensures that it won't be disabled by level filters. --- crates/dtmt/src/main.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index 2e10b17..e41e802 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -36,10 +36,21 @@ struct GlobalConfig { } #[tokio::main] -#[tracing::instrument] +#[tracing::instrument(level = "error", fields(cmd_line = tracing::field::Empty))] async fn main() -> Result<()> { color_eyre::install()?; + { + let span = tracing::Span::current(); + if !span.is_disabled() { + let cmdline: String = std::env::args_os().fold(String::new(), |mut s, arg| { + s.push_str(&arg.to_string_lossy()); + s + }); + span.record("cmd_line", cmdline); + } + } + let matches = command!() .subcommand_required(true) .arg( From 636279edfe376bd9d748347c7195d4e82af90015 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 21 Feb 2025 14:51:42 +0100 Subject: [PATCH 2/3] Use macro to generate file type enum and impls Due to the large amount of variants, and the different kind of values connected to each variant (hash, extension name) being scattered across the various `impl` blocks, the file became rather convoluted. While I don't generally like the indirection of macros or meta programming, it's not that bad with Rust, thanks to Rust Analyzer being able to attach diagnostics to the source inside the macro definition, and the ability to generate the macro's output for validation. Therefore, the new macro allows putting all data used for this enum definition into a single block. --- lib/sdk/src/bundle/filetype.rs | 490 +++++++++------------------------ 1 file changed, 132 insertions(+), 358 deletions(-) diff --git a/lib/sdk/src/bundle/filetype.rs b/lib/sdk/src/bundle/filetype.rs index 0b4f292..68ff6b5 100644 --- a/lib/sdk/src/bundle/filetype.rs +++ b/lib/sdk/src/bundle/filetype.rs @@ -3,232 +3,147 @@ use serde::Serialize; use crate::murmur::Murmur64; -#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] -pub enum BundleFileType { - Animation, - AnimationCurves, - Apb, - BakedLighting, - Bik, - BlendSet, - Bones, - Chroma, - CommonPackage, - Config, - Crypto, - Data, - Entity, - Flow, - Font, - Ies, - Ini, - Input, - Ivf, - Keys, - Level, - Lua, - Material, - Mod, - MouseCursor, - NavData, - NetworkConfig, - OddleNet, - Package, - Particles, - PhysicsProperties, - RenderConfig, - RtPipeline, - Scene, - Shader, - ShaderLibrary, - ShaderLibraryGroup, - ShadingEnvionmentMapping, - ShadingEnvironment, - Slug, - SlugAlbum, - SoundEnvironment, - SpuJob, - StateMachine, - StaticPVS, - Strings, - SurfaceProperties, - Texture, - TimpaniBank, - TimpaniMaster, - Tome, - Ugg, - Unit, - Upb, - VectorField, - Wav, - WwiseBank, - WwiseDep, - WwiseEvent, - WwiseMetadata, - WwiseStream, - Xml, +macro_rules! make_enum { + ( + $( $variant:ident, $hash:expr, $ext:expr $(, $decompiled:expr)? ; )+ + ) => { + #[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] + pub enum BundleFileType { + $( + $variant, + )+ + Unknown(Murmur64), + } - Unknown(Murmur64), + impl BundleFileType { + pub fn ext_name(&self) -> String { + match self { + $( + Self::$variant => String::from($ext), + )+ + Self::Unknown(s) => format!("{s:016X}"), + } + } + + pub fn decompiled_ext_name(&self) -> String { + match self { + $( + $( Self::$variant => String::from($decompiled), )? + )+ + _ => self.ext_name(), + } + } + } + + impl std::str::FromStr for BundleFileType { + type Err = color_eyre::Report; + fn from_str(s: &str) -> Result { + match s { + $( + $ext => Ok(Self::$variant), + )+ + s => eyre::bail!("Unknown type string '{}'", s), + } + } + } + + impl From for BundleFileType { + fn from(h: u64) -> Self { + match h { + $( + $hash => Self::$variant, + )+ + hash => Self::Unknown(hash.into()), + } + } + } + + impl From for u64 { + fn from(t: BundleFileType) -> u64 { + match t { + $( + BundleFileType::$variant => $hash, + )+ + BundleFileType::Unknown(hash) => hash.into(), + } + } + } + } +} + +make_enum! { + AnimationCurves, 0xdcfb9e18fff13984, "animation_curves"; + Animation, 0x931e336d7646cc26, "animation"; + Apb, 0x3eed05ba83af5090, "apb"; + BakedLighting, 0x7ffdb779b04e4ed1, "baked_lighting"; + Bik, 0xaa5965f03029fa18, "bik"; + BlendSet, 0xe301e8af94e3b5a3, "blend_set"; + Bones, 0x18dead01056b72e9, "bones"; + Chroma, 0xb7893adf7567506a, "chroma"; + CommonPackage, 0xfe9754bd19814a47, "common_package"; + Config, 0x82645835e6b73232, "config"; + Crypto, 0x69108ded1e3e634b, "crypto"; + Data, 0x8fd0d44d20650b68, "data"; + Entity, 0x9831ca893b0d087d, "entity"; + Flow, 0x92d3ee038eeb610d, "flow"; + Font, 0x9efe0a916aae7880, "font"; + Ies, 0x8f7d5a2c0f967655, "ies"; + Ini, 0xd526a27da14f1dc5, "ini"; + Input, 0x2bbcabe5074ade9e, "input"; + Ivf, 0xfa4a8e091a91201e, "ivf"; + Keys, 0xa62f9297dc969e85, "keys"; + Level, 0x2a690fd348fe9ac5, "level"; + Lua, 0xa14e8dfa2cd117e2, "lua"; + Material, 0xeac0b497876adedf, "material"; + Mod, 0x3fcdd69156a46417, "mod"; + MouseCursor, 0xb277b11fe4a61d37, "mouse_cursor"; + NavData, 0x169de9566953d264, "nav_data"; + NetworkConfig, 0x3b1fa9e8f6bac374, "network_config"; + OddleNet, 0xb0f2c12eb107f4d8, "oodle_net"; + Package, 0xad9c6d9ed1e5e77a, "package"; + Particles, 0xa8193123526fad64, "particles"; + PhysicsProperties, 0xbf21403a3ab0bbb1, "physics_properties"; + RenderConfig, 0x27862fe24795319c, "render_config"; + RtPipeline, 0x9ca183c2d0e76dee, "rt_pipeline"; + Scene, 0x9d0a795bfe818d19, "scene"; + Shader, 0xcce8d5b5f5ae333f, "shader"; + ShaderLibrary, 0xe5ee32a477239a93, "shader_library"; + ShaderLibraryGroup, 0x9e5c3cc74575aeb5, "shader_library_group"; + ShadingEnvionmentMapping, 0x250e0a11ac8e26f8, "shading_envionment_mapping"; + ShadingEnvironment, 0xfe73c7dcff8a7ca5, "shading_environment"; + Slug, 0xa27b4d04a9ba6f9e, "slug"; + SlugAlbum, 0xe9fc9ea7042e5ec0, "slug_album"; + SoundEnvironment, 0xd8b27864a97ffdd7, "sound_environment"; + SpuJob, 0xf97af9983c05b950, "spu_job"; + StateMachine, 0xa486d4045106165c, "state_machine"; + StaticPVS, 0xe3f0baa17d620321, "static_pvs"; + Strings, 0x0d972bab10b40fd3, "strings"; + SurfaceProperties, 0xad2d3fa30d9ab394, "surface_properties"; + Texture, 0xcd4238c6a0c69e32, "texture", "dds"; + TimpaniBank, 0x99736be1fff739a4, "timpani_bank"; + TimpaniMaster, 0x00a3e6c59a2b9c6c, "timpani_master"; + Tome, 0x19c792357c99f49b, "tome"; + Ugg, 0x712d6e3dd1024c9c, "ugg"; + Unit, 0xe0a48d0be9a7453f, "unit"; + Upb, 0xa99510c6e86dd3c2, "upb"; + VectorField, 0xf7505933166d6755, "vector_field"; + Wav, 0x786f65c00a816b19, "wav"; + WwiseBank, 0x535a7bd3e650d799, "wwise_bank", "bnk"; + WwiseDep, 0xaf32095c82f2b070, "wwise_dep"; + WwiseEvent, 0xaabdd317b58dfc8a, "wwise_event"; + WwiseMetadata, 0xd50a8b7e1c82b110, "wwise_metadata"; + WwiseStream, 0x504b55235d21440e, "wwise_stream", "ogg"; + Xml, 0x76015845a6003765, "xml"; + Theme, 0x38BB9442048A7FBD, "theme"; + MissionThemes, 0x80F2DE893657F83A, "mission_themes"; } impl BundleFileType { - pub fn ext_name(&self) -> String { - match self { - BundleFileType::AnimationCurves => String::from("animation_curves"), - BundleFileType::Animation => String::from("animation"), - BundleFileType::Apb => String::from("apb"), - BundleFileType::BakedLighting => String::from("baked_lighting"), - BundleFileType::Bik => String::from("bik"), - BundleFileType::BlendSet => String::from("blend_set"), - BundleFileType::Bones => String::from("bones"), - BundleFileType::Chroma => String::from("chroma"), - BundleFileType::CommonPackage => String::from("common_package"), - BundleFileType::Config => String::from("config"), - BundleFileType::Crypto => String::from("crypto"), - BundleFileType::Data => String::from("data"), - BundleFileType::Entity => String::from("entity"), - BundleFileType::Flow => String::from("flow"), - BundleFileType::Font => String::from("font"), - BundleFileType::Ies => String::from("ies"), - BundleFileType::Ini => String::from("ini"), - BundleFileType::Input => String::from("input"), - BundleFileType::Ivf => String::from("ivf"), - BundleFileType::Keys => String::from("keys"), - BundleFileType::Level => String::from("level"), - BundleFileType::Lua => String::from("lua"), - BundleFileType::Material => String::from("material"), - BundleFileType::Mod => String::from("mod"), - BundleFileType::MouseCursor => String::from("mouse_cursor"), - BundleFileType::NavData => String::from("nav_data"), - BundleFileType::NetworkConfig => String::from("network_config"), - BundleFileType::OddleNet => String::from("oodle_net"), - BundleFileType::Package => String::from("package"), - BundleFileType::Particles => String::from("particles"), - BundleFileType::PhysicsProperties => String::from("physics_properties"), - BundleFileType::RenderConfig => String::from("render_config"), - BundleFileType::RtPipeline => String::from("rt_pipeline"), - BundleFileType::Scene => String::from("scene"), - BundleFileType::ShaderLibraryGroup => String::from("shader_library_group"), - BundleFileType::ShaderLibrary => String::from("shader_library"), - BundleFileType::Shader => String::from("shader"), - BundleFileType::ShadingEnvionmentMapping => String::from("shading_environment_mapping"), - BundleFileType::ShadingEnvironment => String::from("shading_environment"), - BundleFileType::SlugAlbum => String::from("slug_album"), - BundleFileType::Slug => String::from("slug"), - BundleFileType::SoundEnvironment => String::from("sound_environment"), - BundleFileType::SpuJob => String::from("spu_job"), - BundleFileType::StateMachine => String::from("state_machine"), - BundleFileType::StaticPVS => String::from("static_pvs"), - BundleFileType::Strings => String::from("strings"), - BundleFileType::SurfaceProperties => String::from("surface_properties"), - BundleFileType::Texture => String::from("texture"), - BundleFileType::TimpaniBank => String::from("timpani_bank"), - BundleFileType::TimpaniMaster => String::from("timpani_master"), - BundleFileType::Tome => String::from("tome"), - BundleFileType::Ugg => String::from("ugg"), - BundleFileType::Unit => String::from("unit"), - BundleFileType::Upb => String::from("upb"), - BundleFileType::VectorField => String::from("vector_field"), - BundleFileType::Wav => String::from("wav"), - BundleFileType::WwiseBank => String::from("wwise_bank"), - BundleFileType::WwiseDep => String::from("wwise_dep"), - BundleFileType::WwiseEvent => String::from("wwise_event"), - BundleFileType::WwiseMetadata => String::from("wwise_metadata"), - BundleFileType::WwiseStream => String::from("wwise_stream"), - BundleFileType::Xml => String::from("xml"), - - BundleFileType::Unknown(s) => format!("{s:016X}"), - } - } - - pub fn decompiled_ext_name(&self) -> String { - match self { - BundleFileType::Texture => String::from("dds"), - BundleFileType::WwiseBank => String::from("bnk"), - BundleFileType::WwiseStream => String::from("ogg"), - _ => self.ext_name(), - } - } - pub fn hash(&self) -> Murmur64 { Murmur64::from(*self) } } -impl std::str::FromStr for BundleFileType { - type Err = color_eyre::Report; - - fn from_str(s: &str) -> Result { - let val = match s { - "animation_curves" => BundleFileType::AnimationCurves, - "animation" => BundleFileType::Animation, - "apb" => BundleFileType::Apb, - "baked_lighting" => BundleFileType::BakedLighting, - "bik" => BundleFileType::Bik, - "blend_set" => BundleFileType::BlendSet, - "bones" => BundleFileType::Bones, - "chroma" => BundleFileType::Chroma, - "common_package" => BundleFileType::CommonPackage, - "config" => BundleFileType::Config, - "crypto" => BundleFileType::Crypto, - "data" => BundleFileType::Data, - "entity" => BundleFileType::Entity, - "flow" => BundleFileType::Flow, - "font" => BundleFileType::Font, - "ies" => BundleFileType::Ies, - "ini" => BundleFileType::Ini, - "input" => BundleFileType::Input, - "ivf" => BundleFileType::Ivf, - "keys" => BundleFileType::Keys, - "level" => BundleFileType::Level, - "lua" => BundleFileType::Lua, - "material" => BundleFileType::Material, - "mod" => BundleFileType::Mod, - "mouse_cursor" => BundleFileType::MouseCursor, - "nav_data" => BundleFileType::NavData, - "network_config" => BundleFileType::NetworkConfig, - "oodle_net" => BundleFileType::OddleNet, - "package" => BundleFileType::Package, - "particles" => BundleFileType::Particles, - "physics_properties" => BundleFileType::PhysicsProperties, - "render_config" => BundleFileType::RenderConfig, - "rt_pipeline" => BundleFileType::RtPipeline, - "scene" => BundleFileType::Scene, - "shader_library_group" => BundleFileType::ShaderLibraryGroup, - "shader_library" => BundleFileType::ShaderLibrary, - "shader" => BundleFileType::Shader, - "shading_environment_mapping" => BundleFileType::ShadingEnvionmentMapping, - "shading_environment" => BundleFileType::ShadingEnvironment, - "slug_album" => BundleFileType::SlugAlbum, - "slug" => BundleFileType::Slug, - "sound_environment" => BundleFileType::SoundEnvironment, - "spu_job" => BundleFileType::SpuJob, - "state_machine" => BundleFileType::StateMachine, - "static_pvs" => BundleFileType::StaticPVS, - "strings" => BundleFileType::Strings, - "surface_properties" => BundleFileType::SurfaceProperties, - "texture" => BundleFileType::Texture, - "timpani_bank" => BundleFileType::TimpaniBank, - "timpani_master" => BundleFileType::TimpaniMaster, - "tome" => BundleFileType::Tome, - "ugg" => BundleFileType::Ugg, - "unit" => BundleFileType::Unit, - "upb" => BundleFileType::Upb, - "vector_field" => BundleFileType::VectorField, - "wav" => BundleFileType::Wav, - "wwise_bank" => BundleFileType::WwiseBank, - "wwise_dep" => BundleFileType::WwiseDep, - "wwise_event" => BundleFileType::WwiseEvent, - "wwise_metadata" => BundleFileType::WwiseMetadata, - "wwise_stream" => BundleFileType::WwiseStream, - "xml" => BundleFileType::Xml, - s => eyre::bail!("Unknown type string '{}'", s), - }; - - Ok(val) - } -} - impl Serialize for BundleFileType { fn serialize(&self, serializer: S) -> Result where @@ -245,147 +160,6 @@ impl From for BundleFileType { } } -impl From for BundleFileType { - fn from(hash: u64) -> BundleFileType { - match hash { - 0x931e336d7646cc26 => BundleFileType::Animation, - 0xdcfb9e18fff13984 => BundleFileType::AnimationCurves, - 0x3eed05ba83af5090 => BundleFileType::Apb, - 0x7ffdb779b04e4ed1 => BundleFileType::BakedLighting, - 0xaa5965f03029fa18 => BundleFileType::Bik, - 0xe301e8af94e3b5a3 => BundleFileType::BlendSet, - 0x18dead01056b72e9 => BundleFileType::Bones, - 0xb7893adf7567506a => BundleFileType::Chroma, - 0xfe9754bd19814a47 => BundleFileType::CommonPackage, - 0x82645835e6b73232 => BundleFileType::Config, - 0x69108ded1e3e634b => BundleFileType::Crypto, - 0x8fd0d44d20650b68 => BundleFileType::Data, - 0x9831ca893b0d087d => BundleFileType::Entity, - 0x92d3ee038eeb610d => BundleFileType::Flow, - 0x9efe0a916aae7880 => BundleFileType::Font, - 0x8f7d5a2c0f967655 => BundleFileType::Ies, - 0xd526a27da14f1dc5 => BundleFileType::Ini, - 0x2bbcabe5074ade9e => BundleFileType::Input, - 0xfa4a8e091a91201e => BundleFileType::Ivf, - 0xa62f9297dc969e85 => BundleFileType::Keys, - 0x2a690fd348fe9ac5 => BundleFileType::Level, - 0xa14e8dfa2cd117e2 => BundleFileType::Lua, - 0xeac0b497876adedf => BundleFileType::Material, - 0x3fcdd69156a46417 => BundleFileType::Mod, - 0xb277b11fe4a61d37 => BundleFileType::MouseCursor, - 0x169de9566953d264 => BundleFileType::NavData, - 0x3b1fa9e8f6bac374 => BundleFileType::NetworkConfig, - 0xb0f2c12eb107f4d8 => BundleFileType::OddleNet, - 0xad9c6d9ed1e5e77a => BundleFileType::Package, - 0xa8193123526fad64 => BundleFileType::Particles, - 0xbf21403a3ab0bbb1 => BundleFileType::PhysicsProperties, - 0x27862fe24795319c => BundleFileType::RenderConfig, - 0x9ca183c2d0e76dee => BundleFileType::RtPipeline, - 0x9d0a795bfe818d19 => BundleFileType::Scene, - 0xcce8d5b5f5ae333f => BundleFileType::Shader, - 0xe5ee32a477239a93 => BundleFileType::ShaderLibrary, - 0x9e5c3cc74575aeb5 => BundleFileType::ShaderLibraryGroup, - 0x250e0a11ac8e26f8 => BundleFileType::ShadingEnvionmentMapping, - 0xfe73c7dcff8a7ca5 => BundleFileType::ShadingEnvironment, - 0xa27b4d04a9ba6f9e => BundleFileType::Slug, - 0xe9fc9ea7042e5ec0 => BundleFileType::SlugAlbum, - 0xd8b27864a97ffdd7 => BundleFileType::SoundEnvironment, - 0xf97af9983c05b950 => BundleFileType::SpuJob, - 0xa486d4045106165c => BundleFileType::StateMachine, - 0xe3f0baa17d620321 => BundleFileType::StaticPVS, - 0x0d972bab10b40fd3 => BundleFileType::Strings, - 0xad2d3fa30d9ab394 => BundleFileType::SurfaceProperties, - 0xcd4238c6a0c69e32 => BundleFileType::Texture, - 0x99736be1fff739a4 => BundleFileType::TimpaniBank, - 0x00a3e6c59a2b9c6c => BundleFileType::TimpaniMaster, - 0x19c792357c99f49b => BundleFileType::Tome, - 0x712d6e3dd1024c9c => BundleFileType::Ugg, - 0xe0a48d0be9a7453f => BundleFileType::Unit, - 0xa99510c6e86dd3c2 => BundleFileType::Upb, - 0xf7505933166d6755 => BundleFileType::VectorField, - 0x786f65c00a816b19 => BundleFileType::Wav, - 0x535a7bd3e650d799 => BundleFileType::WwiseBank, - 0xaf32095c82f2b070 => BundleFileType::WwiseDep, - 0xaabdd317b58dfc8a => BundleFileType::WwiseEvent, - 0xd50a8b7e1c82b110 => BundleFileType::WwiseMetadata, - 0x504b55235d21440e => BundleFileType::WwiseStream, - 0x76015845a6003765 => BundleFileType::Xml, - - _ => BundleFileType::Unknown(Murmur64::from(hash)), - } - } -} - -impl From for u64 { - fn from(t: BundleFileType) -> u64 { - match t { - BundleFileType::Animation => 0x931e336d7646cc26, - BundleFileType::AnimationCurves => 0xdcfb9e18fff13984, - BundleFileType::Apb => 0x3eed05ba83af5090, - BundleFileType::BakedLighting => 0x7ffdb779b04e4ed1, - BundleFileType::Bik => 0xaa5965f03029fa18, - BundleFileType::BlendSet => 0xe301e8af94e3b5a3, - BundleFileType::Bones => 0x18dead01056b72e9, - BundleFileType::Chroma => 0xb7893adf7567506a, - BundleFileType::CommonPackage => 0xfe9754bd19814a47, - BundleFileType::Config => 0x82645835e6b73232, - BundleFileType::Crypto => 0x69108ded1e3e634b, - BundleFileType::Data => 0x8fd0d44d20650b68, - BundleFileType::Entity => 0x9831ca893b0d087d, - BundleFileType::Flow => 0x92d3ee038eeb610d, - BundleFileType::Font => 0x9efe0a916aae7880, - BundleFileType::Ies => 0x8f7d5a2c0f967655, - BundleFileType::Ini => 0xd526a27da14f1dc5, - BundleFileType::Input => 0x2bbcabe5074ade9e, - BundleFileType::Ivf => 0xfa4a8e091a91201e, - BundleFileType::Keys => 0xa62f9297dc969e85, - BundleFileType::Level => 0x2a690fd348fe9ac5, - BundleFileType::Lua => 0xa14e8dfa2cd117e2, - BundleFileType::Material => 0xeac0b497876adedf, - BundleFileType::Mod => 0x3fcdd69156a46417, - BundleFileType::MouseCursor => 0xb277b11fe4a61d37, - BundleFileType::NavData => 0x169de9566953d264, - BundleFileType::NetworkConfig => 0x3b1fa9e8f6bac374, - BundleFileType::OddleNet => 0xb0f2c12eb107f4d8, - BundleFileType::Package => 0xad9c6d9ed1e5e77a, - BundleFileType::Particles => 0xa8193123526fad64, - BundleFileType::PhysicsProperties => 0xbf21403a3ab0bbb1, - BundleFileType::RenderConfig => 0x27862fe24795319c, - BundleFileType::RtPipeline => 0x9ca183c2d0e76dee, - BundleFileType::Scene => 0x9d0a795bfe818d19, - BundleFileType::Shader => 0xcce8d5b5f5ae333f, - BundleFileType::ShaderLibrary => 0xe5ee32a477239a93, - BundleFileType::ShaderLibraryGroup => 0x9e5c3cc74575aeb5, - BundleFileType::ShadingEnvionmentMapping => 0x250e0a11ac8e26f8, - BundleFileType::ShadingEnvironment => 0xfe73c7dcff8a7ca5, - BundleFileType::Slug => 0xa27b4d04a9ba6f9e, - BundleFileType::SlugAlbum => 0xe9fc9ea7042e5ec0, - BundleFileType::SoundEnvironment => 0xd8b27864a97ffdd7, - BundleFileType::SpuJob => 0xf97af9983c05b950, - BundleFileType::StateMachine => 0xa486d4045106165c, - BundleFileType::StaticPVS => 0xe3f0baa17d620321, - BundleFileType::Strings => 0x0d972bab10b40fd3, - BundleFileType::SurfaceProperties => 0xad2d3fa30d9ab394, - BundleFileType::Texture => 0xcd4238c6a0c69e32, - BundleFileType::TimpaniBank => 0x99736be1fff739a4, - BundleFileType::TimpaniMaster => 0x00a3e6c59a2b9c6c, - BundleFileType::Tome => 0x19c792357c99f49b, - BundleFileType::Ugg => 0x712d6e3dd1024c9c, - BundleFileType::Unit => 0xe0a48d0be9a7453f, - BundleFileType::Upb => 0xa99510c6e86dd3c2, - BundleFileType::VectorField => 0xf7505933166d6755, - BundleFileType::Wav => 0x786f65c00a816b19, - BundleFileType::WwiseBank => 0x535a7bd3e650d799, - BundleFileType::WwiseDep => 0xaf32095c82f2b070, - BundleFileType::WwiseEvent => 0xaabdd317b58dfc8a, - BundleFileType::WwiseMetadata => 0xd50a8b7e1c82b110, - BundleFileType::WwiseStream => 0x504b55235d21440e, - BundleFileType::Xml => 0x76015845a6003765, - - BundleFileType::Unknown(hash) => hash.into(), - } - } -} impl From for Murmur64 { fn from(t: BundleFileType) -> Murmur64 { let hash: u64 = t.into(); From 7b95918000430088139af81b9aea2d540e25f98b Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 22 Sep 2023 15:42:16 +0200 Subject: [PATCH 3/3] Refactor code for file injection I ended up wrapping the raw data in a `BundleFile` twice. I also made '--compile' the default, as it should be much less often that raw data needs to be inserted. Even files that are essentially raw binary blobs, like `.wwise_event`, still have some custom fields that need to be accounted for. --- crates/dtmt/src/cmd/bundle/inject.rs | 327 +++++++++++++++++++++------ lib/sdk/src/bundle/file.rs | 23 +- lib/sdk/src/bundle/mod.rs | 3 +- lib/sdk/src/lib.rs | 2 +- 4 files changed, 277 insertions(+), 78 deletions(-) diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 2b86691..21f4a91 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -1,112 +1,297 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::str::FromStr as _; -use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::{self, Context, Result}; +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::{self, Context, OptionExt, Result}; use color_eyre::Help; -use sdk::Bundle; -use tokio::fs::{self, File}; -use tokio::io::AsyncReadExt; +use path_slash::PathBufExt as _; +use sdk::murmur::IdString64; +use sdk::{Bundle, BundleFile, BundleFileType}; +use tokio::fs; pub(crate) fn command_definition() -> Command { Command::new("inject") - .about("Inject a file into a bundle.") - .arg( - Arg::new("replace") - .help("The name of a file in the bundle whos content should be replaced.") - .short('r') - .long("replace"), - ) + .subcommand_required(true) + .about("Inject a file into a bundle.\n\ + Raw binary data can be used to directly replace the file's variant data blob without affecting the metadata.\n\ + Alternatively, a compiler format may be specified, and a complete bundle file is created.") .arg( Arg::new("output") .help( "The path to write the changed bundle to. \ - If omitted, the input bundle will be overwritten.", + If omitted, the input bundle will be overwritten.\n\ + Remember to add a `.patch_` suffix if you also use '--patch'.", ) .short('o') .long("output") .value_parser(value_parser!(PathBuf)), ) .arg( - Arg::new("bundle") - .help("Path to the bundle to inject the file into.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("patch") + .help("Create a patch bundle. Optionally, a patch NUMBER may be specified as \ + '--patch=123'.\nThe maximum number is 999, the default is 1.\n\ + If `--output` is not specified, the `.patch_` suffix is added to \ + the given bundle name.") + .short('p') + .long("patch") + .num_args(0..=1) + .require_equals(true) + .default_missing_value("1") + .value_name("NUMBER") + .value_parser(value_parser!(u16)) ) .arg( - Arg::new("file") - .help("Path to the file to inject.") - .required(true) - .value_parser(value_parser!(PathBuf)), + Arg::new("type") + .help("Compile the new file as the given TYPE. If omitted, the file type is \ + is guessed from the file extension.") + .value_name("TYPE") ) + .subcommand( + Command::new("replace") + .about("Replace an existing file in the bundle") + .arg( + Arg::new("variant") + .help("In combination with '--raw', specify the variant index to replace.") + .long("variant") + .default_value("0") + .value_parser(value_parser!(u8)) + ) + .arg( + Arg::new("raw") + .help("Insert the given file as raw binary data.\n\ + Cannot be used with '--patch'.") + .long("raw") + .action(ArgAction::SetTrue) + ) + .arg( + Arg::new("bundle") + .help("Path to the bundle to inject the file into.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ) + .arg( + Arg::new("bundle-file") + .help("The name of a file in the bundle whose content should be replaced.") + .required(true), + ) + .arg( + Arg::new("new-file") + .help("Path to the file to inject.") + .required(true) + .value_parser(value_parser!(PathBuf)), + ), + ) + // .subcommand( + // Command::new("add") + // .about("Add a new file to the bundle") + // .arg( + // Arg::new("new-file") + // .help("Path to the file to inject.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ) + // .arg( + // Arg::new("bundle") + // .help("Path to the bundle to inject the file into.") + // .required(true) + // .value_parser(value_parser!(PathBuf)), + // ), + // ) } -#[tracing::instrument(skip_all)] +#[tracing::instrument] +async fn compile_file( + path: impl AsRef + std::fmt::Debug, + name: impl Into + std::fmt::Debug, + file_type: BundleFileType, +) -> Result { + let path = path.as_ref(); + + let file_data = fs::read(&path) + .await + .wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?; + let _sjson = String::from_utf8(file_data) + .wrap_err_with(|| format!("Invalid UTF8 data in '{}'", path.display()))?; + + let _root = path.parent().ok_or_eyre("File path has no parent")?; + + eyre::bail!( + "Compilation for type '{}' is not implemented, yet", + file_type + ) +} + +#[tracing::instrument( + skip_all, + fields( + bundle_path = tracing::field::Empty, + in_file_path = tracing::field::Empty, + output_path = tracing::field::Empty, + target_name = tracing::field::Empty, + file_type = tracing::field::Empty, + raw = tracing::field::Empty, + ) +)] pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { - let bundle_path = matches + let Some((op, sub_matches)) = matches.subcommand() else { + unreachable!("clap is configured to require a subcommand, and they're all handled above"); + }; + + let bundle_path = sub_matches .get_one::("bundle") .expect("required parameter not found"); - let file_path = matches - .get_one::("file") + let in_file_path = sub_matches + .get_one::("new-file") .expect("required parameter not found"); - tracing::trace!(bundle_path = %bundle_path.display(), file_path = %file_path.display()); + let patch_number = matches + .get_one::("patch") + .map(|num| format!("{:03}", num)); - let mut bundle = { - let binary = fs::read(bundle_path).await?; - let name = Bundle::get_name_from_path(&ctx, bundle_path); - Bundle::from_binary(&ctx, name, binary).wrap_err("Failed to open bundle file")? + let output_path = matches + .get_one::("output") + .cloned() + .unwrap_or_else(|| { + let mut output_path = bundle_path.clone(); + + if let Some(patch_number) = patch_number.as_ref() { + output_path.set_extension(format!("patch_{:03}", patch_number)); + } + + output_path + }); + + let target_name = if op == "replace" { + sub_matches + .get_one::("bundle-file") + .map(|name| match u64::from_str_radix(name, 16) { + Ok(id) => IdString64::from(id), + Err(_) => IdString64::String(name.clone()), + }) + .expect("argument is required") + } else { + let mut path = PathBuf::from(in_file_path); + path.set_extension(""); + IdString64::from(path.to_slash_lossy().to_string()) }; - if let Some(name) = matches.get_one::("replace") { - let mut file = File::open(&file_path) - .await - .wrap_err_with(|| format!("Failed to open '{}'", file_path.display()))?; + let file_type = if let Some(forced_type) = matches.get_one::("type") { + BundleFileType::from_str(forced_type.as_str()).wrap_err("Unknown file type")? + } else { + in_file_path + .extension() + .and_then(|s| s.to_str()) + .ok_or_eyre("File extension missing") + .and_then(BundleFileType::from_str) + .wrap_err("Unknown file type") + .with_suggestion(|| "Use '--type TYPE' to specify the file type")? + }; - if let Some(variant) = bundle - .files_mut() - .filter(|file| file.matches_name(name.clone())) - // TODO: Handle file variants - .find_map(|file| file.variants_mut().next()) - { - let mut data = Vec::new(); - file.read_to_end(&mut data) - .await - .wrap_err("Failed to read input file")?; - variant.set_data(data); - } else { - let err = eyre::eyre!("No file '{}' in this bundle.", name) - .with_suggestion(|| { + { + let span = tracing::Span::current(); + if !span.is_disabled() { + span.record("bundle_path", bundle_path.display().to_string()); + span.record("in_file_path", in_file_path.display().to_string()); + span.record("output_path", output_path.display().to_string()); + span.record("raw", sub_matches.get_flag("raw")); + span.record("target_name", target_name.display().to_string()); + span.record("file_type", format!("{:?}", file_type)); + } + } + + let bundle_name = Bundle::get_name_from_path(&ctx, bundle_path); + let mut bundle = { + let binary = fs::read(bundle_path).await?; + Bundle::from_binary(&ctx, bundle_name.clone(), binary) + .wrap_err_with(|| format!("Failed to open bundle '{}'", bundle_path.display()))? + }; + + if op == "copy" { + unimplemented!("Implement copying a file from one bundle to the other."); + } + + let output_bundle = match op { + "replace" => { + let Some(file) = bundle + .files_mut() + .find(|file| *file.base_name() == target_name) + else { + let err = eyre::eyre!( + "No file with name '{}' in bundle '{}'", + target_name.display(), + bundle_path.display() + ); + + return Err(err).with_suggestion(|| { format!( - "Run '{} bundle list {}' to list the files in this bundle.", + "Run '{} bundle list \"{}\"' to list the files in this bundle.", clap::crate_name!(), bundle_path.display() ) - }) - .with_suggestion(|| { - format!( - "Use '{} bundle inject --add {} {} {}' to add it as a new file", - clap::crate_name!(), - name, - bundle_path.display(), - file_path.display() - ) }); + }; - return Err(err); + if sub_matches.get_flag("raw") { + let variant_index = sub_matches + .get_one::("variant") + .expect("argument with default missing"); + + let Some(variant) = file.variants_mut().nth(*variant_index as usize) else { + let err = eyre::eyre!( + "Variant index '{}' does not exist in '{}'", + variant_index, + target_name.display() + ); + + return Err(err).with_suggestion(|| { + format!( + "See '{} bundle inject add --help' if you want to add it as a new file", + clap::crate_name!(), + ) + }); + }; + + let data = tokio::fs::read(&in_file_path).await.wrap_err_with(|| { + format!("Failed to read file '{}'", in_file_path.display()) + })?; + variant.set_data(data); + file.set_modded(true); + bundle + } else { + let mut bundle_file = compile_file(in_file_path, target_name.clone(), file_type) + .await + .wrap_err("Failed to compile")?; + + bundle_file.set_modded(true); + + if patch_number.is_some() { + let mut output_bundle = Bundle::new(bundle_name); + output_bundle.add_file(bundle_file); + output_bundle + } else { + *file = bundle_file; + + dbg!(&file); + bundle + } + } } + "add" => { + unimplemented!("Implement adding a new file to the bundle."); + } + _ => unreachable!("no other operations exist"), + }; - let out_path = matches.get_one::("output").unwrap_or(bundle_path); - let data = bundle - .to_binary() - .wrap_err("Failed to write changed bundle to output")?; + let data = output_bundle + .to_binary() + .wrap_err("Failed to write changed bundle to output")?; - fs::write(out_path, &data) - .await - .wrap_err("Failed to write data to output file")?; + fs::write(&output_path, &data) + .await + .wrap_err_with(|| format!("Failed to write data to '{}'", output_path.display()))?; - Ok(()) - } else { - eyre::bail!("Currently, only the '--replace' operation is supported."); - } + tracing::info!("Modified bundle written to '{}'", output_path.display()); + + Ok(()) } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index f387409..6d49821 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -20,6 +20,7 @@ struct BundleFileHeader { len_data_file_name: usize, } +#[derive(Clone, Debug)] pub struct BundleFileVariant { property: u32, data: Vec, @@ -109,9 +110,12 @@ bitflags! { #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { const DATA = 0b100; + // A custom flag used by DTMT to signify a file altered by mods. + const MODDED = 1 << 31; } } +#[derive(Clone, Debug)] pub struct BundleFile { file_type: BundleFileType, name: IdString64, @@ -133,6 +137,18 @@ impl BundleFile { self.variants.push(variant) } + pub fn set_variants(&mut self, variants: Vec) { + self.variants = variants; + } + + pub fn set_props(&mut self, props: Properties) { + self.props = props; + } + + pub fn set_modded(&mut self, is_modded: bool) { + self.props.set(Properties::MODDED, is_modded); + } + #[tracing::instrument(name = "File::read", skip(ctx, r))] pub fn from_reader(ctx: &crate::Context, r: &mut R, props: Properties) -> Result where @@ -299,14 +315,13 @@ impl BundleFile { s } - pub fn matches_name(&self, name: impl Into) -> bool { - let name = name.into(); - if self.name == name { + pub fn matches_name(&self, name: &IdString64) -> bool { + if self.name == *name { return true; } if let IdString64::String(name) = name { - self.name(false, None) == name || self.name(true, None) == name + self.name(false, None) == *name || self.name(true, None) == *name } else { false } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 075f4d2..edb71bb 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -7,14 +7,13 @@ use color_eyre::{Help, Report, SectionExt}; use oodle::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE}; use crate::binary::sync::*; -use crate::bundle::file::Properties; use crate::murmur::{HashGroup, IdString64, Murmur64}; pub(crate) mod database; pub(crate) mod file; pub(crate) mod filetype; -pub use file::{BundleFile, BundleFileVariant}; +pub use file::{BundleFile, BundleFileVariant, Properties}; pub use filetype::BundleFileType; #[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index a24b3bd..9b1806b 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -9,5 +9,5 @@ pub mod murmur; pub use binary::{FromBinary, ToBinary}; pub use bundle::database::BundleDatabase; pub use bundle::decompress; -pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant}; +pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant, Properties}; pub use context::{CmdLine, Context};