diff --git a/Cargo.lock b/Cargo.lock index edcb074..941419c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,6 +67,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecount" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" + [[package]] name = "bytes" version = "1.2.1" @@ -403,6 +409,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.5.4" @@ -430,6 +442,27 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom_locate" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37794436ca3029a3089e0b95d42da1f0b565ad271e4d3bb4bad0c7bb70b10605" +dependencies = [ + "bytecount", + "memchr", + "nom", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -633,8 +666,10 @@ dependencies = [ [[package]] name = "serde_sjson" -version = "0.1.0" +version = "0.2.0" dependencies = [ + "nom", + "nom_locate", "serde", ] diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index a1add45..73e3d63 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -15,7 +15,7 @@ libloading = "0.7.4" nanorand = "0.7.0" pin-project-lite = "0.2.9" serde = { version = "1.0.147", features = ["derive"] } -serde_sjson = { path = "../../lib/serde_sjson", version = "0.1.0" } +serde_sjson = { path = "../../lib/serde_sjson", version = "*" } tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] } tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] } tracing = { version = "0.1.37", features = ["async-await"] } diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index d9e1dcc..981eae3 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -2,10 +2,8 @@ use std::path::PathBuf; use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; -use color_eyre::{ - eyre::{self, Context, Result}, - Help, Report, SectionExt, -}; +use color_eyre::eyre::{self, Context, Result}; +use color_eyre::{Help, Report, SectionExt}; use futures::future::try_join_all; use glob::Pattern; use sdk::Bundle; @@ -165,21 +163,51 @@ pub(crate) async fn run(ctx: Arc>, matches: &ArgMatches) -> })) .await?; - let files: Vec<_> = bundles - .iter() - .flat_map(|bundle| bundle.files()) - .filter(|file| { - let name = file.base_name(); + let files: Vec<_> = { + let iter = bundles.iter().flat_map(|bundle| bundle.files()); - // When there is no `includes`, all files are included - let is_included = includes.is_empty() || includes.iter().any(|glob| glob.matches(name)); - // When there is no `excludes`, no file is excluded - let is_excluded = - !excludes.is_empty() && excludes.iter().any(|glob| glob.matches(name)); + // Short-curcit the iteration if there is nothing to filter by + if includes.is_empty() && excludes.is_empty() { + iter.collect() + } else { + iter.filter(|file| { + let name = file.name(false); + let decompiled_name = file.name(true); - is_included && !is_excluded - }) - .collect(); + // When there is no `includes`, all files are included + let is_included = includes.is_empty() + || includes + .iter() + .any(|glob| glob.matches(&name) || glob.matches(&decompiled_name)); + // When there is no `excludes`, no file is excluded + let is_excluded = !excludes.is_empty() + && excludes + .iter() + .any(|glob| glob.matches(&name) || glob.matches(&decompiled_name)); + + is_included && !is_excluded + }) + .collect() + } + }; + + if tracing::enabled!(tracing::Level::DEBUG) { + let includes: Vec<_> = includes.iter().map(|pattern| pattern.as_str()).collect(); + let excludes: Vec<_> = excludes.iter().map(|pattern| pattern.as_str()).collect(); + let bundle_files: Vec<_> = bundles + .iter() + .flat_map(|bundle| bundle.files()) + .map(|file| file.name(false)) + .collect(); + let filtered: Vec<_> = files.iter().map(|file| file.name(false)).collect(); + tracing::debug!( + ?includes, + ?excludes, + files = ?bundle_files, + ?filtered, + "Built file list to extract" + ); + } let should_decompile = matches.get_flag("decompile"); let should_flatten = matches.get_flag("flatten"); diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index c0e6b50..82d2552 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -13,7 +13,7 @@ libloading = "0.7.4" nanorand = "0.7.0" pin-project-lite = "0.2.9" serde = { version = "1.0.147", features = ["derive"] } -serde_sjson = { path = "../../lib/serde_sjson", version = "0.1.0" } +serde_sjson = { path = "../../lib/serde_sjson", version = "*" } tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] } tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] } tracing = { version = "0.1.37", features = ["async-await"] } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index e88069c..066fc25 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -1,7 +1,9 @@ +use std::io::Cursor; use std::sync::Arc; use color_eyre::{Help, Result, SectionExt}; use futures::future::join_all; +use serde::Serialize; use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncWrite, AsyncWriteExt}; use tokio::sync::RwLock; @@ -10,7 +12,7 @@ use crate::context::lookup_hash; use crate::filetype::*; use crate::murmur::{HashGroup, Murmur64}; -#[derive(Debug, PartialEq, Eq, Copy, Clone)] +#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)] pub enum BundleFileType { Animation, AnimationCurves, @@ -162,6 +164,16 @@ impl BundleFileType { } } +impl Serialize for BundleFileType { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let value = self.ext_name(); + value.serialize(serializer) + } +} + impl From for BundleFileType { fn from(value: u64) -> Self { Self::from(Murmur64::from(value)) @@ -357,7 +369,7 @@ impl BundleFileVariant { self.header.size } - pub fn data(&self) -> &Vec { + pub fn data(&self) -> &[u8] { &self.data } @@ -492,7 +504,7 @@ impl BundleFile { .variants .iter() .map(|variant| UserFile { - data: variant.data().clone(), + data: variant.data().to_vec(), name: Some(self.name(false)), }) .collect(); @@ -500,7 +512,7 @@ impl BundleFile { Ok(files) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(name = "File::decompiled", skip_all)] pub async fn decompiled(&self, ctx: Arc>) -> Result> { let file_type = self.file_type(); @@ -516,14 +528,17 @@ impl BundleFile { let ctx = ctx.clone(); async move { + let data = variant.data(); + let res = match file_type { - BundleFileType::Lua => lua::decompile(ctx, variant.data()).await, + BundleFileType::Lua => lua::decompile(ctx, data).await, + BundleFileType::Package => { + let mut c = Cursor::new(data); + package::decompile(ctx, &mut c).await + } _ => { tracing::debug!("Can't decompile, unknown file type"); - Ok(vec![UserFile::with_name( - variant.data.clone(), - self.name(true), - )]) + Ok(vec![UserFile::with_name(data.to_vec(), self.name(true))]) } }; @@ -533,7 +548,7 @@ impl BundleFile { let err = err .wrap_err("failed to decompile file") .with_section(|| self.name(true).header("File:")); - tracing::error!("{}", err); + tracing::error!("{:?}", err); vec![] } } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 0a78e77..7303716 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -18,7 +18,7 @@ use crate::oodle::CHUNK_SIZE; pub(crate) mod file; -use file::BundleFile; +pub use file::BundleFile; #[derive(Clone, Copy, Debug, PartialEq)] enum BundleFormat { diff --git a/lib/sdk/src/filetype/mod.rs b/lib/sdk/src/filetype/mod.rs index f97296a..e1a2715 100644 --- a/lib/sdk/src/filetype/mod.rs +++ b/lib/sdk/src/filetype/mod.rs @@ -1 +1,2 @@ pub mod lua; +pub mod package; diff --git a/lib/sdk/src/filetype/package.rs b/lib/sdk/src/filetype/package.rs new file mode 100644 index 0000000..fd0dd83 --- /dev/null +++ b/lib/sdk/src/filetype/package.rs @@ -0,0 +1,65 @@ +use std::collections::HashMap; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; + +use color_eyre::eyre::Context; +use color_eyre::Result; +use serde::Serialize; +use tokio::io::{AsyncRead, AsyncSeek}; +use tokio::sync::RwLock; + +use crate::binary::*; +use crate::bundle::file::{BundleFileType, UserFile}; +use crate::context::lookup_hash; +use crate::murmur::{HashGroup, Murmur64}; + +#[derive(Serialize)] +struct Package(HashMap>); + +impl Deref for Package { + type Target = HashMap>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Package { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Package { + fn new() -> Self { + Self(HashMap::new()) + } +} + +#[tracing::instrument(skip_all)] +pub async fn decompile(ctx: Arc>, data: &mut R) -> Result> +where + R: AsyncRead + AsyncSeek + std::marker::Unpin, +{ + // TODO: Figure out what this is + let unknown = read_u32(data).await?; + if unknown != 0x2b { + tracing::warn!("Unknown u32 header. Expected 0x2b, got: {unknown:#08X} ({unknown})"); + } + + let file_count = read_u32(data).await? as usize; + let mut package = Package::new(); + + for i in 0..file_count { + let t = BundleFileType::from(read_u64(data).await?); + let hash = Murmur64::from(read_u64(data).await?); + let name = lookup_hash(ctx.clone(), hash, HashGroup::Filename).await; + + tracing::trace!(index = i, r"type" = ?t, %hash, name, "Package entry"); + + package.entry(t).or_insert_with(Vec::new).push(name); + } + + let s = serde_sjson::to_string(&package.0).wrap_err("failed to serialize Package to SJSON")?; + Ok(vec![UserFile::new(s.into_bytes())]) +} diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 932f78f..13ce02b 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -8,6 +8,6 @@ pub mod murmur; mod oodle; pub use bundle::decompress; -pub use bundle::Bundle; +pub use bundle::{Bundle, BundleFile}; pub use context::Context; pub use oodle::Oodle; diff --git a/lib/sdk/src/murmur/mod.rs b/lib/sdk/src/murmur/mod.rs index bc9c54b..cc52f01 100644 --- a/lib/sdk/src/murmur/mod.rs +++ b/lib/sdk/src/murmur/mod.rs @@ -27,10 +27,10 @@ fn _swap_bytes_u64(value: u64) -> u64 { u64::from_le_bytes(value.to_be_bytes()) } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] pub struct Murmur64(u64); -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] pub struct Murmur32(u32); impl Deref for Murmur64 { diff --git a/lib/serde_sjson b/lib/serde_sjson index b53949d..e037ef7 160000 --- a/lib/serde_sjson +++ b/lib/serde_sjson @@ -1 +1 @@ -Subproject commit b53949d5df34f3407e09cdc21da426e64b976832 +Subproject commit e037ef76591387f27d3a7aa31536b3d97c4c9efe