feat: Implement decompilation for package files

Cargo.lock

@@ -67,6 +67,12 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
+
 [[package]]
 name = "bytes"
 version = "1.2.1"
@@ -403,6 +409,12 @@ version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "miniz_oxide"
 version = "0.5.4"
@@ -430,6 +442,27 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
 
+[[package]]
+name = "nom"
+version = "7.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "nom_locate"
+version = "4.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37794436ca3029a3089e0b95d42da1f0b565ad271e4d3bb4bad0c7bb70b10605"
+dependencies = [
+ "bytecount",
+ "memchr",
+ "nom",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -633,8 +666,10 @@ dependencies = [
 
 [[package]]
 name = "serde_sjson"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
+ "nom",
+ "nom_locate",
  "serde",
 ]
 

crates/dtmt/Cargo.toml

@@ -15,7 +15,7 @@ libloading = "0.7.4"
 nanorand = "0.7.0"
 pin-project-lite = "0.2.9"
 serde = { version = "1.0.147", features = ["derive"] }
-serde_sjson = { path = "../../lib/serde_sjson", version = "0.1.0" }
+serde_sjson = { path = "../../lib/serde_sjson", version = "*" }
 tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] }
 tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] }
 tracing = { version = "0.1.37", features = ["async-await"] }

crates/dtmt/src/cmd/bundle/extract.rs

@@ -2,10 +2,8 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
-use color_eyre::{
-    eyre::{self, Context, Result},
-    Help, Report, SectionExt,
-};
+use color_eyre::eyre::{self, Context, Result};
+use color_eyre::{Help, Report, SectionExt};
 use futures::future::try_join_all;
 use glob::Pattern;
 use sdk::Bundle;
@@ -165,21 +163,51 @@ pub(crate) async fn run(ctx: Arc<RwLock<sdk::Context>>, matches: &ArgMatches) ->
     }))
     .await?;
 
-    let files: Vec<_> = bundles
-        .iter()
-        .flat_map(|bundle| bundle.files())
-        .filter(|file| {
-            let name = file.base_name();
+    let files: Vec<_> = {
+        let iter = bundles.iter().flat_map(|bundle| bundle.files());
+
+        // Short-curcit the iteration if there is nothing to filter by
+        if includes.is_empty() && excludes.is_empty() {
+            iter.collect()
+        } else {
+            iter.filter(|file| {
+                let name = file.name(false);
+                let decompiled_name = file.name(true);
 
                 // When there is no `includes`, all files are included
-            let is_included = includes.is_empty() || includes.iter().any(|glob| glob.matches(name));
+                let is_included = includes.is_empty()
+                    || includes
+                        .iter()
+                        .any(|glob| glob.matches(&name) || glob.matches(&decompiled_name));
                 // When there is no `excludes`, no file is excluded
-            let is_excluded =
-                !excludes.is_empty() && excludes.iter().any(|glob| glob.matches(name));
+                let is_excluded = !excludes.is_empty()
+                    && excludes
+                        .iter()
+                        .any(|glob| glob.matches(&name) || glob.matches(&decompiled_name));
 
                 is_included && !is_excluded
             })
+            .collect()
+        }
+    };
+
+    if tracing::enabled!(tracing::Level::DEBUG) {
+        let includes: Vec<_> = includes.iter().map(|pattern| pattern.as_str()).collect();
+        let excludes: Vec<_> = excludes.iter().map(|pattern| pattern.as_str()).collect();
+        let bundle_files: Vec<_> = bundles
+            .iter()
+            .flat_map(|bundle| bundle.files())
+            .map(|file| file.name(false))
             .collect();
+        let filtered: Vec<_> = files.iter().map(|file| file.name(false)).collect();
+        tracing::debug!(
+            ?includes,
+            ?excludes,
+            files = ?bundle_files,
+            ?filtered,
+            "Built file list to extract"
+        );
+    }
 
     let should_decompile = matches.get_flag("decompile");
     let should_flatten = matches.get_flag("flatten");

lib/sdk/Cargo.toml

@@ -13,7 +13,7 @@ libloading = "0.7.4"
 nanorand = "0.7.0"
 pin-project-lite = "0.2.9"
 serde = { version = "1.0.147", features = ["derive"] }
-serde_sjson = { path = "../../lib/serde_sjson", version = "0.1.0" }
+serde_sjson = { path = "../../lib/serde_sjson", version = "*" }
 tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "macros", "tracing", "io-util", "io-std"] }
 tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] }
 tracing = { version = "0.1.37", features = ["async-await"] }

lib/sdk/src/bundle/file.rs

@@ -1,7 +1,9 @@
+use std::io::Cursor;
 use std::sync::Arc;
 
 use color_eyre::{Help, Result, SectionExt};
 use futures::future::join_all;
+use serde::Serialize;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncWrite, AsyncWriteExt};
 use tokio::sync::RwLock;
 
@@ -10,7 +12,7 @@ use crate::context::lookup_hash;
 use crate::filetype::*;
 use crate::murmur::{HashGroup, Murmur64};
 
-#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)]
 pub enum BundleFileType {
     Animation,
     AnimationCurves,
@@ -162,6 +164,16 @@ impl BundleFileType {
     }
 }
 
+impl Serialize for BundleFileType {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let value = self.ext_name();
+        value.serialize(serializer)
+    }
+}
+
 impl From<u64> for BundleFileType {
     fn from(value: u64) -> Self {
         Self::from(Murmur64::from(value))
@@ -357,7 +369,7 @@ impl BundleFileVariant {
         self.header.size
     }
 
-    pub fn data(&self) -> &Vec<u8> {
+    pub fn data(&self) -> &[u8] {
         &self.data
     }
 
@@ -492,7 +504,7 @@ impl BundleFile {
             .variants
             .iter()
             .map(|variant| UserFile {
-                data: variant.data().clone(),
+                data: variant.data().to_vec(),
                 name: Some(self.name(false)),
             })
             .collect();
@@ -500,7 +512,7 @@ impl BundleFile {
         Ok(files)
     }
 
-    #[tracing::instrument(skip_all)]
+    #[tracing::instrument(name = "File::decompiled", skip_all)]
     pub async fn decompiled(&self, ctx: Arc<RwLock<crate::Context>>) -> Result<Vec<UserFile>> {
         let file_type = self.file_type();
 
@@ -516,14 +528,17 @@ impl BundleFile {
             let ctx = ctx.clone();
 
             async move {
+                let data = variant.data();
+
                 let res = match file_type {
-                    BundleFileType::Lua => lua::decompile(ctx, variant.data()).await,
+                    BundleFileType::Lua => lua::decompile(ctx, data).await,
+                    BundleFileType::Package => {
+                        let mut c = Cursor::new(data);
+                        package::decompile(ctx, &mut c).await
+                    }
                     _ => {
                         tracing::debug!("Can't decompile, unknown file type");
-                        Ok(vec![UserFile::with_name(
-                            variant.data.clone(),
-                            self.name(true),
-                        )])
+                        Ok(vec![UserFile::with_name(data.to_vec(), self.name(true))])
                     }
                 };
 
@@ -533,7 +548,7 @@ impl BundleFile {
                         let err = err
                             .wrap_err("failed to decompile file")
                             .with_section(|| self.name(true).header("File:"));
-                        tracing::error!("{}", err);
+                        tracing::error!("{:?}", err);
                         vec![]
                     }
                 }

lib/sdk/src/bundle/mod.rs

@@ -18,7 +18,7 @@ use crate::oodle::CHUNK_SIZE;
 
 pub(crate) mod file;
 
-use file::BundleFile;
+pub use file::BundleFile;
 
 #[derive(Clone, Copy, Debug, PartialEq)]
 enum BundleFormat {

lib/sdk/src/filetype/mod.rs

@@ -1 +1,2 @@
 pub mod lua;
+pub mod package;

lib/sdk/src/filetype/package.rs

@@ -0,0 +1,65 @@
+use std::collections::HashMap;
+use std::ops::{Deref, DerefMut};
+use std::sync::Arc;
+
+use color_eyre::eyre::Context;
+use color_eyre::Result;
+use serde::Serialize;
+use tokio::io::{AsyncRead, AsyncSeek};
+use tokio::sync::RwLock;
+
+use crate::binary::*;
+use crate::bundle::file::{BundleFileType, UserFile};
+use crate::context::lookup_hash;
+use crate::murmur::{HashGroup, Murmur64};
+
+#[derive(Serialize)]
+struct Package(HashMap<BundleFileType, Vec<String>>);
+
+impl Deref for Package {
+    type Target = HashMap<BundleFileType, Vec<String>>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl DerefMut for Package {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
+impl Package {
+    fn new() -> Self {
+        Self(HashMap::new())
+    }
+}
+
+#[tracing::instrument(skip_all)]
+pub async fn decompile<R>(ctx: Arc<RwLock<crate::Context>>, data: &mut R) -> Result<Vec<UserFile>>
+where
+    R: AsyncRead + AsyncSeek + std::marker::Unpin,
+{
+    // TODO: Figure out what this is
+    let unknown = read_u32(data).await?;
+    if unknown != 0x2b {
+        tracing::warn!("Unknown u32 header. Expected 0x2b, got: {unknown:#08X} ({unknown})");
+    }
+
+    let file_count = read_u32(data).await? as usize;
+    let mut package = Package::new();
+
+    for i in 0..file_count {
+        let t = BundleFileType::from(read_u64(data).await?);
+        let hash = Murmur64::from(read_u64(data).await?);
+        let name = lookup_hash(ctx.clone(), hash, HashGroup::Filename).await;
+
+        tracing::trace!(index = i, r"type" = ?t, %hash, name, "Package entry");
+
+        package.entry(t).or_insert_with(Vec::new).push(name);
+    }
+
+    let s = serde_sjson::to_string(&package.0).wrap_err("failed to serialize Package to SJSON")?;
+    Ok(vec![UserFile::new(s.into_bytes())])
+}

lib/sdk/src/lib.rs

@@ -8,6 +8,6 @@ pub mod murmur;
 mod oodle;
 
 pub use bundle::decompress;
-pub use bundle::Bundle;
+pub use bundle::{Bundle, BundleFile};
 pub use context::Context;
 pub use oodle::Oodle;

lib/sdk/src/murmur/mod.rs

@@ -27,10 +27,10 @@ fn _swap_bytes_u64(value: u64) -> u64 {
     u64::from_le_bytes(value.to_be_bytes())
 }
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
 pub struct Murmur64(u64);
 
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
 pub struct Murmur32(u32);
 
 impl Deref for Murmur64 {

lib/serde_sjson

@@ -1 +1 @@
-Subproject commit b53949d5df34f3407e09cdc21da426e64b976832
+Subproject commit e037ef76591387f27d3a7aa31536b3d97c4c9efe