feat: Implement raw file extraction

src/bin/cmd/bundle/extract.rs

@@ -2,9 +2,14 @@ use std::path::PathBuf;
 use std::sync::Arc;
 
 use clap::{value_parser, Arg, ArgAction, ArgMatches, Command};
-use color_eyre::eyre::Result;
+use color_eyre::{
+    eyre::{self, Context, Result},
+    Help, Report, SectionExt,
+};
+use dtmt::Bundle;
+use futures::future::try_join_all;
 use glob::Pattern;
-use tokio::sync::RwLock;
+use tokio::{fs, sync::RwLock};
 
 fn parse_glob_pattern(s: &str) -> Result<Pattern, String> {
     match Pattern::new(s) {
@@ -13,6 +18,10 @@ fn parse_glob_pattern(s: &str) -> Result<Pattern, String> {
     }
 }
 
+fn flatten_name(s: &str) -> String {
+    s.replace('/', "_")
+}
+
 pub(crate) fn command_definition() -> Command {
     Command::new("extract")
         .about("Extract files from the bundle(s).")
@@ -48,7 +57,7 @@ pub(crate) fn command_definition() -> Command {
                 .value_parser(parse_glob_pattern)
                 .help(
                     "Do not extract files that match the given glob pattern(s).\n\
-                                This takes precedence over `include`.",
+                        This takes precedence over `include`.",
                 ),
         )
         .arg(
@@ -72,25 +81,239 @@ pub(crate) fn command_definition() -> Command {
                 .action(ArgAction::SetTrue)
                 .help(
                     "Attempt to decompile files after extracting them. Not all file types \
-                                are supported for this.",
+                        are supported for this.",
                 ),
         )
-        .arg(Arg::new("ljd").long("ljd").help(
+        .arg(
-            "Path to a custom ljd executable. If not set, \
+            Arg::new("ljd")
-                                `ljd` will be called from PATH.",
+                .long("ljd")
-        ))
+                .help(
-        .arg(Arg::new("revorb").long("revorb").help(
+                    "Path to a custom ljd executable. If not set, \
-            "Path to a custom revorb executable. If not set, \
+                        `ljd` will be called from PATH.",
-                                `revorb` will be called from PATH.",
+                )
-        ))
+                .default_value("ljd"),
-        .arg(Arg::new("ww2ogg").long("ww2ogg").help(
+        )
-            "Path to a custom ww2ogg executable. If not set, \
+        .arg(
-                                `ww2ogg` will be called from PATH.\nSee the documentation for how \
+            Arg::new("revorb")
-                                to set up the script.",
+                .long("revorb")
-        ))
+                .help(
+                    "Path to a custom revorb executable. If not set, \
+                        `revorb` will be called from PATH.",
+                )
+                .default_value("revorb"),
+        )
+        .arg(
+            Arg::new("ww2ogg")
+                .long("ww2ogg")
+                .help(
+                    "Path to a custom ww2ogg executable. If not set, \
+                        `ww2ogg` will be called from PATH.\nSee the documentation for how \
+                        to set up the script for this.",
+                )
+                .default_value("ww2ogg"),
+        )
 }
 
 #[tracing::instrument(skip_all)]
-pub(crate) async fn run(_ctx: Arc<RwLock<dtmt::Context>>, _matches: &ArgMatches) -> Result<()> {
+pub(crate) async fn run(ctx: Arc<RwLock<dtmt::Context>>, matches: &ArgMatches) -> Result<()> {
-    unimplemented!()
+    {
+        let ljd_bin = matches
+            .get_one::<String>("ljd")
+            .expect("no default value for 'ljd' parameter");
+        let revorb_bin = matches
+            .get_one::<String>("revorb")
+            .expect("no default value for 'revorb' parameter");
+        let ww2ogg_bin = matches
+            .get_one::<String>("ww2ogg")
+            .expect("no default value for 'ww2ogg' parameter");
+
+        let mut ctx = ctx.write().await;
+        ctx.ljd = Some(ljd_bin.clone());
+        ctx.revorb = Some(revorb_bin.clone());
+        ctx.ww2ogg = Some(ww2ogg_bin.clone());
+    }
+
+    let includes = match matches.get_many::<Pattern>("include") {
+        Some(values) => values.collect(),
+        None => Vec::new(),
+    };
+
+    let excludes = match matches.get_many::<Pattern>("exclude") {
+        Some(values) => values.collect(),
+        None => Vec::new(),
+    };
+
+    let bundles = matches
+        .get_many::<PathBuf>("bundle")
+        .unwrap_or_default()
+        .cloned();
+
+    let bundles = try_join_all(bundles.into_iter().map(|p| async {
+        let ctx = ctx.clone();
+        let path_display = p.display().to_string();
+        async move { Bundle::open(ctx, &p).await }
+            .await
+            .with_section(|| path_display.header("Bundle Path:"))
+    }))
+    .await?;
+
+    let files: Vec<_> = bundles
+        .iter()
+        .flat_map(|bundle| bundle.files())
+        .filter(|file| {
+            let name = file.base_name();
+
+            // When there is no `includes`, all files are included
+            let is_included = includes.is_empty() || includes.iter().any(|glob| glob.matches(name));
+            // When there is no `excludes`, no file is excluded
+            let is_excluded =
+                !excludes.is_empty() && excludes.iter().any(|glob| glob.matches(name));
+
+            is_included && !is_excluded
+        })
+        .collect();
+
+    let should_decompile = matches.get_flag("decompile");
+    let should_flatten = matches.get_flag("flatten");
+    let is_dry_run = matches.get_flag("dry-run");
+
+    let dest = matches
+        .get_one::<PathBuf>("destination")
+        .expect("required argument 'destination' missing");
+
+    {
+        let res = match fs::metadata(&dest).await {
+            Ok(meta) if !meta.is_dir() => Err(eyre::eyre!("Destination path is not a directory")),
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                Err(eyre::eyre!("Destination path does not exist"))
+                    .with_suggestion(|| "Create the directory")
+            }
+            Err(err) => Err(Report::new(err)),
+            _ => Ok(()),
+        };
+
+        if res.is_err() {
+            return res.wrap_err(format!(
+                "Failed to open destination directory: {}",
+                dest.display()
+            ));
+        }
+    }
+
+    let mut tasks = Vec::with_capacity(files.len());
+
+    for file in files {
+        let name = file.name(should_decompile);
+        let data = if should_decompile {
+            file.decompiled(ctx.clone()).await
+        } else {
+            file.raw()
+        };
+
+        match data {
+            Ok(mut files) => {
+                match files.len() {
+                    0 => {
+                        println!(
+                            "Decompilation did not produce any data for file {}",
+                            file.name(should_decompile)
+                        );
+                    }
+                    // For a single file we want to use the bundle file's name.
+                    1 => {
+                        // We already checked `files.len()`.
+                        let file = files.pop().unwrap();
+
+                        let name = file.name().unwrap_or(&name);
+                        let name = if should_flatten {
+                            flatten_name(name)
+                        } else {
+                            name.clone()
+                        };
+
+                        let mut path = dest.clone();
+                        path.push(name);
+
+                        if is_dry_run {
+                            tracing::info!(path = %path.display(), "Writing file");
+                        } else {
+                            tracing::debug!(path = %path.display(), "Writing file");
+                            tasks.push(tokio::spawn(async move {
+                                fs::write(&path, file.data())
+                                    .await
+                                    .wrap_err("failed to write extracted file to disc")
+                                    .with_section(|| path.display().to_string().header("Path"))
+                            }));
+                        }
+                    }
+                    // For multiple files we create a directory and name files
+                    // by index.
+                    _ => {
+                        for (i, file) in files.into_iter().enumerate() {
+                            let mut path = dest.clone();
+
+                            let name = file
+                                .name()
+                                .map(|name| {
+                                    if should_flatten {
+                                        flatten_name(name)
+                                    } else {
+                                        name.clone()
+                                    }
+                                })
+                                .unwrap_or(format!("{}", i));
+
+                            path.push(name);
+
+                            if is_dry_run {
+                                tracing::info!(path = %path.display(), "Writing file");
+                            } else {
+                                tracing::debug!(path = %path.display(), "Writing file");
+                                tasks.push(tokio::spawn(async move {
+                                    let parent = match path.parent() {
+                                        Some(parent) => parent,
+                                        None => {
+                                            eyre::bail!(
+                                                "Decompilation produced invalid path: {}",
+                                                &path.display()
+                                            )
+                                        }
+                                    };
+
+                                    fs::create_dir_all(parent)
+                                        .await
+                                        .wrap_err("failed to create parent directory")
+                                        .with_section(|| {
+                                            parent.display().to_string().header("Path")
+                                        })?;
+
+                                    fs::write(&path, file.data())
+                                        .await
+                                        .wrap_err("failed to write extracted file to disc")
+                                        .with_section(|| path.display().to_string().header("Path"))
+                                }));
+                            }
+                        }
+                    }
+                }
+            }
+            Err(err) => {
+                let err = err
+                    .wrap_err("Failed to decompile")
+                    .with_section(|| name.header("File"));
+
+                tracing::error!("{:#}", err);
+            }
+        };
+    }
+
+    let results = try_join_all(tasks).await?;
+
+    for res in results {
+        if let Err(err) = res {
+            tracing::error!("{:#}", err);
+        }
+    }
+
+    Ok(())
 }

src/bin/cmd/bundle/list.rs

@@ -61,7 +61,7 @@ pub(crate) async fn run(ctx: Arc<RwLock<dtmt::Context>>, matches: &ArgMatches) -
                 let v = &f.variants()[0];
                 println!(
                     "\t{}.{}: {} bytes",
-                    f.name(),
+                    f.base_name(),
                     f.file_type().ext_name(),
                     v.size()
                 );

src/bundle/file.rs

@@ -2,11 +2,13 @@ use std::ops::Deref;
 use std::sync::Arc;
 
 use color_eyre::{Help, Result, SectionExt};
+use futures::future::join_all;
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek};
 use tokio::sync::RwLock;
 
 use crate::binary::*;
 use crate::context::lookup_hash;
+use crate::filetype::*;
 use crate::murmur::{HashGroup, Murmur64};
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
@@ -407,10 +409,18 @@ impl BundleFile {
         })
     }
 
-    pub fn name(&self) -> &String {
+    pub fn base_name(&self) -> &String {
         &self.name
     }
 
+    pub fn name(&self, decompiled: bool) -> String {
+        if decompiled {
+            format!("{}.{}", self.name, self.file_type.decompiled_ext_name())
+        } else {
+            format!("{}.{}", self.name, self.file_type.ext_name())
+        }
+    }
+
     pub fn hash(&self) -> Murmur64 {
         self.hash
     }
@@ -422,4 +432,89 @@ impl BundleFile {
     pub fn variants(&self) -> &Vec<BundleFileVariant> {
         &self.variants
     }
+
+    pub fn raw(&self) -> Result<Vec<UserFile>> {
+        let files = self
+            .variants
+            .iter()
+            .map(|variant| UserFile {
+                data: variant.data().clone(),
+                name: Some(self.name(false)),
+            })
+            .collect();
+
+        Ok(files)
+    }
+
+    #[tracing::instrument(skip_all)]
+    pub async fn decompiled(&self, ctx: Arc<RwLock<crate::Context>>) -> Result<Vec<UserFile>> {
+        let file_type = self.file_type();
+
+        if tracing::enabled!(tracing::Level::DEBUG) {
+            tracing::debug!(
+                name = self.name(true),
+                variants = self.variants.len(),
+                "Attempting to decompile"
+            );
+        }
+
+        let tasks = self.variants.iter().map(|variant| {
+            let ctx = ctx.clone();
+
+            async move {
+                let res = match file_type {
+                    BundleFileType::Lua => lua::decompile(ctx, variant.data()).await,
+                    _ => {
+                        tracing::debug!("Can't decompile, unknown file type");
+                        Ok(vec![UserFile::with_name(
+                            variant.data.clone(),
+                            self.name(true),
+                        )])
+                    }
+                };
+
+                match res {
+                    Ok(files) => files,
+                    Err(err) => {
+                        let err = err
+                            .wrap_err("failed to decompile file")
+                            .with_section(|| self.name(true).header("File:"));
+                        tracing::error!("{}", err);
+                        vec![]
+                    }
+                }
+            }
+        });
+
+        let results = join_all(tasks).await;
+
+        Ok(results.into_iter().flatten().collect())
+    }
+}
+
+pub struct UserFile {
+    // TODO: Might be able to avoid some allocations with a Cow here
+    data: Vec<u8>,
+    name: Option<String>,
+}
+
+impl UserFile {
+    pub fn new(data: Vec<u8>) -> Self {
+        Self { data, name: None }
+    }
+
+    pub fn with_name(data: Vec<u8>, name: String) -> Self {
+        Self {
+            data,
+            name: Some(name),
+        }
+    }
+
+    pub fn data(&self) -> &[u8] {
+        &self.data
+    }
+
+    pub fn name(&self) -> Option<&String> {
+        self.name.as_ref()
+    }
 }

src/bundle/mod.rs

@@ -14,7 +14,7 @@ use crate::context::lookup_hash;
 use crate::murmur::{HashGroup, Murmur64};
 use crate::oodle;
 
-mod file;
+pub(crate) mod file;
 
 use file::BundleFile;
 

src/context.rs

@@ -7,6 +7,9 @@ use crate::murmur::{Dictionary, HashGroup, Murmur32, Murmur64};
 pub struct Context {
     pub lookup: Dictionary,
     pub oodle: Option<String>,
+    pub ljd: Option<String>,
+    pub revorb: Option<String>,
+    pub ww2ogg: Option<String>,
 }
 
 impl Context {
@@ -14,6 +17,9 @@ impl Context {
         Self {
             lookup: Dictionary::new(),
             oodle: None,
+            ljd: None,
+            revorb: None,
+            ww2ogg: None,
         }
     }
 }

src/filetype/lua.rs

@@ -0,0 +1,19 @@
+use std::io::Cursor;
+use std::sync::Arc;
+
+use color_eyre::Result;
+use tokio::sync::RwLock;
+
+use crate::bundle::file::UserFile;
+
+#[tracing::instrument(skip_all,fields(buf_len = data.as_ref().len()))]
+pub(crate) async fn decompile<T>(
+    _ctx: Arc<RwLock<crate::Context>>,
+    data: T,
+) -> Result<Vec<UserFile>>
+where
+    T: AsRef<[u8]>,
+{
+    let mut _r = Cursor::new(data.as_ref());
+    todo!();
+}

src/filetype/mod.rs

@@ -0,0 +1 @@
+pub mod lua;

src/lib.rs

@@ -1,6 +1,7 @@
 mod binary;
 mod bundle;
 mod context;
+mod filetype;
 pub mod murmur;
 mod oodle;