diff --git a/src/bin/cmd/bundle/extract.rs b/src/bin/cmd/bundle/extract.rs index 83aef46..ea30355 100644 --- a/src/bin/cmd/bundle/extract.rs +++ b/src/bin/cmd/bundle/extract.rs @@ -2,9 +2,14 @@ use std::path::PathBuf; use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; -use color_eyre::eyre::Result; +use color_eyre::{ + eyre::{self, Context, Result}, + Help, Report, SectionExt, +}; +use dtmt::Bundle; +use futures::future::try_join_all; use glob::Pattern; -use tokio::sync::RwLock; +use tokio::{fs, sync::RwLock}; fn parse_glob_pattern(s: &str) -> Result { match Pattern::new(s) { @@ -13,6 +18,10 @@ fn parse_glob_pattern(s: &str) -> Result { } } +fn flatten_name(s: &str) -> String { + s.replace('/', "_") +} + pub(crate) fn command_definition() -> Command { Command::new("extract") .about("Extract files from the bundle(s).") @@ -48,7 +57,7 @@ pub(crate) fn command_definition() -> Command { .value_parser(parse_glob_pattern) .help( "Do not extract files that match the given glob pattern(s).\n\ - This takes precedence over `include`.", + This takes precedence over `include`.", ), ) .arg( @@ -72,25 +81,239 @@ pub(crate) fn command_definition() -> Command { .action(ArgAction::SetTrue) .help( "Attempt to decompile files after extracting them. Not all file types \ - are supported for this.", + are supported for this.", ), ) - .arg(Arg::new("ljd").long("ljd").help( - "Path to a custom ljd executable. If not set, \ - `ljd` will be called from PATH.", - )) - .arg(Arg::new("revorb").long("revorb").help( - "Path to a custom revorb executable. If not set, \ - `revorb` will be called from PATH.", - )) - .arg(Arg::new("ww2ogg").long("ww2ogg").help( - "Path to a custom ww2ogg executable. If not set, \ - `ww2ogg` will be called from PATH.\nSee the documentation for how \ - to set up the script.", - )) + .arg( + Arg::new("ljd") + .long("ljd") + .help( + "Path to a custom ljd executable. If not set, \ + `ljd` will be called from PATH.", + ) + .default_value("ljd"), + ) + .arg( + Arg::new("revorb") + .long("revorb") + .help( + "Path to a custom revorb executable. If not set, \ + `revorb` will be called from PATH.", + ) + .default_value("revorb"), + ) + .arg( + Arg::new("ww2ogg") + .long("ww2ogg") + .help( + "Path to a custom ww2ogg executable. If not set, \ + `ww2ogg` will be called from PATH.\nSee the documentation for how \ + to set up the script for this.", + ) + .default_value("ww2ogg"), + ) } #[tracing::instrument(skip_all)] -pub(crate) async fn run(_ctx: Arc>, _matches: &ArgMatches) -> Result<()> { - unimplemented!() +pub(crate) async fn run(ctx: Arc>, matches: &ArgMatches) -> Result<()> { + { + let ljd_bin = matches + .get_one::("ljd") + .expect("no default value for 'ljd' parameter"); + let revorb_bin = matches + .get_one::("revorb") + .expect("no default value for 'revorb' parameter"); + let ww2ogg_bin = matches + .get_one::("ww2ogg") + .expect("no default value for 'ww2ogg' parameter"); + + let mut ctx = ctx.write().await; + ctx.ljd = Some(ljd_bin.clone()); + ctx.revorb = Some(revorb_bin.clone()); + ctx.ww2ogg = Some(ww2ogg_bin.clone()); + } + + let includes = match matches.get_many::("include") { + Some(values) => values.collect(), + None => Vec::new(), + }; + + let excludes = match matches.get_many::("exclude") { + Some(values) => values.collect(), + None => Vec::new(), + }; + + let bundles = matches + .get_many::("bundle") + .unwrap_or_default() + .cloned(); + + let bundles = try_join_all(bundles.into_iter().map(|p| async { + let ctx = ctx.clone(); + let path_display = p.display().to_string(); + async move { Bundle::open(ctx, &p).await } + .await + .with_section(|| path_display.header("Bundle Path:")) + })) + .await?; + + let files: Vec<_> = bundles + .iter() + .flat_map(|bundle| bundle.files()) + .filter(|file| { + let name = file.base_name(); + + // When there is no `includes`, all files are included + let is_included = includes.is_empty() || includes.iter().any(|glob| glob.matches(name)); + // When there is no `excludes`, no file is excluded + let is_excluded = + !excludes.is_empty() && excludes.iter().any(|glob| glob.matches(name)); + + is_included && !is_excluded + }) + .collect(); + + let should_decompile = matches.get_flag("decompile"); + let should_flatten = matches.get_flag("flatten"); + let is_dry_run = matches.get_flag("dry-run"); + + let dest = matches + .get_one::("destination") + .expect("required argument 'destination' missing"); + + { + let res = match fs::metadata(&dest).await { + Ok(meta) if !meta.is_dir() => Err(eyre::eyre!("Destination path is not a directory")), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Err(eyre::eyre!("Destination path does not exist")) + .with_suggestion(|| "Create the directory") + } + Err(err) => Err(Report::new(err)), + _ => Ok(()), + }; + + if res.is_err() { + return res.wrap_err(format!( + "Failed to open destination directory: {}", + dest.display() + )); + } + } + + let mut tasks = Vec::with_capacity(files.len()); + + for file in files { + let name = file.name(should_decompile); + let data = if should_decompile { + file.decompiled(ctx.clone()).await + } else { + file.raw() + }; + + match data { + Ok(mut files) => { + match files.len() { + 0 => { + println!( + "Decompilation did not produce any data for file {}", + file.name(should_decompile) + ); + } + // For a single file we want to use the bundle file's name. + 1 => { + // We already checked `files.len()`. + let file = files.pop().unwrap(); + + let name = file.name().unwrap_or(&name); + let name = if should_flatten { + flatten_name(name) + } else { + name.clone() + }; + + let mut path = dest.clone(); + path.push(name); + + if is_dry_run { + tracing::info!(path = %path.display(), "Writing file"); + } else { + tracing::debug!(path = %path.display(), "Writing file"); + tasks.push(tokio::spawn(async move { + fs::write(&path, file.data()) + .await + .wrap_err("failed to write extracted file to disc") + .with_section(|| path.display().to_string().header("Path")) + })); + } + } + // For multiple files we create a directory and name files + // by index. + _ => { + for (i, file) in files.into_iter().enumerate() { + let mut path = dest.clone(); + + let name = file + .name() + .map(|name| { + if should_flatten { + flatten_name(name) + } else { + name.clone() + } + }) + .unwrap_or(format!("{}", i)); + + path.push(name); + + if is_dry_run { + tracing::info!(path = %path.display(), "Writing file"); + } else { + tracing::debug!(path = %path.display(), "Writing file"); + tasks.push(tokio::spawn(async move { + let parent = match path.parent() { + Some(parent) => parent, + None => { + eyre::bail!( + "Decompilation produced invalid path: {}", + &path.display() + ) + } + }; + + fs::create_dir_all(parent) + .await + .wrap_err("failed to create parent directory") + .with_section(|| { + parent.display().to_string().header("Path") + })?; + + fs::write(&path, file.data()) + .await + .wrap_err("failed to write extracted file to disc") + .with_section(|| path.display().to_string().header("Path")) + })); + } + } + } + } + } + Err(err) => { + let err = err + .wrap_err("Failed to decompile") + .with_section(|| name.header("File")); + + tracing::error!("{:#}", err); + } + }; + } + + let results = try_join_all(tasks).await?; + + for res in results { + if let Err(err) = res { + tracing::error!("{:#}", err); + } + } + + Ok(()) } diff --git a/src/bin/cmd/bundle/list.rs b/src/bin/cmd/bundle/list.rs index edb04c0..b5dca5d 100644 --- a/src/bin/cmd/bundle/list.rs +++ b/src/bin/cmd/bundle/list.rs @@ -61,7 +61,7 @@ pub(crate) async fn run(ctx: Arc>, matches: &ArgMatches) - let v = &f.variants()[0]; println!( "\t{}.{}: {} bytes", - f.name(), + f.base_name(), f.file_type().ext_name(), v.size() ); diff --git a/src/bundle/file.rs b/src/bundle/file.rs index 7bda17e..eb57b9f 100644 --- a/src/bundle/file.rs +++ b/src/bundle/file.rs @@ -2,11 +2,13 @@ use std::ops::Deref; use std::sync::Arc; use color_eyre::{Help, Result, SectionExt}; +use futures::future::join_all; use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek}; use tokio::sync::RwLock; use crate::binary::*; use crate::context::lookup_hash; +use crate::filetype::*; use crate::murmur::{HashGroup, Murmur64}; #[derive(Debug, PartialEq, Eq, Copy, Clone)] @@ -407,10 +409,18 @@ impl BundleFile { }) } - pub fn name(&self) -> &String { + pub fn base_name(&self) -> &String { &self.name } + pub fn name(&self, decompiled: bool) -> String { + if decompiled { + format!("{}.{}", self.name, self.file_type.decompiled_ext_name()) + } else { + format!("{}.{}", self.name, self.file_type.ext_name()) + } + } + pub fn hash(&self) -> Murmur64 { self.hash } @@ -422,4 +432,89 @@ impl BundleFile { pub fn variants(&self) -> &Vec { &self.variants } + + pub fn raw(&self) -> Result> { + let files = self + .variants + .iter() + .map(|variant| UserFile { + data: variant.data().clone(), + name: Some(self.name(false)), + }) + .collect(); + + Ok(files) + } + + #[tracing::instrument(skip_all)] + pub async fn decompiled(&self, ctx: Arc>) -> Result> { + let file_type = self.file_type(); + + if tracing::enabled!(tracing::Level::DEBUG) { + tracing::debug!( + name = self.name(true), + variants = self.variants.len(), + "Attempting to decompile" + ); + } + + let tasks = self.variants.iter().map(|variant| { + let ctx = ctx.clone(); + + async move { + let res = match file_type { + BundleFileType::Lua => lua::decompile(ctx, variant.data()).await, + _ => { + tracing::debug!("Can't decompile, unknown file type"); + Ok(vec![UserFile::with_name( + variant.data.clone(), + self.name(true), + )]) + } + }; + + match res { + Ok(files) => files, + Err(err) => { + let err = err + .wrap_err("failed to decompile file") + .with_section(|| self.name(true).header("File:")); + tracing::error!("{}", err); + vec![] + } + } + } + }); + + let results = join_all(tasks).await; + + Ok(results.into_iter().flatten().collect()) + } +} + +pub struct UserFile { + // TODO: Might be able to avoid some allocations with a Cow here + data: Vec, + name: Option, +} + +impl UserFile { + pub fn new(data: Vec) -> Self { + Self { data, name: None } + } + + pub fn with_name(data: Vec, name: String) -> Self { + Self { + data, + name: Some(name), + } + } + + pub fn data(&self) -> &[u8] { + &self.data + } + + pub fn name(&self) -> Option<&String> { + self.name.as_ref() + } } diff --git a/src/bundle/mod.rs b/src/bundle/mod.rs index b854970..2fe8e9f 100644 --- a/src/bundle/mod.rs +++ b/src/bundle/mod.rs @@ -14,7 +14,7 @@ use crate::context::lookup_hash; use crate::murmur::{HashGroup, Murmur64}; use crate::oodle; -mod file; +pub(crate) mod file; use file::BundleFile; diff --git a/src/context.rs b/src/context.rs index 0d0ff41..4a9abf1 100644 --- a/src/context.rs +++ b/src/context.rs @@ -7,6 +7,9 @@ use crate::murmur::{Dictionary, HashGroup, Murmur32, Murmur64}; pub struct Context { pub lookup: Dictionary, pub oodle: Option, + pub ljd: Option, + pub revorb: Option, + pub ww2ogg: Option, } impl Context { @@ -14,6 +17,9 @@ impl Context { Self { lookup: Dictionary::new(), oodle: None, + ljd: None, + revorb: None, + ww2ogg: None, } } } diff --git a/src/filetype/lua.rs b/src/filetype/lua.rs new file mode 100644 index 0000000..ea0893e --- /dev/null +++ b/src/filetype/lua.rs @@ -0,0 +1,19 @@ +use std::io::Cursor; +use std::sync::Arc; + +use color_eyre::Result; +use tokio::sync::RwLock; + +use crate::bundle::file::UserFile; + +#[tracing::instrument(skip_all,fields(buf_len = data.as_ref().len()))] +pub(crate) async fn decompile( + _ctx: Arc>, + data: T, +) -> Result> +where + T: AsRef<[u8]>, +{ + let mut _r = Cursor::new(data.as_ref()); + todo!(); +} diff --git a/src/filetype/mod.rs b/src/filetype/mod.rs new file mode 100644 index 0000000..f97296a --- /dev/null +++ b/src/filetype/mod.rs @@ -0,0 +1 @@ +pub mod lua; diff --git a/src/lib.rs b/src/lib.rs index 113859e..233a777 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ mod binary; mod bundle; mod context; +mod filetype; pub mod murmur; mod oodle;