diff --git a/crates/dtmt/src/cmd/bundle/decompress.rs b/crates/dtmt/src/cmd/bundle/decompress.rs index 62fbe6d..44182e9 100644 --- a/crates/dtmt/src/cmd/bundle/decompress.rs +++ b/crates/dtmt/src/cmd/bundle/decompress.rs @@ -4,8 +4,7 @@ use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::Result; use sdk::decompress; -use tokio::fs::{self, File}; -use tokio::io::BufReader; +use tokio::fs; pub(crate) fn command_definition() -> Command { Command::new("decompress") @@ -40,11 +39,11 @@ where P1: AsRef + std::fmt::Debug, P2: AsRef + std::fmt::Debug, { - let in_file = File::open(bundle).await?; - let out_file = File::create(destination).await?; + let binary = fs::read(bundle).await?; + let data = decompress(ctx, binary)?; + fs::write(destination, &data).await?; - // A `BufWriter` does not help here, as we're mostly just out chunks. - decompress(ctx, BufReader::new(in_file), out_file).await + Ok(()) } #[tracing::instrument(skip_all)] diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index 63fac62..22d8a64 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -1,10 +1,11 @@ use std::path::{Path, PathBuf}; +use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use color_eyre::eyre::{self, Context, Result}; use color_eyre::{Help, Report, SectionExt}; use futures::future::try_join_all; -use futures::{StreamExt, TryFutureExt}; +use futures::StreamExt; use glob::Pattern; use sdk::{Bundle, BundleFile}; use tokio::fs; @@ -174,58 +175,66 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( } } - let mut paths = Box::pin(resolve_bundle_paths(bundles)); + let includes = Arc::new(includes); + let excludes = Arc::new(excludes); + let ctx = Arc::new(ctx); - // TODO: Find a way to do this with `for_each_concurrent`. The first attempt - // just kept head-butting into a "use of moved value" wall. - while let Some(path) = paths.next().await { - let res = Bundle::open(&ctx, &path) - .and_then(|bundle| { - extract_bundle( - &ctx, - bundle, - &dest, - ExtractOptions { - includes: &includes, - excludes: &excludes, - decompile: should_decompile, - flatten: should_flatten, - dry_run: is_dry_run, - }, - ) - }) + resolve_bundle_paths(bundles) + .for_each_concurrent(10, |p| async { + let ctx = ctx.clone(); + let includes = includes.clone(); + let excludes = excludes.clone(); + + let options = ExtractOptions { + includes, + excludes, + decompile: should_decompile, + flatten: should_flatten, + dry_run: is_dry_run, + }; + + async move { + match extract_bundle(ctx, &p, &dest, options).await { + Ok(_) => {} + Err(err) => tracing::error!("{err:#}"), + } + } .await - .wrap_err_with(|| format!("failed to extract from bundle '{}'", path.display())); - - if let Err(err) = res { - tracing::error!("{:#}", err) - } - } + }) + .await; Ok(()) } +#[derive(Clone)] struct ExtractOptions<'a> { decompile: bool, flatten: bool, dry_run: bool, - includes: &'a dyn AsRef<[&'a Pattern]>, - excludes: &'a dyn AsRef<[&'a Pattern]>, + includes: Arc>, + excludes: Arc>, } #[tracing::instrument( - skip(ctx, bundle, options), + skip(ctx, options), fields(decompile = options.decompile, flatten = options.flatten, dry_run = options.dry_run) )] -async fn extract_bundle

( - ctx: &sdk::Context, - bundle: Bundle, - dest: P, +async fn extract_bundle( + ctx: Arc, + path: P1, + dest: P2, options: ExtractOptions<'_>, ) -> Result<()> where - P: AsRef + std::fmt::Debug, + P1: AsRef + std::fmt::Debug, + P2: AsRef + std::fmt::Debug, { + let bundle = { + let data = fs::read(path.as_ref()).await?; + let name = Bundle::get_name_from_path(&ctx, path.as_ref()); + Bundle::from_binary(&ctx, name, data)? + }; + let includes = options.includes.as_ref(); let excludes = options.excludes.as_ref(); let dest = dest.as_ref(); @@ -275,7 +284,7 @@ where for file in files { let name = file.name(options.decompile, None); let data = if options.decompile { - file.decompiled(ctx).await + file.decompiled(&ctx).await } else { file.raw() }; diff --git a/crates/dtmt/src/cmd/bundle/inject.rs b/crates/dtmt/src/cmd/bundle/inject.rs index 4b26252..f8e001c 100644 --- a/crates/dtmt/src/cmd/bundle/inject.rs +++ b/crates/dtmt/src/cmd/bundle/inject.rs @@ -4,7 +4,7 @@ use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::{self, Context, Result}; use color_eyre::Help; use sdk::Bundle; -use tokio::fs::File; +use tokio::fs::{self, File}; use tokio::io::AsyncReadExt; pub(crate) fn command_definition() -> Command { @@ -52,9 +52,11 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { tracing::trace!(bundle_path = %bundle_path.display(), file_path = %file_path.display()); - let mut bundle = Bundle::open(&ctx, bundle_path) - .await - .wrap_err("Failed to open bundle file")?; + let mut bundle = { + let binary = fs::read(bundle_path).await?; + let name = Bundle::get_name_from_path(&ctx, bundle_path); + Bundle::from_binary(&ctx, name, binary).wrap_err("Failed to open bundle file")? + }; if let Some(_name) = matches.get_one::("replace") { let mut file = File::open(&file_path) @@ -95,14 +97,14 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { } let out_path = matches.get_one::("output").unwrap_or(bundle_path); - let mut out_file = File::create(out_path) - .await - .wrap_err_with(|| format!("failed to open output file {}", out_path.display()))?; - bundle - .write(&ctx, &mut out_file) - .await + let data = bundle + .to_binary(&ctx) .wrap_err("failed to write changed bundle to output")?; + fs::write(out_path, &data) + .await + .wrap_err("failed to write data to output file")?; + Ok(()) } else { eyre::bail!("Currently, only the '--replace' operation is supported."); diff --git a/crates/dtmt/src/cmd/bundle/list.rs b/crates/dtmt/src/cmd/bundle/list.rs index 7ffef5c..ec869ba 100644 --- a/crates/dtmt/src/cmd/bundle/list.rs +++ b/crates/dtmt/src/cmd/bundle/list.rs @@ -1,12 +1,12 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; -use color_eyre::eyre::{self, Result}; +use color_eyre::eyre::{self, Context, Result}; use color_eyre::{Help, SectionExt}; use futures::StreamExt; use sdk::Bundle; -use tracing::Instrument; +use tokio::fs; use crate::cmd::util::resolve_bundle_paths; @@ -31,12 +31,23 @@ pub(crate) fn command_definition() -> Command { ) } -#[derive(Copy, Clone)] +#[derive(Copy, Clone, Debug)] enum OutputFormat { Text, } -fn print_bundle_list(bundle: Bundle, fmt: OutputFormat) { +#[tracing::instrument(skip(ctx))] +async fn print_bundle_contents

(ctx: &sdk::Context, path: P, fmt: OutputFormat) -> Result<()> +where + P: AsRef + std::fmt::Debug, +{ + let p = path.as_ref(); + let bundle = { + let binary = fs::read(p).await?; + let name = Bundle::get_name_from_path(ctx, p); + Bundle::from_binary(ctx, name, binary)? + }; + match fmt { OutputFormat::Text => { println!("Bundle: {}", bundle.name()); @@ -60,6 +71,8 @@ fn print_bundle_list(bundle: Bundle, fmt: OutputFormat) { } } } + + Ok(()) } #[tracing::instrument(skip_all)] @@ -81,20 +94,16 @@ pub(crate) async fn run(ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { paths .for_each_concurrent(10, |p| async { - let span = tracing::info_span!("list bundle"); let ctx = ctx.clone(); async move { - let span = tracing::info_span!("open bundle"); - if let Err(err) = Bundle::open(&ctx, &p) - .instrument(span) + if let Err(err) = print_bundle_contents(&ctx, &p, fmt) .await - .map(|bundle| print_bundle_list(bundle, fmt)) + .wrap_err_with(|| format!("failed to list contents of bundle {}", p.display())) { - tracing::error!("Failed to open bundle '{}': {:?}", p.display(), err); + tracing::error!("{err:?}"); } } - .instrument(span) - .await + .await; }) .await; diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 29d1093..4782440 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -1,164 +1,9 @@ -use std::io::SeekFrom; - -use color_eyre::eyre::WrapErr; -use color_eyre::{Help, Result, SectionExt}; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt}; - -// TODO: Add versions for each write and read function that can work without `AsyncSeek` - -macro_rules! make_read { - ($func:ident, $op:ident, $type:ty) => { - pub(crate) async fn $func(r: &mut R) -> Result<$type> - where - R: AsyncRead + AsyncSeek + std::marker::Unpin, - { - let res = r - .$op() - .await - .wrap_err(concat!("failed to read ", stringify!($type))); - - if res.is_ok() { - return res; - } - - let pos = r.stream_position().await; - if pos.is_ok() { - res.with_section(|| { - format!("{pos:#X} ({pos})", pos = pos.unwrap()).header("Position: ") - }) - } else { - res - } - } - }; -} - -macro_rules! make_write { - ($func:ident, $op:ident, $type:ty) => { - pub(crate) async fn $func(w: &mut W, val: $type) -> Result<()> - where - W: AsyncWrite + AsyncSeek + std::marker::Unpin, - { - let res = w - .$op(val) - .await - .wrap_err(concat!("failed to write ", stringify!($type))); - - if res.is_ok() { - return res; - } - - let pos = w.stream_position().await; - if pos.is_ok() { - res.with_section(|| { - format!("{pos:#X} ({pos})", pos = pos.unwrap()).header("Position: ") - }) - } else { - res - } - } - }; -} - -macro_rules! make_skip { - ($func:ident, $read:ident, $type:ty) => { - pub(crate) async fn $func(r: &mut R, cmp: $type) -> Result<()> - where - R: AsyncRead + AsyncSeek + std::marker::Unpin, - { - let val = $read(r).await?; - - if val != cmp { - let pos = r.stream_position().await.unwrap_or(u64::MAX); - tracing::debug!( - pos, - expected = cmp, - actual = val, - "Unexpected value for skipped {}", - stringify!($type) - ); - } - - Ok(()) - } - }; -} - -make_read!(read_u8, read_u8, u8); -make_read!(read_u32, read_u32_le, u32); -make_read!(read_u64, read_u64_le, u64); - -make_write!(write_u8, write_u8, u8); -make_write!(write_u32, write_u32_le, u32); -make_write!(write_u64, write_u64_le, u64); - -make_skip!(skip_u8, read_u8, u8); -make_skip!(skip_u32, read_u32, u32); - -pub(crate) async fn skip_padding(stream: &mut S) -> Result<()> -where - S: AsyncSeek + std::marker::Unpin, -{ - let pos = stream.stream_position().await?; - let padding_size = 16 - (pos % 16); - - if padding_size < 16 && padding_size > 0 { - tracing::trace!(pos, padding_size, "Skipping padding"); - stream.seek(SeekFrom::Current(padding_size as i64)).await?; - } else { - tracing::trace!(pos, padding_size, "No padding to skip"); - } - - Ok(()) -} - -pub(crate) async fn _read_up_to(r: &mut R, buf: &mut Vec) -> Result -where - R: AsyncRead + AsyncSeek + std::marker::Unpin, -{ - let pos = r.stream_position().await?; - - let err = { - match r.read_exact(buf).await { - Ok(_) => return Ok(buf.len()), - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => { - r.seek(SeekFrom::Start(pos)).await?; - match r.read_to_end(buf).await { - Ok(read) => return Ok(read), - Err(err) => err, - } - } - Err(err) => err, - } - }; - - Err(err).with_section(|| format!("{pos:#X} ({pos})").header("Position: ")) -} - -pub(crate) async fn write_padding(w: &mut W) -> Result -where - W: AsyncWrite + AsyncSeek + std::marker::Unpin, -{ - let pos = w.stream_position().await?; - let size = 16 - (pos % 16) as usize; - - tracing::trace!(padding_size = size, "Writing padding"); - - if size > 0 && size < 16 { - let buf = vec![0; size]; - w.write_all(&buf).await?; - Ok(size) - } else { - Ok(0) - } -} - pub mod sync { use std::io::{self, Read, Seek, SeekFrom}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use color_eyre::eyre::WrapErr; - use color_eyre::{Help, Result, SectionExt}; + use color_eyre::{Help, Report, Result, SectionExt}; macro_rules! make_read { ($func:ident, $read:ident, $type:ty) => { @@ -257,9 +102,42 @@ pub mod sync { Ok(()) } + + fn read_string_len(&mut self, len: usize) -> Result { + let mut buf = vec![0; len]; + let res = self + .read_exact(&mut buf) + .map_err(Report::new) + .and_then(|_| { + String::from_utf8(buf).map_err(|err| { + let ascii = String::from_utf8_lossy(err.as_bytes()).to_string(); + let bytes = format!("{:?}", err.as_bytes()); + Report::new(err) + .with_section(move || bytes.header("Bytes:")) + .with_section(move || ascii.header("ASCII:")) + }) + }); + + if res.is_ok() { + return res; + } + + let pos = self.stream_position(); + if pos.is_ok() { + res.with_section(|| { + format!("{pos:#X} ({pos})", pos = pos.unwrap()).header("Position: ") + }) + } else { + res + } + } } pub trait WriteExt: WriteBytesExt + Seek { + fn write_u8(&mut self, val: u8) -> io::Result<()> { + WriteBytesExt::write_u8(self, val) + } + make_write!(write_u32, write_u32_le, u32); make_write!(write_u64, write_u64_le, u64); diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index 10516a7..c7edd90 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -1,11 +1,11 @@ -use std::io::Cursor; +use std::io::{Cursor, Read, Seek, Write}; +use color_eyre::eyre::Context; use color_eyre::{Help, Result, SectionExt}; use futures::future::join_all; use serde::Serialize; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncWrite, AsyncWriteExt}; -use crate::binary::*; +use crate::binary::sync::*; use crate::filetype::*; use crate::murmur::{HashGroup, Murmur64}; @@ -326,16 +326,16 @@ struct BundleFileHeader { } impl BundleFileHeader { - #[tracing::instrument(name = "FileHeader::read", skip_all)] - async fn read(r: &mut R) -> Result + #[tracing::instrument(name = "FileHeader::from_reader", skip_all)] + fn from_reader(r: &mut R) -> Result where - R: AsyncRead + AsyncSeek + std::marker::Unpin, + R: Read + Seek, { - let variant = read_u32(r).await?; - skip_u8(r, 0).await?; - let size = read_u32(r).await? as usize; - skip_u8(r, 1).await?; - let len_data_file_name = read_u32(r).await? as usize; + let variant = r.read_u32()?; + r.skip_u8(0)?; + let size = r.read_u32()? as usize; + r.skip_u8(1)?; + let len_data_file_name = r.read_u32()? as usize; Ok(Self { size, @@ -343,6 +343,20 @@ impl BundleFileHeader { len_data_file_name, }) } + + #[tracing::instrument(name = "FileHeader::to_writer", skip_all)] + fn to_writer(&self, w: &mut W) -> Result<()> + where + W: Write + Seek, + { + w.write_u32(self.variant)?; + w.write_u8(0)?; + w.write_u32(self.size as u32)?; + w.write_u8(1)?; + w.write_u32(self.len_data_file_name as u32)?; + + Ok(()) + } } pub struct BundleFileVariant { @@ -379,41 +393,35 @@ pub struct BundleFile { impl BundleFile { #[tracing::instrument(name = "File::read", skip_all)] - pub async fn read(ctx: &crate::Context, r: &mut R) -> Result + pub fn from_reader(ctx: &crate::Context, r: &mut R) -> Result where - R: AsyncRead + AsyncSeek + std::marker::Unpin, + R: Read + Seek, { - let file_type = BundleFileType::from(read_u64(r).await?); - let hash = Murmur64::from(read_u64(r).await?); + let file_type = BundleFileType::from(r.read_u64()?); + let hash = Murmur64::from(r.read_u64()?); let name = ctx.lookup_hash(hash, HashGroup::Filename); - let header_count = read_u32(r) - .await - .with_section(|| format!("{}.{}", name, file_type.ext_name()).header("File:"))?; - let header_count = header_count as usize; + tracing::trace!(name, ?file_type); + let header_count = r.read_u32()? as usize; let mut headers = Vec::with_capacity(header_count); - skip_u32(r, 0).await?; + r.skip_u32(0)?; for _ in 0..header_count { - let header = BundleFileHeader::read(r) - .await - .with_section(|| format!("{}.{}", name, file_type.ext_name()).header("File:"))?; + let header = BundleFileHeader::from_reader(r)?; headers.push(header); } let mut variants = Vec::with_capacity(header_count); - - for header in headers.into_iter() { + for (i, header) in headers.into_iter().enumerate() { + let _span = tracing::trace_span!("Read file header {}", i, size = header.size); let mut data = vec![0; header.size]; - r.read_exact(&mut data).await?; + r.read_exact(&mut data) + .wrap_err_with(|| format!("failed to read header {i}"))?; - let data_file_name = { - let mut buf = vec![0; header.len_data_file_name]; - r.read_exact(&mut buf).await?; - - String::from_utf8(buf)? - }; + let data_file_name = r + .read_string_len(header.len_data_file_name) + .wrap_err("failed to read data file name")?; let variant = BundleFileVariant { header, @@ -432,38 +440,25 @@ impl BundleFile { }) } - #[tracing::instrument(name = "File::write", skip_all)] - pub async fn write(&self, w: &mut W) -> Result<()> - where - W: AsyncWrite + AsyncSeek + std::marker::Unpin, - { - write_u64(w, *self.file_type.hash()).await?; - write_u64(w, *self.hash).await?; + #[tracing::instrument(name = "File::to_binary", skip_all)] + pub fn to_binary(&self) -> Result> { + let mut w = Cursor::new(Vec::new()); + + w.write_u64(*self.file_type.hash())?; + w.write_u64(*self.hash)?; let header_count = self.variants.len(); - write_u32(w, header_count as u32).await?; - // TODO: Unknown what this is - write_u32(w, 0).await?; + w.write_u8(header_count as u8)?; for variant in self.variants.iter() { - // TODO: Unknown what these are - write_u32(w, variant.header.variant).await?; - // TODO: Unknown what this is - write_u8(w, 0).await?; - write_u32(w, variant.data.len() as u32).await?; - // TODO: Unknown what this is - write_u8(w, 1).await?; - // TODO: The previous size value and this one are somehow connected, - // but so far it is unknown how - write_u32(w, variant.data_file_name.len() as u32).await?; + variant.header.to_writer(&mut w)?; } for variant in self.variants.iter() { - w.write_all(&variant.data).await?; - w.write_all(variant.data_file_name.as_bytes()).await?; + w.write_all(&variant.data)?; } - Ok(()) + Ok(w.into_inner()) } pub fn base_name(&self) -> &String { @@ -558,10 +553,7 @@ impl BundleFile { let res = match file_type { BundleFileType::Lua => lua::decompile(ctx, data).await, - BundleFileType::Package => { - let mut c = Cursor::new(data); - package::decompile(ctx, &mut c).await - } + BundleFileType::Package => package::decompile(ctx, data), _ => { tracing::debug!("Can't decompile, unknown file type"); Ok(vec![UserFile::with_name(data.to_vec(), name.clone())]) diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 7265c96..dc8b594 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -1,22 +1,17 @@ -use std::io::{Cursor, SeekFrom}; +use std::io::{BufReader, Cursor, Read, Seek, SeekFrom, Write}; use std::path::Path; use color_eyre::eyre::{self, Context, Result}; use color_eyre::{Help, Report, SectionExt}; -use tokio::fs; -use tokio::io::{ - AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt, AsyncWrite, AsyncWriteExt, BufReader, -}; -use tracing::Instrument; -use crate::binary::*; +use crate::binary::sync::*; use crate::murmur::{HashGroup, Murmur64}; use crate::oodle::types::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe}; use crate::oodle::CHUNK_SIZE; pub(crate) mod file; -pub use file::BundleFile; +pub use file::{BundleFile, BundleFileType}; #[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] enum BundleFormat { @@ -52,14 +47,14 @@ struct EntryHeader { } impl EntryHeader { - #[tracing::instrument(name = "FileMeta::read", skip_all)] - async fn read(r: &mut R) -> Result + #[tracing::instrument(name = "EntryHeader::from_reader", skip_all)] + fn from_reader(r: &mut R) -> Result where - R: AsyncRead + AsyncSeek + std::marker::Unpin, + R: Read + Seek, { - let extension_hash = read_u64(r).await?; - let name_hash = read_u64(r).await?; - let flags = read_u32(r).await?; + let extension_hash = r.read_u64()?; + let name_hash = r.read_u64()?; + let flags = r.read_u32()?; // NOTE: Known values so far: // - 0x0: seems to be the default @@ -68,9 +63,7 @@ impl EntryHeader { if flags != 0x0 { tracing::debug!( flags, - "Unexpected meta flags for file {:016X}.{:016X}", - name_hash, - extension_hash + "Unexpected meta flags for file {name_hash:016X}.{extension_hash:016X}", ); } @@ -81,15 +74,14 @@ impl EntryHeader { }) } - #[tracing::instrument(name = "FileMeta::write", skip_all)] - async fn write(&self, w: &mut W) -> Result<()> + #[tracing::instrument(name = "EntryHeader::to_writer", skip_all)] + fn to_writer(&self, w: &mut W) -> Result<()> where - W: AsyncWrite + AsyncSeek + std::marker::Unpin, + W: Write + Seek, { - write_u64(w, self.extension_hash).await?; - write_u64(w, self.name_hash).await?; - write_u32(w, self.flags).await?; - + w.write_u64(self.extension_hash)?; + w.write_u64(self.name_hash)?; + w.write_u32(self.flags)?; Ok(()) } } @@ -104,43 +96,33 @@ pub struct Bundle { } impl Bundle { - #[tracing::instrument(name = "Bundle::open", skip(ctx))] - pub async fn open

(ctx: &crate::Context, path: P) -> Result + pub fn get_name_from_path

(ctx: &crate::Context, path: P) -> String where - P: AsRef + std::fmt::Debug, + P: AsRef, { - // We need to know the bundle name, so it's easier to be given the - // file path and open the File internally, than to be given a generic - // `AsyncRead` and the bundle name separately. let path = path.as_ref(); - let bundle_name = if let Some(name) = path.file_name() { - match Murmur64::try_from(name.to_string_lossy().as_ref()) { - Ok(hash) => ctx.lookup_hash(hash, HashGroup::Filename), - Err(err) => { - tracing::debug!("failed to turn bundle name into hash: {}", err); - name.to_string_lossy().to_string() - } - } - } else { - eyre::bail!("Invalid path to bundle file: {}", path.display()); - }; + path.file_name() + .and_then(|name| name.to_str()) + .and_then(|name| Murmur64::try_from(name).ok()) + .map(|hash| ctx.lookup_hash(hash, HashGroup::Filename)) + .unwrap_or_else(|| path.display().to_string()) + } - let f = fs::File::open(path) - .await - .wrap_err_with(|| format!("failed to open bundle file {}", path.display()))?; + #[tracing::instrument(skip(ctx, binary), fields(len_binary = binary.as_ref().len()))] + pub fn from_binary(ctx: &crate::Context, name: String, binary: B) -> Result + where + B: AsRef<[u8]>, + { + let bundle_name = name; + let mut r = BufReader::new(Cursor::new(binary)); - let mut r = BufReader::new(f); - - let format = read_u32(&mut r) - .await - .wrap_err("failed to read from file") - .and_then(BundleFormat::try_from)?; + let format = r.read_u32().and_then(BundleFormat::try_from)?; if !matches!(format, BundleFormat::F7 | BundleFormat::F8) { return Err(eyre::eyre!("Unknown bundle format: {:?}", format)); } - let unknown_1 = read_u32(&mut r).await?; + let unknown_1 = r.read_u32()?; if unknown_1 != 0x3 { tracing::warn!( "Unexpected value for unknown header. Expected {:#08X}, got {:#08X}", @@ -149,80 +131,73 @@ impl Bundle { ); } - let num_entries = read_u32(&mut r).await? as usize; + let num_entries = r.read_u32()? as usize; let mut unknown_header = [0; 256]; - r.read_exact(&mut unknown_header).await?; + r.read_exact(&mut unknown_header)?; let mut meta = Vec::with_capacity(num_entries); for _ in 0..num_entries { - meta.push(EntryHeader::read(&mut r).await?); + meta.push(EntryHeader::from_reader(&mut r)?); } - let num_chunks = read_u32(&mut r).await? as usize; + let num_chunks = r.read_u32()? as usize; tracing::debug!(num_chunks); let mut chunk_sizes = Vec::with_capacity(num_chunks); for _ in 0..num_chunks { - chunk_sizes.push(read_u32(&mut r).await? as usize); + chunk_sizes.push(r.read_u32()? as usize); } - skip_padding(&mut r).await?; + r.skip_padding()?; - let unpacked_size = read_u32(&mut r).await? as usize; + let unpacked_size = r.read_u32()? as usize; // Skip 4 unknown bytes - r.seek(SeekFrom::Current(4)).await?; + r.skip_u32(0)?; let mut decompressed = Vec::with_capacity(unpacked_size); let mut unpacked_size_tracked = unpacked_size; for (chunk_index, chunk_size) in chunk_sizes.into_iter().enumerate() { - let span = tracing::debug_span!("Decompressing chunk", chunk_index, chunk_size); + let _span = tracing::debug_span!("Decompressing chunk", chunk_index, chunk_size); - async { - let inner_chunk_size = read_u32(&mut r).await? as usize; + let inner_chunk_size = r.read_u32()? as usize; - if inner_chunk_size != chunk_size { - eyre::bail!( - "Chunk sizes do not match. Expected {}, got {}", - inner_chunk_size, - chunk_size, - ); - } - - skip_padding(&mut r).await?; - - let mut compressed_buffer = vec![0u8; chunk_size]; - r.read_exact(&mut compressed_buffer).await?; - - if format >= BundleFormat::F8 && chunk_size == CHUNK_SIZE { - decompressed.append(&mut compressed_buffer); - } else { - // TODO: Optimize to not reallocate? - let oodle_lib = ctx.oodle.as_ref().unwrap(); - let mut raw_buffer = oodle_lib - .decompress( - &compressed_buffer, - OodleLZ_FuzzSafe::No, - OodleLZ_CheckCRC::No, - ) - .wrap_err_with(|| format!("failed to decompress chunk {chunk_index}"))?; - - if unpacked_size_tracked < CHUNK_SIZE { - raw_buffer.resize(unpacked_size_tracked, 0); - } else { - unpacked_size_tracked -= CHUNK_SIZE; - } - - tracing::trace!(raw_size = raw_buffer.len()); - - decompressed.append(&mut raw_buffer); - } - Ok(()) + if inner_chunk_size != chunk_size { + eyre::bail!( + "Chunk sizes do not match. Expected {inner_chunk_size}, got {chunk_size}", + ); + } + + r.skip_padding()?; + + let mut compressed_buffer = vec![0u8; chunk_size]; + r.read_exact(&mut compressed_buffer)?; + + if format >= BundleFormat::F8 && chunk_size == CHUNK_SIZE { + decompressed.append(&mut compressed_buffer); + } else { + // TODO: Optimize to not reallocate? + let oodle_lib = ctx.oodle.as_ref().unwrap(); + let mut raw_buffer = oodle_lib + .decompress( + &compressed_buffer, + OodleLZ_FuzzSafe::No, + OodleLZ_CheckCRC::No, + ) + .wrap_err_with(|| format!("failed to decompress chunk {chunk_index}"))?; + + if unpacked_size_tracked < CHUNK_SIZE { + raw_buffer.resize(unpacked_size_tracked, 0); + } else { + unpacked_size_tracked -= CHUNK_SIZE; + } + + tracing::trace!(raw_size = raw_buffer.len()); + + decompressed.append(&mut raw_buffer); } - .instrument(span) - .await?; } if decompressed.len() < unpacked_size { @@ -236,8 +211,8 @@ impl Bundle { let mut r = Cursor::new(decompressed); let mut files = Vec::with_capacity(num_entries); for i in 0..num_entries { - let span = tracing::trace_span!("", file_index = i); - let file = BundleFile::read(ctx, &mut r).instrument(span).await?; + let file = BundleFile::from_reader(ctx, &mut r) + .wrap_err_with(|| format!("failed to read file {i}"))?; files.push(file); } @@ -251,56 +226,47 @@ impl Bundle { }) } - #[tracing::instrument(name = "Bundle::write", skip_all)] - pub async fn write(&self, ctx: &crate::Context, w: &mut W) -> Result<()> - where - W: AsyncWrite + AsyncSeek + std::marker::Unpin, - { - write_u32(w, self.format.into()).await?; - write_u32(w, self.unknown_1).await?; - write_u32(w, self.files.len() as u32).await?; - w.write_all(&self.unknown_header).await?; + #[tracing::instrument(skip_all)] + pub fn to_binary(&self, ctx: &crate::Context) -> Result> { + let mut w = Cursor::new(Vec::new()); + w.write_u32(self.format.into())?; + w.write_u32(self.unknown_1)?; + w.write_u32(self.files.len() as u32)?; + w.write_all(&self.unknown_header)?; for meta in self._headers.iter() { - meta.write(w).await?; + meta.to_writer(&mut w)?; } let unpacked_data = { - let span = tracing::trace_span!("Write bundle files"); - let buf = Vec::new(); - let mut c = Cursor::new(buf); - + let _span = tracing::trace_span!("Write bundle files"); tracing::trace!(num_files = self.files.len()); - async { - for file in self.files.iter() { - file.write(&mut c).await?; - } - - Ok::<(), Report>(()) - } - .instrument(span) - .await?; - - c.into_inner() + self.files + .iter() + .fold(Ok::, Report>(Vec::new()), |data, file| { + let mut data = data?; + data.append(&mut file.to_binary()?); + Ok(data) + })? }; // Ceiling division (or division toward infinity) to calculate // the number of chunks required to fit the unpacked data. let num_chunks = (unpacked_data.len() + CHUNK_SIZE - 1) / CHUNK_SIZE; tracing::trace!(num_chunks); - write_u32(w, num_chunks as u32).await?; + w.write_u32(num_chunks as u32)?; - let chunk_sizes_start = w.stream_position().await?; + let chunk_sizes_start = w.stream_position()?; tracing::trace!(chunk_sizes_start); - w.seek(SeekFrom::Current(num_chunks as i64 * 4)).await?; + w.seek(SeekFrom::Current(num_chunks as i64 * 4))?; - write_padding(w).await?; + w.write_padding()?; tracing::trace!(unpacked_size = unpacked_data.len()); - write_u32(w, unpacked_data.len() as u32).await?; + w.write_u32(unpacked_data.len() as u32)?; // NOTE: Unknown u32 that's always been 0 so far - write_u32(w, 0).await?; + w.write_u32(0)?; let chunks = unpacked_data.chunks(CHUNK_SIZE); @@ -314,18 +280,18 @@ impl Bundle { compressed_chunk_size = compressed.len() ); chunk_sizes.push(compressed.len()); - write_u32(w, compressed.len() as u32).await?; - write_padding(w).await?; - w.write_all(&compressed).await?; + w.write_u32(compressed.len() as u32)?; + w.write_padding()?; + w.write_all(&compressed)?; } - w.seek(SeekFrom::Start(chunk_sizes_start)).await?; + w.seek(SeekFrom::Start(chunk_sizes_start))?; for size in chunk_sizes { - write_u32(w, size as u32).await?; + w.write_u32(size as u32)?; } - Ok(()) + Ok(w.into_inner()) } pub fn name(&self) -> &String { @@ -345,93 +311,84 @@ impl Bundle { /// This is mainly useful for debugging purposes or /// to manullay inspect the raw data. #[tracing::instrument(skip_all)] -pub async fn decompress(ctx: &crate::Context, mut r: R, mut w: W) -> Result<()> +pub fn decompress(ctx: &crate::Context, binary: B) -> Result> where - R: AsyncRead + AsyncSeek + std::marker::Unpin, - W: AsyncWrite + std::marker::Unpin, + B: AsRef<[u8]>, { - let format = read_u32(&mut r).await.and_then(BundleFormat::try_from)?; + let mut r = BufReader::new(Cursor::new(binary.as_ref())); + let mut w = Cursor::new(Vec::new()); + let format = r.read_u32().and_then(BundleFormat::try_from)?; if !matches!(format, BundleFormat::F7 | BundleFormat::F8) { eyre::bail!("Unknown bundle format: {:?}", format); } // Skip unknown 4 bytes - r.seek(SeekFrom::Current(4)).await?; + r.seek(SeekFrom::Current(4))?; - let num_entries = read_u32(&mut r).await? as i64; + let num_entries = r.read_u32()? as i64; tracing::debug!(num_entries); // Skip unknown 256 bytes - r.seek(SeekFrom::Current(256)).await?; + r.seek(SeekFrom::Current(256))?; // Skip file meta - r.seek(SeekFrom::Current(num_entries * 20)).await?; + r.seek(SeekFrom::Current(num_entries * 20))?; - let num_chunks = read_u32(&mut r).await? as usize; + let num_chunks = r.read_u32()? as usize; tracing::debug!(num_chunks); // Skip chunk sizes - r.seek(SeekFrom::Current(num_chunks as i64 * 4)).await?; + r.seek(SeekFrom::Current(num_chunks as i64 * 4))?; - skip_padding(&mut r).await?; + r.skip_padding()?; - let mut unpacked_size = read_u32(&mut r).await? as usize; + let mut unpacked_size = r.read_u32()? as usize; tracing::debug!(unpacked_size); // Skip unknown 4 bytes - r.seek(SeekFrom::Current(4)).await?; + r.seek(SeekFrom::Current(4))?; - let chunks_start = r.stream_position().await?; + let chunks_start = r.stream_position()?; tracing::trace!(chunks_start); // Pipe the header into the output { - let span = tracing::debug_span!("Pipe file header", chunks_start); - async { - r.seek(SeekFrom::Start(0)).await?; + let _span = tracing::debug_span!("Pipe file header", chunks_start); + r.rewind()?; - let mut buf = vec![0; chunks_start as usize]; - r.read_exact(&mut buf).await?; - w.write_all(&buf).await?; + let mut buf = vec![0; chunks_start as usize]; + r.read_exact(&mut buf)?; + w.write_all(&buf)?; - r.seek(SeekFrom::Start(chunks_start)).await - } - .instrument(span) - .await?; + r.seek(SeekFrom::Start(chunks_start))?; } for chunk_index in 0..num_chunks { - let span = tracing::debug_span!("Decompressing chunk", chunk_index); - async { - let chunk_size = read_u32(&mut r).await? as usize; + let _span = tracing::debug_span!("Decompressing chunk", chunk_index); + let chunk_size = r.read_u32()? as usize; - tracing::trace!(chunk_size); + tracing::trace!(chunk_size); - skip_padding(&mut r).await?; + r.skip_padding()?; - let mut compressed_buffer = vec![0u8; chunk_size]; - r.read_exact(&mut compressed_buffer).await?; + let mut compressed_buffer = vec![0u8; chunk_size]; + r.read_exact(&mut compressed_buffer)?; - let oodle_lib = ctx.oodle.as_ref().unwrap(); - // TODO: Optimize to not reallocate? - let mut raw_buffer = oodle_lib.decompress( - &compressed_buffer, - OodleLZ_FuzzSafe::No, - OodleLZ_CheckCRC::No, - )?; + let oodle_lib = ctx.oodle.as_ref().unwrap(); + // TODO: Optimize to not reallocate? + let mut raw_buffer = oodle_lib.decompress( + &compressed_buffer, + OodleLZ_FuzzSafe::No, + OodleLZ_CheckCRC::No, + )?; - if unpacked_size < CHUNK_SIZE { - raw_buffer.resize(unpacked_size, 0); - } else { - unpacked_size -= CHUNK_SIZE; - } - - w.write_all(&raw_buffer).await?; - - Ok::<(), color_eyre::Report>(()) + if unpacked_size < CHUNK_SIZE { + raw_buffer.resize(unpacked_size, 0); + } else { + unpacked_size -= CHUNK_SIZE; } - .instrument(span) - .await?; + + w.write_all(&raw_buffer)?; } - Ok(()) + Ok(w.into_inner()) } diff --git a/lib/sdk/src/filetype/package.rs b/lib/sdk/src/filetype/package.rs index 61ae361..4124068 100644 --- a/lib/sdk/src/filetype/package.rs +++ b/lib/sdk/src/filetype/package.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; +use std::io::Cursor; use std::ops::{Deref, DerefMut}; use color_eyre::eyre::Context; use color_eyre::Result; use serde::Serialize; -use tokio::io::{AsyncRead, AsyncSeek}; -use crate::binary::*; +use crate::binary::sync::ReadExt; use crate::bundle::file::{BundleFileType, UserFile}; use crate::murmur::{HashGroup, Murmur64}; @@ -34,22 +34,23 @@ impl Package { } #[tracing::instrument(skip_all)] -pub async fn decompile(ctx: &crate::Context, data: &mut R) -> Result> +pub fn decompile(ctx: &crate::Context, binary: B) -> Result> where - R: AsyncRead + AsyncSeek + std::marker::Unpin, + B: AsRef<[u8]>, { + let mut r = Cursor::new(binary.as_ref()); // TODO: Figure out what this is - let unknown = read_u32(data).await?; + let unknown = r.read_u32()?; if unknown != 0x2b { tracing::warn!("Unknown u32 header. Expected 0x2b, got: {unknown:#08X} ({unknown})"); } - let file_count = read_u32(data).await? as usize; + let file_count = r.read_u32()? as usize; let mut package = Package::new(); for i in 0..file_count { - let t = BundleFileType::from(read_u64(data).await?); - let hash = Murmur64::from(read_u64(data).await?); + let t = BundleFileType::from(r.read_u64()?); + let hash = Murmur64::from(r.read_u64()?); let name = ctx.lookup_hash(hash, HashGroup::Filename); tracing::trace!(index = i, r"type" = ?t, %hash, name, "Package entry"); diff --git a/lib/sdk/src/murmur/mod.rs b/lib/sdk/src/murmur/mod.rs index df958ee..d2b57e8 100644 --- a/lib/sdk/src/murmur/mod.rs +++ b/lib/sdk/src/murmur/mod.rs @@ -1,7 +1,8 @@ use std::fmt; -use std::num::ParseIntError; use std::ops::Deref; +use color_eyre::eyre::Context; +use color_eyre::Report; use serde::de::Visitor; use serde::{Deserialize, Serialize}; use serde::{Deserializer, Serializer}; @@ -54,10 +55,12 @@ impl From for Murmur64 { } impl TryFrom<&str> for Murmur64 { - type Error = ParseIntError; + type Error = Report; fn try_from(value: &str) -> Result { - u64::from_str_radix(value, 16).map(Self) + u64::from_str_radix(value, 16) + .map(Self) + .wrap_err_with(|| format!("failed to convert value to Murmur64: {value}")) } } @@ -148,10 +151,12 @@ impl From for Murmur32 { } impl TryFrom<&str> for Murmur32 { - type Error = ParseIntError; + type Error = Report; fn try_from(value: &str) -> Result { - u32::from_str_radix(value, 16).map(Self) + u32::from_str_radix(value, 16) + .map(Self) + .wrap_err_with(|| format!("failed to convert value to Murmur32: {value}")) } }