From 9077d791b24d21a424be6c398b12ffaf6bc308cc Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 17 Feb 2023 22:46:54 +0100 Subject: [PATCH] feat(sdk): Implement file properties --- Cargo.lock | 1 + lib/sdk/Cargo.toml | 1 + lib/sdk/src/bundle/file.rs | 23 +++++++--- lib/sdk/src/bundle/mod.rs | 93 ++++++++++++-------------------------- lib/sdk/src/lib.rs | 2 +- 5 files changed, 49 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23846e0..4dddb00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1934,6 +1934,7 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" name = "sdk" version = "0.2.0" dependencies = [ + "bitflags", "byteorder", "color-eyre", "csv-async", diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 02f3964..e504a76 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -4,6 +4,7 @@ version = "0.2.0" edition = "2021" [dependencies] +bitflags = "1.3.2" byteorder = "1.4.3" color-eyre = "0.6.2" csv-async = { version = "1.2.4", features = ["tokio", "serde"] } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index 2c68185..dd3d1a0 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -1,6 +1,7 @@ use std::io::{Cursor, Read, Seek, Write}; use std::path::Path; +use bitflags::bitflags; use color_eyre::eyre::Context; use color_eyre::{eyre, Result}; use futures::future::join_all; @@ -489,10 +490,18 @@ impl BundleFileVariant { } } +bitflags! { + #[derive(Default)] + pub struct Properties: u32 { + const DATA = 0b100; + } +} + pub struct BundleFile { file_type: BundleFileType, name: String, variants: Vec, + props: Properties, } impl BundleFile { @@ -501,6 +510,7 @@ impl BundleFile { file_type, name, variants: Vec::new(), + props: Properties::empty(), } } @@ -508,12 +518,8 @@ impl BundleFile { self.variants.push(variant) } - #[tracing::instrument( - name = "File::read", - skip_all, - fields(name = %meta.name_hash, ext = %meta.extension_hash, flags = meta.flags) - )] - pub fn from_reader(ctx: &crate::Context, r: &mut R, meta: &EntryHeader) -> Result + #[tracing::instrument(name = "File::read", skip(ctx, r))] + pub fn from_reader(ctx: &crate::Context, r: &mut R, props: Properties) -> Result where R: Read + Seek, { @@ -561,6 +567,7 @@ impl BundleFile { variants, file_type, name, + props, }) } @@ -617,6 +624,10 @@ impl BundleFile { } } + pub fn props(&self) -> Properties { + self.props + } + pub fn base_name(&self) -> &String { &self.name } diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 4cdc88e..b9534e4 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -1,4 +1,5 @@ use std::io::{BufReader, Cursor, Read, Seek, SeekFrom, Write}; +use std::mem::size_of; use std::path::Path; use color_eyre::eyre::{self, Context, Result}; @@ -6,12 +7,13 @@ use color_eyre::{Help, Report, SectionExt}; use oodle_sys::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE}; use crate::binary::sync::*; +use crate::bundle::file::Properties; use crate::murmur::{HashGroup, Murmur64}; pub(crate) mod database; pub(crate) mod file; -pub use file::{BundleFile, BundleFileType}; +pub use file::{BundleFile, BundleFileType, BundleFileVariant}; #[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] enum BundleFormat { @@ -40,56 +42,9 @@ impl From for u32 { } } -pub struct EntryHeader { - name_hash: Murmur64, - extension_hash: Murmur64, - flags: u32, -} - -impl EntryHeader { - #[tracing::instrument(name = "EntryHeader::from_reader", skip_all)] - fn from_reader(r: &mut R) -> Result - where - R: Read + Seek, - { - let extension_hash = Murmur64::from(r.read_u64()?); - let name_hash = Murmur64::from(r.read_u64()?); - let flags = r.read_u32()?; - - // NOTE: Known values so far: - // - 0x0: seems to be the default - // - 0x4: seems to be used for files that point to something in `data/` - // seems to correspond to a change in value in the header's 'unknown_3' - if flags != 0x0 { - tracing::debug!( - flags, - "Unexpected meta flags for file {name_hash:016X}.{extension_hash:016X}", - ); - } - - Ok(Self { - name_hash, - extension_hash, - flags, - }) - } - - #[tracing::instrument(name = "EntryHeader::to_writer", skip_all)] - fn to_writer(&self, w: &mut W) -> Result<()> - where - W: Write + Seek, - { - w.write_u64(self.extension_hash.into())?; - w.write_u64(self.name_hash.into())?; - w.write_u32(self.flags)?; - Ok(()) - } -} - pub struct Bundle { format: BundleFormat, properties: [Murmur64; 32], - headers: Vec, files: Vec, name: String, } @@ -100,7 +55,6 @@ impl Bundle { name, format: BundleFormat::F8, properties: [0.into(); 32], - headers: Vec::new(), files: Vec::new(), } } @@ -119,15 +73,22 @@ impl Bundle { pub fn add_file(&mut self, file: BundleFile) { tracing::trace!("Adding file {}", file.name(false, None)); - let header = EntryHeader { - extension_hash: file.file_type().into(), - name_hash: Murmur64::hash(file.base_name().as_bytes()), - // TODO: Hard coded until we know what this is - flags: 0x0, - }; + let existing_index = self + .files + .iter() + .enumerate() + .find(|(_, f)| **f == file) + .map(|val| val.0); self.files.push(file); - self.headers.push(header); + + if let Some(i) = existing_index { + self.files.swap_remove(i); + } + } + + pub fn get_file>(&self, name: S) -> Option<&BundleFile> { + self.files.iter().find(|f| f.base_name().eq(name.as_ref())) } #[tracing::instrument(skip(ctx, binary), fields(len_binary = binary.as_ref().len()))] @@ -154,9 +115,13 @@ impl Bundle { *prop = Murmur64::from(r.read_u64()?); } - let mut headers = Vec::with_capacity(num_entries); + let mut file_props = Vec::with_capacity(num_entries); for _ in 0..num_entries { - headers.push(EntryHeader::from_reader(&mut r)?); + // Skip two u64 that contain the extension hash and file name hash. + // We don't need them here, since we're reading the whole bundle into memory + // anyways. + r.seek(SeekFrom::Current((2 * size_of::()) as i64))?; + file_props.push(Properties::from_bits_truncate(r.read_u32()?)); } let num_chunks = r.read_u32()? as usize; @@ -227,9 +192,8 @@ impl Bundle { let mut r = Cursor::new(decompressed); let mut files = Vec::with_capacity(num_entries); - for i in 0..num_entries { - let meta = headers.get(i).unwrap(); - let file = BundleFile::from_reader(ctx, &mut r, meta) + for (i, props) in file_props.iter().enumerate() { + let file = BundleFile::from_reader(ctx, &mut r, *props) .wrap_err_with(|| format!("failed to read file {i}"))?; files.push(file); } @@ -237,7 +201,6 @@ impl Bundle { Ok(Self { name: bundle_name, format, - headers, files, properties, }) @@ -255,8 +218,10 @@ impl Bundle { w.write_u64((*prop).into())?; } - for meta in self.headers.iter() { - meta.to_writer(&mut w)?; + for file in self.files.iter() { + w.write_u64(file.file_type().into())?; + w.write_u64(Murmur64::hash(file.base_name().as_bytes()).into())?; + w.write_u32(file.props().bits())?; } let unpacked_data = { diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 01b87ee..e229e28 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -7,5 +7,5 @@ pub mod murmur; pub use binary::{FromBinary, ToBinary}; pub use bundle::database::BundleDatabase; pub use bundle::decompress; -pub use bundle::{Bundle, BundleFile, BundleFileType}; +pub use bundle::{Bundle, BundleFile, BundleFileType, BundleFileVariant}; pub use context::Context;