From 61b3a07666ee07b0efad20e4e1e9340ffd17d6e7 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 23 Jan 2023 16:22:16 +0100 Subject: [PATCH] feat(sdk): Implement bundle database handling --- lib/sdk/src/binary.rs | 44 +++++++ lib/sdk/src/bundle/database.rs | 228 +++++++++++++++++++++++++++++++++ lib/sdk/src/bundle/mod.rs | 1 + lib/sdk/src/lib.rs | 2 + 4 files changed, 275 insertions(+) create mode 100644 lib/sdk/src/bundle/database.rs diff --git a/lib/sdk/src/binary.rs b/lib/sdk/src/binary.rs index 4782440..9ce3f11 100644 --- a/lib/sdk/src/binary.rs +++ b/lib/sdk/src/binary.rs @@ -1,3 +1,47 @@ +use std::io::{Cursor, Read, Seek, Write}; + +use color_eyre::Result; + +use self::sync::{ReadExt, WriteExt}; + +pub trait FromBinary: Sized { + fn from_binary(r: &mut R) -> Result; +} + +pub trait ToBinary { + fn to_binary(&self) -> Result>; +} + +impl ToBinary for Vec { + fn to_binary(&self) -> Result> { + // TODO: Allocations for the vector could be optimized by first + // serializing one value, then calculating the size from that. + let mut bin = Cursor::new(Vec::new()); + bin.write_u32(self.len() as u32)?; + + for val in self.iter() { + let buf = val.to_binary()?; + bin.write_all(&buf)?; + } + + Ok(bin.into_inner()) + } +} + +impl FromBinary for Vec { + fn from_binary(r: &mut R) -> Result { + let size = r.read_u32()? as usize; + + let mut list = Vec::with_capacity(size); + + for _ in 0..size { + list.push(T::from_binary(r)?); + } + + Ok(list) + } +} + pub mod sync { use std::io::{self, Read, Seek, SeekFrom}; diff --git a/lib/sdk/src/bundle/database.rs b/lib/sdk/src/bundle/database.rs new file mode 100644 index 0000000..b3d9296 --- /dev/null +++ b/lib/sdk/src/bundle/database.rs @@ -0,0 +1,228 @@ +use std::collections::HashMap; +use std::io::Cursor; +use std::io::Read; +use std::io::Seek; +use std::io::Write; + +use color_eyre::eyre; +use color_eyre::Result; + +use crate::binary::sync::*; +use crate::binary::FromBinary; +use crate::binary::ToBinary; +use crate::murmur::Murmur64; +use crate::Bundle; + +use super::file::BundleFileType; + +const DATABASE_VERSION: u32 = 0x6; +const FILE_VERSION: u32 = 0x4; + +pub struct BundleFile { + name: String, + stream: String, + platform_specific: bool, + file_time: u64, +} + +pub struct FileName { + extension: BundleFileType, + name: Murmur64, +} + +pub struct BundleDatabase { + stored_files: HashMap>, + resource_hashes: HashMap, + bundle_contents: HashMap>, +} + +impl BundleDatabase { + pub fn add_bundle(&mut self, bundle: &Bundle) { + let hash = Murmur64::hash(bundle.name().as_bytes()); + let name = hash.to_string(); + let stream = format!("{}.stream", &name); + let file = BundleFile { + name, + stream, + file_time: 0, + platform_specific: false, + }; + + self.stored_files.entry(hash).or_default().push(file); + + // TODO: Resource hashes + + for f in bundle.files() { + let file_name = FileName { + extension: f.file_type(), + name: Murmur64::hash(f.name(false, None).as_bytes()), + }; + + self.bundle_contents + .entry(hash) + .or_default() + .push(file_name); + } + } +} + +impl FromBinary for BundleDatabase { + #[tracing::instrument(name = "BundleDatabase::from_binary", skip_all)] + fn from_binary(r: &mut R) -> Result { + { + let format = r.read_u32()?; + eyre::ensure!( + format == DATABASE_VERSION, + "invalid file format, expected {:#X}, got {:#X}", + DATABASE_VERSION, + format + ); + } + + let num_entries = r.read_u32()? as usize; + let mut stored_files = HashMap::with_capacity(num_entries); + + for _ in 0..num_entries { + let hash = Murmur64::from(r.read_u64()?); + + let num_files = r.read_u32()? as usize; + let mut files = Vec::with_capacity(num_files); + + for _ in 0..num_files { + { + let version = r.read_u32()?; + eyre::ensure!( + version == FILE_VERSION, + "invalid file version, expected {:#X}, got {:#X}", + FILE_VERSION, + version + ); + } + + let len_name = r.read_u32()? as usize; + let mut buf = vec![0; len_name]; + r.read_exact(&mut buf)?; + + let name = String::from_utf8(buf)?; + + let len_stream = r.read_u32()? as usize; + let mut buf = vec![0; len_stream]; + r.read_exact(&mut buf)?; + + let stream = String::from_utf8(buf)?; + + let platform_specific = r.read_u8()? != 0; + + // TODO: Unknown what this is. In VT2's SDK, it's simply ignored, + // and always written as `0`, but in DT, it seems to be used. + let mut buffer = [0; 20]; + r.read_exact(&mut buffer)?; + + let file_time = r.read_u64()?; + + let file = BundleFile { + name, + stream, + platform_specific, + file_time, + }; + + files.push(file); + } + + stored_files.insert(hash, files); + } + + let num_hashes = r.read_u32()? as usize; + let mut resource_hashes = HashMap::with_capacity(num_hashes); + + for _ in 0..num_hashes { + let name = Murmur64::from(r.read_u64()?); + let hash = r.read_u64()?; + + resource_hashes.insert(name, hash); + } + + let num_contents = r.read_u32()? as usize; + let mut bundle_contents = HashMap::with_capacity(num_contents); + + for _ in 0..num_contents { + let hash = Murmur64::from(r.read_u64()?); + + let num_files = r.read_u32()? as usize; + let mut files = Vec::with_capacity(num_files); + + for _ in 0..num_files { + let extension = BundleFileType::from(r.read_u64()?); + let name = Murmur64::from(r.read_u64()?); + + files.push(FileName { extension, name }); + } + + bundle_contents.insert(hash, files); + } + + Ok(Self { + stored_files, + resource_hashes, + bundle_contents, + }) + } +} + +impl ToBinary for BundleDatabase { + #[tracing::instrument(name = "BundleDatabase::to_binary", skip_all)] + fn to_binary(&self) -> Result> { + let mut binary = Vec::new(); + + { + let mut w = Cursor::new(&mut binary); + + w.write_u32(DATABASE_VERSION)?; + + w.write_u32(self.stored_files.len() as u32)?; + + for (hash, files) in self.stored_files.iter() { + w.write_u64((*hash).into())?; + w.write_u32(files.len() as u32)?; + + for f in files.iter() { + w.write_u32(FILE_VERSION)?; + w.write_u32(f.name.len() as u32)?; + w.write_all(f.name.as_bytes())?; + w.write_u32(f.stream.len() as u32)?; + w.write_all(f.stream.as_bytes())?; + + w.write_u8(if f.platform_specific { 1 } else { 0 })?; + + // TODO: Don't know what goes here + let buffer = [0; 20]; + w.write_all(&buffer)?; + + w.write_u64(f.file_time)?; + } + } + + w.write_u32(self.resource_hashes.len() as u32)?; + + for (name, hash) in self.resource_hashes.iter() { + w.write_u64((*name).into())?; + w.write_u64(*hash)?; + } + + w.write_u32(self.bundle_contents.len() as u32)?; + + for (hash, contents) in self.bundle_contents.iter() { + w.write_u64((*hash).into())?; + w.write_u32(contents.len() as u32)?; + + for FileName { extension, name } in contents.iter() { + w.write_u64((*extension).into())?; + w.write_u64((*name).into())?; + } + } + } + + Ok(binary) + } +} diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 000df1c..4cdc88e 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -8,6 +8,7 @@ use oodle_sys::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE}; use crate::binary::sync::*; use crate::murmur::{HashGroup, Murmur64}; +pub(crate) mod database; pub(crate) mod file; pub use file::{BundleFile, BundleFileType}; diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 6890317..01b87ee 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -4,6 +4,8 @@ mod context; pub mod filetype; pub mod murmur; +pub use binary::{FromBinary, ToBinary}; +pub use bundle::database::BundleDatabase; pub use bundle::decompress; pub use bundle::{Bundle, BundleFile, BundleFileType}; pub use context::Context;