feat(sdk): Implement bundle database handling

lib/sdk/src/binary.rs

@@ -1,3 +1,47 @@
+use std::io::{Cursor, Read, Seek, Write};
+
+use color_eyre::Result;
+
+use self::sync::{ReadExt, WriteExt};
+
+pub trait FromBinary: Sized {
+    fn from_binary<R: Read + Seek>(r: &mut R) -> Result<Self>;
+}
+
+pub trait ToBinary {
+    fn to_binary(&self) -> Result<Vec<u8>>;
+}
+
+impl<T: ToBinary> ToBinary for Vec<T> {
+    fn to_binary(&self) -> Result<Vec<u8>> {
+        // TODO: Allocations for the vector could be optimized by first
+        // serializing one value, then calculating the size from that.
+        let mut bin = Cursor::new(Vec::new());
+        bin.write_u32(self.len() as u32)?;
+
+        for val in self.iter() {
+            let buf = val.to_binary()?;
+            bin.write_all(&buf)?;
+        }
+
+        Ok(bin.into_inner())
+    }
+}
+
+impl<T: FromBinary> FromBinary for Vec<T> {
+    fn from_binary<R: Read + Seek>(r: &mut R) -> Result<Self> {
+        let size = r.read_u32()? as usize;
+
+        let mut list = Vec::with_capacity(size);
+
+        for _ in 0..size {
+            list.push(T::from_binary(r)?);
+        }
+
+        Ok(list)
+    }
+}
+
 pub mod sync {
     use std::io::{self, Read, Seek, SeekFrom};
 

lib/sdk/src/bundle/database.rs

@@ -0,0 +1,228 @@
+use std::collections::HashMap;
+use std::io::Cursor;
+use std::io::Read;
+use std::io::Seek;
+use std::io::Write;
+
+use color_eyre::eyre;
+use color_eyre::Result;
+
+use crate::binary::sync::*;
+use crate::binary::FromBinary;
+use crate::binary::ToBinary;
+use crate::murmur::Murmur64;
+use crate::Bundle;
+
+use super::file::BundleFileType;
+
+const DATABASE_VERSION: u32 = 0x6;
+const FILE_VERSION: u32 = 0x4;
+
+pub struct BundleFile {
+    name: String,
+    stream: String,
+    platform_specific: bool,
+    file_time: u64,
+}
+
+pub struct FileName {
+    extension: BundleFileType,
+    name: Murmur64,
+}
+
+pub struct BundleDatabase {
+    stored_files: HashMap<Murmur64, Vec<BundleFile>>,
+    resource_hashes: HashMap<Murmur64, u64>,
+    bundle_contents: HashMap<Murmur64, Vec<FileName>>,
+}
+
+impl BundleDatabase {
+    pub fn add_bundle(&mut self, bundle: &Bundle) {
+        let hash = Murmur64::hash(bundle.name().as_bytes());
+        let name = hash.to_string();
+        let stream = format!("{}.stream", &name);
+        let file = BundleFile {
+            name,
+            stream,
+            file_time: 0,
+            platform_specific: false,
+        };
+
+        self.stored_files.entry(hash).or_default().push(file);
+
+        // TODO: Resource hashes
+
+        for f in bundle.files() {
+            let file_name = FileName {
+                extension: f.file_type(),
+                name: Murmur64::hash(f.name(false, None).as_bytes()),
+            };
+
+            self.bundle_contents
+                .entry(hash)
+                .or_default()
+                .push(file_name);
+        }
+    }
+}
+
+impl FromBinary for BundleDatabase {
+    #[tracing::instrument(name = "BundleDatabase::from_binary", skip_all)]
+    fn from_binary<R: Read + Seek>(r: &mut R) -> Result<Self> {
+        {
+            let format = r.read_u32()?;
+            eyre::ensure!(
+                format == DATABASE_VERSION,
+                "invalid file format, expected {:#X}, got {:#X}",
+                DATABASE_VERSION,
+                format
+            );
+        }
+
+        let num_entries = r.read_u32()? as usize;
+        let mut stored_files = HashMap::with_capacity(num_entries);
+
+        for _ in 0..num_entries {
+            let hash = Murmur64::from(r.read_u64()?);
+
+            let num_files = r.read_u32()? as usize;
+            let mut files = Vec::with_capacity(num_files);
+
+            for _ in 0..num_files {
+                {
+                    let version = r.read_u32()?;
+                    eyre::ensure!(
+                        version == FILE_VERSION,
+                        "invalid file version, expected {:#X}, got {:#X}",
+                        FILE_VERSION,
+                        version
+                    );
+                }
+
+                let len_name = r.read_u32()? as usize;
+                let mut buf = vec![0; len_name];
+                r.read_exact(&mut buf)?;
+
+                let name = String::from_utf8(buf)?;
+
+                let len_stream = r.read_u32()? as usize;
+                let mut buf = vec![0; len_stream];
+                r.read_exact(&mut buf)?;
+
+                let stream = String::from_utf8(buf)?;
+
+                let platform_specific = r.read_u8()? != 0;
+
+                // TODO: Unknown what this is. In VT2's SDK, it's simply ignored,
+                // and always written as `0`, but in DT, it seems to be used.
+                let mut buffer = [0; 20];
+                r.read_exact(&mut buffer)?;
+
+                let file_time = r.read_u64()?;
+
+                let file = BundleFile {
+                    name,
+                    stream,
+                    platform_specific,
+                    file_time,
+                };
+
+                files.push(file);
+            }
+
+            stored_files.insert(hash, files);
+        }
+
+        let num_hashes = r.read_u32()? as usize;
+        let mut resource_hashes = HashMap::with_capacity(num_hashes);
+
+        for _ in 0..num_hashes {
+            let name = Murmur64::from(r.read_u64()?);
+            let hash = r.read_u64()?;
+
+            resource_hashes.insert(name, hash);
+        }
+
+        let num_contents = r.read_u32()? as usize;
+        let mut bundle_contents = HashMap::with_capacity(num_contents);
+
+        for _ in 0..num_contents {
+            let hash = Murmur64::from(r.read_u64()?);
+
+            let num_files = r.read_u32()? as usize;
+            let mut files = Vec::with_capacity(num_files);
+
+            for _ in 0..num_files {
+                let extension = BundleFileType::from(r.read_u64()?);
+                let name = Murmur64::from(r.read_u64()?);
+
+                files.push(FileName { extension, name });
+            }
+
+            bundle_contents.insert(hash, files);
+        }
+
+        Ok(Self {
+            stored_files,
+            resource_hashes,
+            bundle_contents,
+        })
+    }
+}
+
+impl ToBinary for BundleDatabase {
+    #[tracing::instrument(name = "BundleDatabase::to_binary", skip_all)]
+    fn to_binary(&self) -> Result<Vec<u8>> {
+        let mut binary = Vec::new();
+
+        {
+            let mut w = Cursor::new(&mut binary);
+
+            w.write_u32(DATABASE_VERSION)?;
+
+            w.write_u32(self.stored_files.len() as u32)?;
+
+            for (hash, files) in self.stored_files.iter() {
+                w.write_u64((*hash).into())?;
+                w.write_u32(files.len() as u32)?;
+
+                for f in files.iter() {
+                    w.write_u32(FILE_VERSION)?;
+                    w.write_u32(f.name.len() as u32)?;
+                    w.write_all(f.name.as_bytes())?;
+                    w.write_u32(f.stream.len() as u32)?;
+                    w.write_all(f.stream.as_bytes())?;
+
+                    w.write_u8(if f.platform_specific { 1 } else { 0 })?;
+
+                    // TODO: Don't know what goes here
+                    let buffer = [0; 20];
+                    w.write_all(&buffer)?;
+
+                    w.write_u64(f.file_time)?;
+                }
+            }
+
+            w.write_u32(self.resource_hashes.len() as u32)?;
+
+            for (name, hash) in self.resource_hashes.iter() {
+                w.write_u64((*name).into())?;
+                w.write_u64(*hash)?;
+            }
+
+            w.write_u32(self.bundle_contents.len() as u32)?;
+
+            for (hash, contents) in self.bundle_contents.iter() {
+                w.write_u64((*hash).into())?;
+                w.write_u32(contents.len() as u32)?;
+
+                for FileName { extension, name } in contents.iter() {
+                    w.write_u64((*extension).into())?;
+                    w.write_u64((*name).into())?;
+                }
+            }
+        }
+
+        Ok(binary)
+    }
+}

lib/sdk/src/bundle/mod.rs

@@ -8,6 +8,7 @@ use oodle_sys::{OodleLZ_CheckCRC, OodleLZ_FuzzSafe, CHUNK_SIZE};
 use crate::binary::sync::*;
 use crate::murmur::{HashGroup, Murmur64};
 
+pub(crate) mod database;
 pub(crate) mod file;
 
 pub use file::{BundleFile, BundleFileType};

lib/sdk/src/lib.rs

@@ -4,6 +4,8 @@ mod context;
 pub mod filetype;
 pub mod murmur;
 
+pub use binary::{FromBinary, ToBinary};
+pub use bundle::database::BundleDatabase;
 pub use bundle::decompress;
 pub use bundle::{Bundle, BundleFile, BundleFileType};
 pub use context::Context;