dtmt: Implement name overrides

For most of the game files, we don't know the actual name, only the hash
of that name. To still allow building bundles that contain files with
that name (e.g. to override a game file with a custom one), there needs
to be a way to tell DTMT to name a file such that its hash is the same
as the one in the game.
The initial idea was to just expect the file name on disk to be the
hash, but that wouldn't allow for arbitrary folder structures anymore.

So instead, there is now a new, optional setting in `dtmt.cfg`, where
the modder can map a file path to an override name.

crates/dtmm/src/controller/deploy.rs

@@ -324,11 +324,11 @@ async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
 
     let mut bundles = Vec::new();
 
-    let mut add_lua_asset = |name, data: &str| {
+    let mut add_lua_asset = |name: &str, data: &str| {
         let span = tracing::info_span!("Compiling Lua", name, data_len = data.len());
         let _enter = span.enter();
 
-        let file = lua::compile(name, data).wrap_err("Failed to compile Lua")?;
+        let file = lua::compile(name.to_string(), data).wrap_err("Failed to compile Lua")?;
 
         mod_bundle.add_file(file);
 
@@ -517,8 +517,8 @@ async fn patch_boot_bundle(
             .wrap_err("Failed to render template `mod_main.lua`")?;
 
         tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
-        let file =
+        let file = lua::compile(MOD_BOOT_SCRIPT.to_string(), lua)
-            lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
+            .wrap_err("Failed to compile mod main Lua file")?;
 
         boot_bundle.add_file(file);
     }

crates/dtmm/src/controller/import.rs

@@ -297,6 +297,7 @@ fn extract_mod_config<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Result<(Mo
             packages: Vec::new(),
             resources,
             depends: Vec::new(),
+            name_overrides: Default::default(),
         };
 
         Ok((cfg, root))

crates/dtmt/src/cmd/build.rs

@@ -103,38 +103,41 @@ async fn find_project_config(dir: Option<PathBuf>) -> Result<ModConfig> {
 }
 
 #[tracing::instrument(skip_all)]
-async fn compile_package_files<P>(pkg: &Package, root: P) -> Result<Vec<BundleFile>>
+async fn compile_package_files(pkg: &Package, cfg: &ModConfig) -> Result<Vec<BundleFile>> {
-where
+    let root = Arc::new(&cfg.dir);
-    P: AsRef<Path> + std::fmt::Debug,
+    let name_overrides = &cfg.name_overrides;
-{
-    let root = Arc::new(root.as_ref());
 
     let tasks = pkg
         .iter()
-        .flat_map(|(file_type, paths)| {
+        .flat_map(|(file_type, names)| {
-            paths.iter().map(|path| {
+            names.iter().map(|name| {
                 (
                     *file_type,
-                    path,
+                    name,
                     // Cloning the `Arc` here solves the issue that in the next `.map`, I need to
                     // `move` the closure parameters, but can't `move` `root` before it was cloned.
                     root.clone(),
                 )
             })
         })
-        .map(|(file_type, path, root)| async move {
+        .map(|(file_type, name, root)| async move {
-            let sjson = fs::read_to_string(&path).await?;
+            let path = PathBuf::from(name);
+            let sjson = fs::read_to_string(&path)
+                .await
+                .wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?;
 
-            let mut path = path.clone();
+            let name = path.with_extension("").to_slash_lossy().to_string();
-            path.set_extension("");
+            let name = if let Some(new_name) = name_overrides.get(&name) {
-
+                let new_name = match u64::from_str_radix(new_name, 16) {
-            BundleFile::from_sjson(
+                    Ok(hash) => IdString64::from(hash),
-                path.to_slash_lossy().to_string(),
+                    Err(_) => IdString64::from(new_name.clone()),
-                file_type,
+                };
-                sjson,
+                tracing::info!("Overriding '{}' -> '{}'", name, new_name.display());
-                root.as_ref(),
+                new_name
-            )
+            } else {
-            .await
+                IdString64::from(name.clone())
+            };
+            BundleFile::from_sjson(name, file_type, sjson, root.as_ref()).await
         });
 
     let results = futures::stream::iter(tasks)
@@ -146,12 +149,11 @@ where
 }
 
 #[tracing::instrument]
-async fn build_package<P1, P2>(package: P1, root: P2) -> Result<Bundle>
+async fn build_package(
-where
+    cfg: &ModConfig,
-    P1: AsRef<Path> + std::fmt::Debug,
+    package: impl AsRef<Path> + std::fmt::Debug,
-    P2: AsRef<Path> + std::fmt::Debug,
+) -> Result<Bundle> {
-{
+    let root = &cfg.dir;
-    let root = root.as_ref();
     let package = package.as_ref();
 
     let mut path = root.join(package);
@@ -165,7 +167,7 @@ where
         .await
         .wrap_err_with(|| format!("Invalid package file {}", &pkg_name))?;
 
-    let files = compile_package_files(&pkg, root).await?;
+    let files = compile_package_files(&pkg, cfg).await?;
     let mut bundle = Bundle::new(pkg_name);
     for file in files {
         bundle.add_file(file);
@@ -254,14 +256,14 @@ pub(crate) async fn read_project_config(dir: Option<PathBuf>) -> Result<ModConfi
     Ok(cfg)
 }
 
-pub(crate) async fn build<P1, P2>(
+#[tracing::instrument]
+pub(crate) async fn build<P>(
     cfg: &ModConfig,
-    out_path: P1,
+    out_path: impl AsRef<Path> + std::fmt::Debug,
-    game_dir: Arc<Option<P2>>,
+    game_dir: Arc<Option<P>>,
 ) -> Result<()>
 where
-    P1: AsRef<Path>,
+    P: AsRef<Path> + std::fmt::Debug,
-    P2: AsRef<Path>,
 {
     let out_path = out_path.as_ref();
 
@@ -286,7 +288,7 @@ where
                 );
             }
 
-            let bundle = build_package(path, &cfg.dir).await.wrap_err_with(|| {
+            let bundle = build_package(&cfg, path).await.wrap_err_with(|| {
                 format!(
                     "Failed to build package '{}' at '{}'",
                     path.display(),

crates/dtmt/src/cmd/migrate.rs

@@ -351,6 +351,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
         },
         depends: vec![ModDependency::ID(String::from("DMF"))],
         bundled: true,
+        name_overrides: HashMap::new(),
     };
 
     tracing::debug!(?dtmt_cfg);

crates/dtmt/src/cmd/watch.rs

@@ -77,17 +77,14 @@ pub(crate) fn command_definition() -> Command {
         )
 }
 
-async fn compile<P1, P2, P3>(
+#[tracing::instrument]
+async fn compile(
     cfg: &ModConfig,
-    out_path: P1,
+    out_path: impl AsRef<Path> + std::fmt::Debug,
-    archive_path: P2,
+    archive_path: impl AsRef<Path> + std::fmt::Debug,
-    game_dir: Arc<Option<P3>>,
+    game_dir: Arc<Option<impl AsRef<Path> + std::fmt::Debug>>,
-) -> Result<()>
+) -> Result<()> {
-where
+    let out_path = out_path.as_ref();
-    P1: AsRef<Path> + std::marker::Copy,
-    P2: AsRef<Path>,
-    P3: AsRef<Path>,
-{
     build(cfg, out_path, game_dir)
         .await
         .wrap_err("Failed to build bundles")?;

lib/dtmt-shared/src/lib.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::path::PathBuf;
 
 use color_eyre::eyre::{OptionExt as _, WrapErr as _};
@@ -67,6 +68,8 @@ pub struct ModConfig {
     pub depends: Vec<ModDependency>,
     #[serde(default = "default_true", skip_serializing_if = "is_true")]
     pub bundled: bool,
+    #[serde(default)]
+    pub name_overrides: HashMap<String, String>,
 }
 
 pub const STEAMAPP_ID: u32 = 1361210;

lib/sdk/src/bundle/file.rs

@@ -120,7 +120,7 @@ pub struct BundleFile {
 }
 
 impl BundleFile {
-    pub fn new(name: String, file_type: BundleFileType) -> Self {
+    pub fn new(name: impl Into<IdString64>, file_type: BundleFileType) -> Self {
         Self {
             file_type,
             name: name.into(),
@@ -252,20 +252,15 @@ impl BundleFile {
         Ok(w.into_inner())
     }
 
-    #[tracing::instrument(name = "File::from_sjson", skip(sjson))]
+    #[tracing::instrument("File::from_sjson", skip(sjson, name), fields(name = %name.display()))]
-    pub async fn from_sjson<P, S>(
+    pub async fn from_sjson(
-        name: String,
+        name: IdString64,
         file_type: BundleFileType,
-        sjson: S,
+        sjson: impl AsRef<str>,
-        root: P,
+        root: impl AsRef<Path> + std::fmt::Debug,
-    ) -> Result<Self>
+    ) -> Result<Self> {
-    where
-        P: AsRef<Path> + std::fmt::Debug,
-        S: AsRef<str>,
-    {
         match file_type {
-            BundleFileType::Lua => lua::compile(name.clone(), sjson)
+            BundleFileType::Lua => lua::compile(name, sjson).wrap_err("Failed to compile Lua file"),
-                .wrap_err_with(|| format!("Failed to compile Lua file '{}'", name)),
             BundleFileType::Unknown(_) => {
                 eyre::bail!("Unknown file type. Cannot compile from SJSON");
             }
@@ -304,10 +299,7 @@ impl BundleFile {
         s
     }
 
-    pub fn matches_name<S>(&self, name: S) -> bool
+    pub fn matches_name(&self, name: impl Into<IdString64>) -> bool {
-    where
-        S: Into<IdString64>,
-    {
         let name = name.into();
         if self.name == name {
             return true;

lib/sdk/src/filetype/lua.rs

@@ -15,6 +15,7 @@ use tokio::fs;
 use crate::binary::sync::ReadExt;
 use crate::binary::sync::WriteExt;
 use crate::bundle::file::{BundleFileVariant, UserFile};
+use crate::murmur::IdString64;
 use crate::{BundleFile, BundleFileType};
 
 const BITSQUID_LUAJIT_HEADER: u32 = 0x8253461B;
@@ -117,17 +118,13 @@ where
 }
 
 #[tracing::instrument(skip_all)]
-pub fn compile<S, C>(name: S, code: C) -> Result<BundleFile>
+pub fn compile(name: impl Into<IdString64>, code: impl AsRef<str>) -> Result<BundleFile> {
-where
-    S: Into<String>,
-    C: AsRef<str>,
-{
     let name = name.into();
     let code = code.as_ref();
 
     tracing::trace!(
         "Compiling '{}', {} bytes of code",
-        name,
+        name.display(),
         code.as_bytes().len()
     );
 
@@ -135,8 +132,8 @@ where
         let state = lua::luaL_newstate();
         lua::luaL_openlibs(state);
 
-        let name = CString::new(format!("@{name}").into_bytes())
+        let name = CString::new(format!("@{}", name.display()).into_bytes())
-            .wrap_err_with(|| format!("Cannot convert name into CString: {}", name))?;
+            .wrap_err_with(|| format!("Cannot convert name into CString: {}", name.display()))?;
         match lua::luaL_loadbuffer(
             state,
             code.as_ptr() as _,

lib/sdk/src/filetype/package.rs

@@ -7,13 +7,12 @@ use std::str::FromStr;
 use async_recursion::async_recursion;
 use color_eyre::eyre::{self, Context};
 use color_eyre::Result;
-use path_slash::PathBufExt;
 use tokio::fs;
 
 use crate::binary::sync::{ReadExt, WriteExt};
 use crate::bundle::file::UserFile;
 use crate::bundle::filetype::BundleFileType;
-use crate::murmur::{HashGroup, Murmur64};
+use crate::murmur::{HashGroup, IdString64, Murmur64};
 
 #[tracing::instrument]
 #[async_recursion]
@@ -91,12 +90,12 @@ where
     Ok(paths)
 }
 
-type PackageType = HashMap<BundleFileType, HashSet<PathBuf>>;
+type PackageType = HashMap<BundleFileType, HashSet<String>>;
 type PackageDefinition = HashMap<String, HashSet<String>>;
 
 #[derive(Default)]
 pub struct Package {
-    _name: String,
+    _name: IdString64,
     _root: PathBuf,
     inner: PackageType,
     flags: u8,
@@ -117,9 +116,9 @@ impl DerefMut for Package {
 }
 
 impl Package {
-    pub fn new(name: String, root: PathBuf) -> Self {
+    pub fn new(name: impl Into<IdString64>, root: PathBuf) -> Self {
         Self {
-            _name: name,
+            _name: name.into(),
             _root: root,
             inner: Default::default(),
             flags: 1,
@@ -130,17 +129,22 @@ impl Package {
         self.values().fold(0, |total, files| total + files.len())
     }
 
-    pub fn add_file<P: Into<PathBuf>>(&mut self, file_type: BundleFileType, name: P) {
+    pub fn add_file(&mut self, file_type: BundleFileType, name: impl Into<String>) {
         self.inner.entry(file_type).or_default().insert(name.into());
     }
 
     #[tracing::instrument("Package::from_sjson", skip(sjson), fields(sjson_len = sjson.as_ref().len()))]
-    pub async fn from_sjson<P, S>(sjson: S, name: String, root: P) -> Result<Self>
+    pub async fn from_sjson<P, S>(
+        sjson: S,
+        name: impl Into<IdString64> + std::fmt::Debug,
+        root: P,
+    ) -> Result<Self>
     where
         P: AsRef<Path> + std::fmt::Debug,
         S: AsRef<str>,
     {
         let root = root.as_ref();
+        let name = name.into();
         let definition: PackageDefinition = serde_sjson::from_str(sjson.as_ref())?;
         let mut inner: PackageType = Default::default();
 
@@ -174,7 +178,11 @@ impl Package {
                         continue;
                     };
 
-                    inner.entry(t).or_default().insert(path);
+                    tracing::debug!("Adding file {}", path.display());
+                    inner
+                        .entry(t)
+                        .or_default()
+                        .insert(path.display().to_string());
                 }
             }
         }
@@ -193,11 +201,9 @@ impl Package {
     pub fn to_sjson(&self) -> Result<String> {
         let mut map: PackageDefinition = Default::default();
 
-        for (t, paths) in self.iter() {
+        for (t, names) in self.iter() {
-            for path in paths.iter() {
+            for name in names.iter() {
-                map.entry(t.ext_name())
+                map.entry(t.ext_name()).or_default().insert(name.clone());
-                    .or_default()
-                    .insert(path.display().to_string());
             }
         }
 
@@ -223,11 +229,11 @@ impl Package {
         for _ in 0..file_count {
             let t = BundleFileType::from(r.read_u64()?);
             let hash = Murmur64::from(r.read_u64()?);
-            let path = ctx.lookup_hash(hash, HashGroup::Filename);
+            let name = ctx.lookup_hash(hash, HashGroup::Filename);
             inner
                 .entry(t)
                 .or_default()
-                .insert(PathBuf::from(path.display().to_string()));
+                .insert(name.display().to_string());
         }
 
         let flags = r.read_u8()?;
@@ -240,7 +246,7 @@ impl Package {
 
         let pkg = Self {
             inner,
-            _name: name,
+            _name: name.into(),
             _root: PathBuf::new(),
             flags,
         };
@@ -256,12 +262,10 @@ impl Package {
         w.write_u32(0x2b)?;
         w.write_u32(self.values().flatten().count() as u32)?;
 
-        for (t, paths) in self.iter() {
+        for (t, names) in self.iter() {
-            for path in paths.iter() {
+            for name in names.iter() {
                 w.write_u64(t.hash().into())?;
-
+                w.write_u64(Murmur64::hash(name.as_bytes()).into())?;
-                let hash = Murmur64::hash(path.to_slash_lossy().as_bytes());
-                w.write_u64(hash.into())?;
             }
         }
 

lib/sdk/src/murmur/types.rs

@@ -1,3 +1,7 @@
+use std::path::Path;
+
+use path_slash::PathExt;
+
 use self::util::{parse_hex32, parse_hex64};
 
 use super::*;
@@ -263,11 +267,23 @@ impl IdString64 {
             IdString64::String(_) => false,
         }
     }
+
+    // Would love to have this as a proper `impl From`, but
+    // rustc will complain that it overlaps with the `impl From<Into<String>>`.
+    pub fn from_path(p: impl AsRef<Path>) -> Self {
+        Self::String(p.as_ref().to_slash_lossy().to_string())
+    }
 }
 
-impl<S: Into<String>> From<S> for IdString64 {
+impl From<String> for IdString64 {
-    fn from(value: S) -> Self {
+    fn from(value: String) -> Self {
-        Self::String(value.into())
+        Self::String(value)
+    }
+}
+
+impl From<u64> for IdString64 {
+    fn from(value: u64) -> Self {
+        Self::Hash(value.into())
     }
 }
 
@@ -283,6 +299,12 @@ impl From<IdString64> for Murmur64 {
     }
 }
 
+impl Default for IdString64 {
+    fn default() -> Self {
+        Self::Hash(0.into())
+    }
+}
+
 impl PartialEq for IdString64 {
     fn eq(&self, other: &Self) -> bool {
         self.to_murmur64() == other.to_murmur64()