feat(dtmt): Implement mod migration utility

Closes #87.

CHANGELOG.adoc

@@ -13,6 +13,7 @@
 - dtmm: check mod order before deployment
 - dtmt: add mod dependencies to config
 - dtmm: match mods to Nexus and check for updates
+- dtmt: add utility to migrate mod projects
 
 === Fixed
 

Cargo.lock

@@ -849,6 +849,7 @@ dependencies = [
  "futures-util",
  "glob",
  "libloading",
+ "luajit2-sys",
  "nanorand",
  "notify",
  "oodle",

crates/dtmt/Cargo.toml

@@ -32,6 +32,7 @@ path-clean = "1.0.1"
 path-slash = "0.2.1"
 async-recursion = "1.0.2"
 notify = "5.1.0"
+luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" }
 
 [dev-dependencies]
 tempfile = "3.3.0"

crates/dtmt/src/cmd/migrate.rs

@@ -0,0 +1,408 @@
+use std::collections::HashMap;
+use std::ffi::{CStr, CString};
+use std::path::{Path, PathBuf};
+
+use clap::{value_parser, Arg, ArgMatches, Command};
+use color_eyre::eyre::{self, Context};
+use color_eyre::{Help, Report, Result};
+use dtmt_shared::{ModConfig, ModConfigResources, ModDependency};
+use futures::FutureExt;
+use luajit2_sys as lua;
+use tokio::fs;
+use tokio_stream::wrappers::ReadDirStream;
+use tokio_stream::StreamExt;
+
+pub(crate) fn command_definition() -> Command {
+    Command::new("migrate")
+        .about("Migrate a mod project from the loose file structure to DTMT.")
+        .arg(
+            Arg::new("mod-file")
+                .required(true)
+                .value_parser(value_parser!(PathBuf))
+                .help("The path to the mod's '<id>.mod' file."),
+        )
+        .arg(
+            Arg::new("directory")
+                .required(true)
+                .value_parser(value_parser!(PathBuf))
+                .help(
+                    "The directory to create the mod in. Within this directory, \
+                        DTMT will create a new folder named after the mod ID and migrate files \
+                        into that folder.",
+                ),
+        )
+}
+
+#[derive(Clone, Debug)]
+struct ModFile {
+    id: String,
+    init: PathBuf,
+    data: Option<PathBuf>,
+    localization: Option<PathBuf>,
+}
+
+// This piece of Lua code stubs DMF functions and runs a mod's `.mod` file to extract
+// the contained information.
+static MOD_FILE_RUNNER: &str = r#"
+_DATA = {}
+
+function fassert() end
+
+function new_mod(id, options)
+    _DATA.id = id
+    _DATA.init = options.mod_script
+    _DATA.data = options.mod_data
+    _DATA.localization = options.mod_localization
+end
+
+dmf = {
+    dofile = function(self, file)
+        _DATA.init = file
+    end
+}
+
+_MOD().run()
+"#;
+
+#[tracing::instrument]
+async fn evaluate_mod_file(path: impl AsRef<Path> + std::fmt::Debug) -> Result<ModFile> {
+    let path = path.as_ref();
+    let code = fs::read(path)
+        .await
+        .wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?;
+
+    tokio::task::spawn_blocking(move || unsafe {
+        let state = lua::luaL_newstate();
+        lua::luaL_openlibs(state);
+
+        let code = CString::new(code).expect("Cannot build CString");
+        let name = CString::new("_MOD").expect("Cannot build CString");
+
+        match lua::luaL_loadstring(state, code.as_ptr()) as u32 {
+            lua::LUA_OK => {}
+            lua::LUA_ERRSYNTAX => {
+                let err = lua::lua_tostring(state, -1);
+                let err = CStr::from_ptr(err).to_string_lossy().to_string();
+
+                lua::lua_close(state);
+
+                eyre::bail!("Invalid syntax: {}", err);
+            }
+            lua::LUA_ERRMEM => {
+                lua::lua_close(state);
+                eyre::bail!("Failed to allocate sufficient memory")
+            }
+            _ => unreachable!(),
+        }
+
+        tracing::trace!("Loaded '.mod' code");
+
+        lua::lua_setglobal(state, name.as_ptr());
+
+        let code = CString::new(MOD_FILE_RUNNER).expect("Cannot build CString");
+        match lua::luaL_loadstring(state, code.as_ptr()) as u32 {
+            lua::LUA_OK => {}
+            lua::LUA_ERRSYNTAX => {
+                let err = lua::lua_tostring(state, -1);
+                let err = CStr::from_ptr(err).to_string_lossy().to_string();
+
+                lua::lua_close(state);
+
+                eyre::bail!("Invalid syntax: {}", err);
+            }
+            lua::LUA_ERRMEM => {
+                lua::lua_close(state);
+                eyre::bail!("Failed to allocate sufficient memory")
+            }
+            _ => unreachable!(),
+        }
+
+        match lua::lua_pcall(state, 0, 1, 0) as u32 {
+            lua::LUA_OK => {}
+            lua::LUA_ERRRUN => {
+                let err = lua::lua_tostring(state, -1);
+                let err = CStr::from_ptr(err).to_string_lossy().to_string();
+
+                lua::lua_close(state);
+
+                eyre::bail!("Failed to evaluate '.mod' file: {}", err);
+            }
+            lua::LUA_ERRMEM => {
+                lua::lua_close(state);
+                eyre::bail!("Failed to allocate sufficient memory")
+            }
+            // We don't use an error handler function, so this should be unreachable
+            lua::LUA_ERRERR => unreachable!(),
+            _ => unreachable!(),
+        }
+
+        tracing::trace!("Loaded file runner code");
+
+        let name = CString::new("_DATA").expect("Cannot build CString");
+        lua::lua_getglobal(state, name.as_ptr());
+
+        let id = {
+            let name = CString::new("id").expect("Cannot build CString");
+            lua::lua_getfield(state, -1, name.as_ptr());
+            let val = {
+                let ptr = lua::lua_tostring(state, -1);
+                let str = CStr::from_ptr(ptr);
+                str.to_str()
+                    .expect("ID value is not a valid string")
+                    .to_string()
+            };
+            lua::lua_pop(state, 1);
+            val
+        };
+
+        let path_prefix = format!("{id}/");
+
+        let init = {
+            let name = CString::new("init").expect("Cannot build CString");
+            lua::lua_getfield(state, -1, name.as_ptr());
+            let val = {
+                let ptr = lua::lua_tostring(state, -1);
+                let str = CStr::from_ptr(ptr);
+                str.to_str().expect("ID value is not a valid string")
+            };
+            lua::lua_pop(state, 1);
+            PathBuf::from(val.strip_prefix(&path_prefix).unwrap_or(val))
+        };
+
+        let data = {
+            let name = CString::new("data").expect("Cannot build CString");
+            lua::lua_getfield(state, -1, name.as_ptr());
+
+            if lua::lua_isnil(state, -1) > 0 {
+                None
+            } else {
+                let val = {
+                    let ptr = lua::lua_tostring(state, -1);
+                    let str = CStr::from_ptr(ptr);
+                    str.to_str().expect("ID value is not a valid string")
+                };
+                lua::lua_pop(state, 1);
+                Some(PathBuf::from(val.strip_prefix(&path_prefix).unwrap_or(val)))
+            }
+        };
+
+        let localization = {
+            let name = CString::new("localization").expect("Cannot build CString");
+            lua::lua_getfield(state, -1, name.as_ptr());
+
+            if lua::lua_isnil(state, -1) > 0 {
+                None
+            } else {
+                let val = {
+                    let ptr = lua::lua_tostring(state, -1);
+                    let str = CStr::from_ptr(ptr);
+                    str.to_str().expect("ID value is not a valid string")
+                };
+                lua::lua_pop(state, 1);
+                Some(PathBuf::from(val.strip_prefix(&path_prefix).unwrap_or(val)))
+            }
+        };
+
+        lua::lua_close(state);
+
+        let mod_file = ModFile {
+            id,
+            init,
+            data,
+            localization,
+        };
+
+        tracing::trace!(?mod_file);
+
+        Ok(mod_file)
+    })
+    .await
+    .map_err(Report::new)
+    .flatten()
+    .wrap_err("Failed to run mod file handler")
+}
+
+#[async_recursion::async_recursion]
+#[tracing::instrument]
+async fn process_directory<P1, P2>(path: P1, prefix: P2) -> Result<()>
+where
+    P1: AsRef<Path> + std::fmt::Debug + std::marker::Send,
+    P2: AsRef<Path> + std::fmt::Debug + std::marker::Send,
+{
+    let path = path.as_ref();
+    let prefix = prefix.as_ref();
+
+    let read_dir = fs::read_dir(&path)
+        .await
+        .wrap_err_with(|| format!("Failed to read directory '{}'", path.display()))?;
+
+    let stream = ReadDirStream::new(read_dir).map(|res| res.wrap_err("Failed to read dir entry"));
+    tokio::pin!(stream);
+
+    while let Some(res) = stream.next().await {
+        let entry = res?;
+        let in_path = entry.path();
+        let out_path = prefix.join(entry.file_name());
+
+        let t = entry.file_type().await?;
+
+        if t.is_dir() {
+            process_directory(in_path, out_path).await?;
+        } else {
+            tracing::trace!(
+                "Copying file '{}' -> '{}'",
+                in_path.display(),
+                out_path.display()
+            );
+            let res = fs::create_dir_all(prefix)
+                .then(|_| fs::copy(&in_path, &out_path))
+                .await
+                .wrap_err_with(|| {
+                    format!(
+                        "Failed to copy '{}' -> '{}'",
+                        in_path.display(),
+                        out_path.display()
+                    )
+                });
+            if let Err(err) = res {
+                tracing::error!("{:?}", err);
+            }
+        }
+    }
+
+    Ok(())
+}
+
+#[tracing::instrument(skip_all)]
+pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
+    let (mod_file, in_dir) = {
+        let path = matches
+            .get_one::<PathBuf>("mod-file")
+            .expect("Parameter is required");
+
+        let mod_file = evaluate_mod_file(&path)
+            .await
+            .wrap_err("Failed to evaluate '.mod' file")?;
+
+        (
+            mod_file,
+            path.parent().expect("A file path always has a parent"),
+        )
+    };
+
+    let out_dir = matches
+        .get_one::<PathBuf>("directory")
+        .expect("Parameter is required");
+
+    {
+        let is_dir = fs::metadata(out_dir)
+            .await
+            .map(|meta| meta.is_dir())
+            .unwrap_or(false);
+
+        if !is_dir {
+            let err = eyre::eyre!("Invalid output directory '{}'", out_dir.display());
+            return Err(err)
+                .with_suggestion(|| "Make sure the directory exists and is writable.".to_string());
+        }
+    }
+
+    let out_dir = out_dir.join(&mod_file.id);
+
+    println!(
+        "Enter additional information about your mod '{}'!",
+        &mod_file.id
+    );
+
+    let name = promptly::prompt_default("Display name", mod_file.id.clone())
+        .map(|s: String| s.trim().to_string())?;
+    let summary = promptly::prompt("Short summary").map(|s: String| s.trim().to_string())?;
+    let author =
+        promptly::prompt_opt("Author").map(|opt| opt.map(|s: String| s.trim().to_string()))?;
+    let version = promptly::prompt_default("Version", String::from("0.1.0"))
+        .map(|s: String| s.trim().to_string())?;
+    let categories = promptly::prompt("Categories (comma separated list)")
+        .map(|s: String| s.trim().to_string())
+        .map(|s: String| s.split(',').map(|s| s.trim().to_string()).collect())?;
+
+    let packages = vec![PathBuf::from("packages/mods").join(&mod_file.id)];
+
+    let dtmt_cfg = ModConfig {
+        dir: out_dir,
+        id: mod_file.id,
+        name,
+        summary,
+        author,
+        version,
+        description: None,
+        image: None,
+        categories,
+        packages,
+        resources: ModConfigResources {
+            init: mod_file.init,
+            data: mod_file.data,
+            localization: mod_file.localization,
+        },
+        depends: vec![ModDependency::ID(String::from("DMF"))],
+    };
+
+    tracing::debug!(?dtmt_cfg);
+
+    fs::create_dir(&dtmt_cfg.dir).await.wrap_err_with(|| {
+        format!(
+            "Failed to create mod directory '{}'",
+            dtmt_cfg.dir.display()
+        )
+    })?;
+
+    tracing::info!("Created mod directory '{}'", dtmt_cfg.dir.display());
+
+    {
+        let path = dtmt_cfg.dir.join("dtmt.cfg");
+        let data = serde_sjson::to_string(&dtmt_cfg).wrap_err("Failed to serialize dtmt.cfg")?;
+        fs::write(&path, &data)
+            .await
+            .wrap_err_with(|| format!("Failed to write '{}'", path.display()))?;
+
+        tracing::info!("Created mod configuration at '{}'", path.display());
+    }
+
+    {
+        let path = dtmt_cfg
+            .dir
+            .join(&dtmt_cfg.packages[0])
+            .with_extension("package");
+
+        let data = {
+            let mut map = HashMap::new();
+            map.insert("lua", vec![format!("scripts/mods/{}/*", dtmt_cfg.id)]);
+            map
+        };
+        let data = serde_sjson::to_string(&data).wrap_err("Failed to serialize package file")?;
+
+        fs::create_dir_all(path.parent().unwrap())
+            .then(|_| fs::write(&path, &data))
+            .await
+            .wrap_err_with(|| format!("Failed to write '{}'", path.display()))?;
+
+        tracing::info!("Created package file at '{}'", path.display());
+    }
+
+    {
+        let path = in_dir.join("scripts");
+        let scripts_dir = dtmt_cfg.dir.join("scripts");
+        process_directory(&path, &scripts_dir)
+            .await
+            .wrap_err_with(|| {
+                format!(
+                    "Failed to copy files from '{}' to '{}'",
+                    path.display(),
+                    scripts_dir.display()
+                )
+            })?;
+
+        tracing::info!("Copied script files to '{}'", scripts_dir.display());
+    }
+
+    Ok(())
+}

crates/dtmt/src/main.rs

@@ -1,5 +1,6 @@
 #![feature(io_error_more)]
 #![feature(let_chains)]
+#![feature(result_flattening)]
 #![windows_subsystem = "console"]
 
 use std::path::PathBuf;
@@ -19,6 +20,7 @@ mod cmd {
     pub mod build;
     pub mod bundle;
     pub mod dictionary;
+    pub mod migrate;
     pub mod murmur;
     pub mod new;
     pub mod package;
@@ -52,6 +54,7 @@ async fn main() -> Result<()> {
         .subcommand(cmd::build::command_definition())
         .subcommand(cmd::bundle::command_definition())
         .subcommand(cmd::dictionary::command_definition())
+        .subcommand(cmd::migrate::command_definition())
         .subcommand(cmd::murmur::command_definition())
         .subcommand(cmd::new::command_definition())
         .subcommand(cmd::package::command_definition())
@@ -128,6 +131,7 @@ async fn main() -> Result<()> {
         Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?,
         Some(("bundle", sub_matches)) => cmd::bundle::run(ctx, sub_matches).await?,
         Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?,
+        Some(("migrate", sub_matches)) => cmd::migrate::run(ctx, sub_matches).await?,
         Some(("murmur", sub_matches)) => cmd::murmur::run(ctx, sub_matches).await?,
         Some(("new", sub_matches)) => cmd::new::run(ctx, sub_matches).await?,
         Some(("package", sub_matches)) => cmd::package::run(ctx, sub_matches).await?,

lib/dtmt-shared/src/lib.rs

@@ -3,46 +3,50 @@ use std::path::PathBuf;
 mod log;
 
 pub use log::*;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use steamlocate::SteamDir;
 use time::OffsetDateTime;
 
-#[derive(Clone, Debug, Default, Deserialize)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct ModConfigResources {
     pub init: PathBuf,
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub data: Option<PathBuf>,
-    #[serde(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
     pub localization: Option<PathBuf>,
 }
 
-#[derive(Clone, Debug, PartialEq, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
 #[serde(rename_all = "snake_case")]
 pub enum ModOrder {
     Before,
     After,
 }
 
-#[derive(Clone, Debug, PartialEq, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
 #[serde(untagged)]
 pub enum ModDependency {
     ID(String),
     Config { id: String, order: ModOrder },
 }
 
-#[derive(Clone, Debug, Default, Deserialize)]
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct ModConfig {
     #[serde(skip)]
     pub dir: PathBuf,
     pub id: String,
     pub name: String,
     pub summary: String,
-    pub description: Option<String>,
-    pub author: Option<String>,
     pub version: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub author: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub image: Option<PathBuf>,
     #[serde(default)]
     pub categories: Vec<String>,
+    #[serde(default)]
     pub packages: Vec<PathBuf>,
     pub resources: ModConfigResources,
     #[serde(default)]