diff --git a/Cargo.lock b/Cargo.lock index a7885f2..8385e67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,12 +39,10 @@ dependencies = [ [[package]] name = "ansi-parser" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127" +version = "0.9.0" dependencies = [ "heapless", - "nom 4.2.3", + "nom", ] [[package]] @@ -381,7 +379,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -896,6 +894,7 @@ name = "dtmm" version = "0.1.0" dependencies = [ "ansi-parser", + "async-recursion", "bitflags 1.3.2", "clap", "color-eyre", @@ -906,6 +905,8 @@ dependencies = [ "dtmt-shared", "futures", "lazy_static", + "luajit2-sys", + "minijinja", "nexusmods", "oodle", "path-slash", @@ -1390,7 +1391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -1614,12 +1615,12 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heapless" -version = "0.5.6" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1" +checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422" dependencies = [ "as-slice", - "generic-array 0.13.3", + "generic-array 0.14.7", "hash32", "stable_deref_trait", ] @@ -1747,7 +1748,7 @@ dependencies = [ "serde", "sized-chunks", "typenum", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -2084,6 +2085,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2175,16 +2185,6 @@ dependencies = [ "memoffset 0.6.5", ] -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - [[package]] name = "nom" version = "7.1.3" @@ -2203,7 +2203,7 @@ checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3" dependencies = [ "bytecount", "memchr", - "nom 7.1.3", + "nom", ] [[package]] @@ -2673,7 +2673,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -2684,7 +2684,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -3102,7 +3102,7 @@ dependencies = [ name = "serde_sjson" version = "1.0.0" dependencies = [ - "nom 7.1.3", + "nom", "nom_locate", "serde", ] @@ -3832,7 +3832,7 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ - "version_check 0.9.4", + "version_check", ] [[package]] @@ -3966,12 +3966,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.4" @@ -4384,7 +4378,3 @@ dependencies = [ "cc", "pkg-config", ] - -[[patch.unused]] -name = "ansi-parser" -version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 123b2ef..451cd09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,10 +7,7 @@ members = [ "lib/oodle", "lib/sdk", "lib/serde_sjson", -] -exclude = [ - "lib/color-eyre", - "lib/ansi-parser", + "lib/luajit2-sys", ] [patch.crates-io] diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 86444be..5f60220 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -31,5 +31,8 @@ lazy_static = "1.4.0" colors-transform = "0.2.11" usvg = "0.25.0" druid-widget-nursery = "0.1" -ansi-parser = "0.8.0" +ansi-parser = "0.9.0" string_template = "0.2.1" +luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } +async-recursion = "1.0.5" +minijinja = "1.0.10" diff --git a/crates/dtmm/assets/mod_data.lua.j2 b/crates/dtmm/assets/mod_data.lua.j2 new file mode 100644 index 0000000..9f87ad1 --- /dev/null +++ b/crates/dtmm/assets/mod_data.lua.j2 @@ -0,0 +1,27 @@ +return { +{% for mod in mods %} +{ + id = "{{ mod.id }}", + name = "{{ mod.name }}", + bundled = {{ mod.bundled }}, + packages = { + {% for pkg in mod.packages %} + "{{ pkg }}", + {% endfor %} + }, + run = function() + {% if mod.data is none %} + return dofile("{{ mod.init }}") + {% else %} + new_mod("{{ mod.id }}", { + mod_script = "{{ mod.init }}", + mod_data = "{{ mod.data }}", + {% if not mod.localization is none %} + mod_localization = "{{ mod.localization }}", + {% endif %} + }) + {% endif %} + end, +}, +{% endfor %} +} diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua index 6dbf3e2..e4006f6 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua @@ -11,100 +11,6 @@ local log = function(category, format, ...) end end --- Patch `GameStateMachine.init` to add our own state for loading mods. --- In the future, Fatshark might provide us with a dedicated way to do this. -local function patch_mod_loading_state() - local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") - - -- A necessary override. - -- The original does not proxy `dt` to `_state_update`, but we need that. - StateBootSubStateBase.update = function(self, dt) - local done, error = self:_state_update(dt) - local params = self._params - - if error then - return StateError, { error } - elseif done then - local next_index = params.sub_state_index + 1 - params.sub_state_index = next_index - local next_state_data = params.states[next_index] - - if next_state_data then - return next_state_data[1], self._params - else - self._parent:sub_states_done() - end - end - end - - local StateBootLoadMods = class("StateBootLoadMods", "StateBootSubStateBase") - - StateBootLoadMods.on_enter = function(self, parent, params) - log("StateBootLoadMods", "Entered") - StateBootLoadMods.super.on_enter(self, parent, params) - - local state_params = self:_state_params() - local package_manager = state_params.package_manager - - self._state = "load_package" - self._package_manager = package_manager - self._package_handles = { - ["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadMods", nil), - ["packages/dml"] = package_manager:load("packages/dml", "StateBootLoadMods", nil), - } - end - - StateBootLoadMods._state_update = function(self, dt) - local state = self._state - local package_manager = self._package_manager - - if state == "load_package" and package_manager:update() then - log("StateBootLoadMods", "Packages loaded, loading mods") - self._state = "load_mods" - local ModLoader = require("scripts/mods/dml/init") - - local mod_data = require("scripts/mods/mod_data") - local mod_loader = ModLoader:new(mod_data, self._parent:gui()) - - self._mod_loader = mod_loader - Managers.mod = mod_loader - elseif state == "load_mods" and self._mod_loader:update(dt) then - log("StateBootLoadMods", "Mods loaded, exiting") - return true, false - end - - return false, false - end - - local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") - - local patched = false - - local GameStateMachine_init = GameStateMachine.init - GameStateMachine.init = function(self, parent, start_state, params, ...) - if not patched then - log("mod_main", "Injecting mod loading state") - patched = true - - -- Hardcoded position after `StateRequireScripts`. - -- We do want to wait until then, so that most of the game's core - -- systems are at least loaded and can be hooked, even if they aren't - -- running, yet. - local pos = 4 - table.insert(params.states, pos, { - StateBootLoadMods, - { - package_manager = params.package_manager, - }, - }) - end - - GameStateMachine_init(self, parent, start_state, params, ...) - end - - log("mod_main", "Mod patching complete") -end - log("mod_main", "Initializing mods...") local require_store = {} @@ -112,7 +18,7 @@ local require_store = {} -- This token is treated as a string template and filled by DTMM during deployment. -- This allows hiding unsafe I/O functions behind a setting. -- It's also a valid table definition, thereby degrading gracefully when not replaced. -local is_io_enabled = { { is_io_enabled } } -- luacheck: ignore 113 +local is_io_enabled = {{ is_io_enabled }} -- luacheck: ignore 113 local lua_libs = { debug = debug, os = { @@ -138,7 +44,8 @@ Mods = { -- Fatshark's code scrubs them. -- The loader can then decide to pass them on to mods, or ignore them lua = setmetatable({}, { __index = lua_libs }), - require_store = require_store + require_store = require_store, + original_require = require, } local can_insert = function(filepath, new_result) @@ -198,6 +105,98 @@ end require("scripts/main") log("mod_main", "'scripts/main' loaded") +-- Inject our state into the game. The state needs to run after `StateGame._init_managers`, +-- since some parts of DMF, and presumably other mods, depend on some of those managers to exist. +local function patch_mod_loading_state() + local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") + local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase") + local StateGameLoadMods = class("StateGameLoadMods") + + StateBootLoadDML.on_enter = function(self, parent, params) + log("StateBootLoadDML", "Entered") + StateBootLoadDML.super.on_enter(self, parent, params) + + local state_params = self:_state_params() + local package_manager = state_params.package_manager + + self._package_manager = package_manager + self._package_handles = { + ["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil), + ["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil), + } + end + + StateBootLoadDML._state_update = function(self, dt) + local package_manager = self._package_manager + + if package_manager:update() then + local DML = require("scripts/mods/dml/init") + local mod_data = require("scripts/mods/mod_data") + local mod_loader = DML.create_loader(mod_data) + Managers.mod = mod_loader + log("StateBootLoadDML", "DML loaded, exiting") + return true, false + end + + return false, false + end + + + function StateGameLoadMods:on_enter(_, params) + log("StateGameLoadMods", "Entered") + self._next_state = require("scripts/game_states/game/state_splash") + self._next_state_params = params + end + + function StateGameLoadMods:update(main_dt) + local state = self._loading_state + + -- We're relying on the fact that DML internally makes sure + -- that `Managers.mod:update()` is being called appropriately. + -- The implementation as of this writing is to hook `StateGame.update`. + if Managers.mod:all_mods_loaded() then + Log.info("StateGameLoadMods", "Mods loaded, exiting") + return self._next_state, self._next_state_params + end + end + + local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") + local GameStateMachine_init = GameStateMachine.init + GameStateMachine.init = function(self, parent, start_state, params, creation_context, state_change_callbacks, name) + if name == "Main" then + log("mod_main", "Injecting StateBootLoadDML") + + -- Hardcoded position after `StateRequireScripts`. + -- We need to wait until then to even begin most of our stuff, + -- so that most of the game's core systems are at least loaded and can be hooked, + -- even if they aren't running, yet. + local pos = 4 + table.insert(params.states, pos, { + StateBootLoadDML, + { + package_manager = params.package_manager, + }, + }) + + GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) + elseif name == "Game" then + log("mod_main", "Injection StateGameLoadMods") + -- The second time around, we want to be the first, so we pass our own + -- 'start_state'. + -- We can't just have the state machine be initialized and then change its `_next_state`, as by the end of + -- `init`, a bunch of stuff will already be initialized. + GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name) + -- And since we're done now, we can revert the function to its original + GameStateMachine.init = GameStateMachine_init + + return + else + -- In all other cases, simply call the original + GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) + end + end +end + -- Override `init` to run our injection function init() patch_mod_loading_state() diff --git a/crates/dtmm/src/controller/app.rs b/crates/dtmm/src/controller/app.rs index 5c5d980..cb6e2f0 100644 --- a/crates/dtmm/src/controller/app.rs +++ b/crates/dtmm/src/controller/app.rs @@ -1,18 +1,17 @@ use std::collections::HashMap; -use std::io::{Cursor, ErrorKind, Read}; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::Arc; use color_eyre::eyre::{self, Context}; use color_eyre::{Help, Report, Result}; use druid::im::Vector; -use druid::{FileInfo, ImageBuf}; +use druid::ImageBuf; use dtmt_shared::ModConfig; use nexusmods::Api as NexusApi; use tokio::fs::{self, DirEntry, File}; use tokio_stream::wrappers::ReadDirStream; use tokio_stream::StreamExt; -use zip::ZipArchive; use crate::state::{ActionState, InitialLoadResult, ModInfo, ModOrder, NexusInfo, PackageInfo}; use crate::util; @@ -20,161 +19,6 @@ use crate::util::config::{ConfigSerialize, LoadOrderEntry}; use super::read_sjson_file; -#[tracing::instrument(skip(state))] -pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { - let data = fs::read(&info.path) - .await - .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; - let data = Cursor::new(data); - - let nexus = if let Some((_, id, _, _)) = info - .path - .file_name() - .and_then(|s| s.to_str()) - .and_then(NexusApi::parse_file_name) - { - if !state.nexus_api_key.is_empty() { - let api = NexusApi::new(state.nexus_api_key.to_string())?; - let mod_info = api - .mods_id(id) - .await - .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; - Some(NexusInfo::from(mod_info)) - } else { - None - } - } else { - None - }; - - let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; - - if tracing::enabled!(tracing::Level::DEBUG) { - let names = archive.file_names().fold(String::new(), |mut s, name| { - s.push('\n'); - s.push_str(name); - s - }); - tracing::debug!("Archive contents:{}", names); - } - - let dir_name = { - let f = archive.by_index(0).wrap_err("Archive is empty")?; - - if !f.is_dir() { - let err = eyre::eyre!("archive does not have a top-level directory"); - return Err(err).with_suggestion(|| "Use 'dtmt build' to create the mod archive."); - } - - let name = f.name(); - // The directory name is returned with a trailing slash, which we don't want - name[..(name.len().saturating_sub(1))].to_string() - }; - - tracing::info!("Importing mod {}", dir_name); - - let names: Vec<_> = archive.file_names().map(|s| s.to_string()).collect(); - - let mod_cfg: ModConfig = { - let name = names - .iter() - .find(|name| name.ends_with("dtmt.cfg")) - .ok_or_else(|| eyre::eyre!("archive does not contain mod config"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read mod config from archive")?; - - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read mod config from archive")?; - - let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; - - serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")? - }; - - tracing::debug!(?mod_cfg); - - let files: HashMap> = { - let name = names - .iter() - .find(|name| name.ends_with("files.sjson")) - .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read file index from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; - - serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? - }; - - tracing::trace!(?files); - - let image = if let Some(path) = &mod_cfg.image { - let name = names - .iter() - .find(|name| name.ends_with(&path.display().to_string())) - .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read image file from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - // Druid somehow doesn't return an error compatible with eyre, here. - // So we have to wrap through `Display` manually. - let img = match ImageBuf::from_data(&buf) { - Ok(img) => img, - Err(err) => { - let err = Report::msg(err.to_string()).wrap_err("Invalid image data"); - return Err(err).with_suggestion(|| { - "Supported formats are: PNG, JPEG, Bitmap and WebP".to_string() - }); - } - }; - - Some(img) - } else { - None - }; - - let mod_dir = state.mod_dir; - - tracing::trace!("Creating mods directory {}", mod_dir.display()); - fs::create_dir_all(Arc::as_ref(&mod_dir)) - .await - .wrap_err_with(|| format!("Failed to create data directory {}", mod_dir.display()))?; - - tracing::trace!("Extracting mod archive to {}", mod_dir.display()); - archive - .extract(Arc::as_ref(&mod_dir)) - .wrap_err_with(|| format!("Failed to extract archive to {}", mod_dir.display()))?; - - if let Some(nexus) = &nexus { - let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; - let path = mod_dir.join(&mod_cfg.id).join("nexus.sjson"); - fs::write(&path, data.as_bytes()) - .await - .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; - } - - let packages = files - .into_iter() - .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) - .collect(); - let info = ModInfo::new(mod_cfg, packages, image, nexus); - - Ok(info) -} - #[tracing::instrument(skip(state))] pub(crate) async fn delete_mod(state: ActionState, info: &ModInfo) -> Result<()> { let mod_dir = state.mod_dir.join(&info.id); @@ -229,9 +73,13 @@ async fn read_mod_dir_entry(res: Result) -> Result { Err(err) => return Err(err), }; - let files: HashMap> = read_sjson_file(&index_path) - .await - .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?; + let files: HashMap> = if cfg.bundled { + read_sjson_file(&index_path) + .await + .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))? + } else { + Default::default() + }; let image = if let Some(path) = &cfg.image { let path = entry.path().join(path); diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs new file mode 100644 index 0000000..53c4ef1 --- /dev/null +++ b/crates/dtmm/src/controller/deploy.rs @@ -0,0 +1,868 @@ +use std::collections::HashMap; +use std::io::{Cursor, ErrorKind}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; + +use color_eyre::eyre::Context; +use color_eyre::{eyre, Help, Report, Result}; +use futures::StreamExt; +use futures::{stream, TryStreamExt}; +use minijinja::Environment; +use sdk::filetype::lua; +use sdk::filetype::package::Package; +use sdk::murmur::Murmur64; +use sdk::{ + Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, +}; +use serde::{Deserialize, Serialize}; +use string_template::Template; +use time::OffsetDateTime; +use tokio::fs::{self, DirEntry}; +use tokio::io::AsyncWriteExt; +use tracing::Instrument; + +use super::read_sjson_file; +use crate::controller::app::check_mod_order; +use crate::state::{ActionState, PackageInfo}; + +pub const MOD_BUNDLE_NAME: &str = "packages/mods"; +pub const BOOT_BUNDLE_NAME: &str = "packages/boot"; +pub const DML_BUNDLE_NAME: &str = "packages/dml"; +pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; +pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; +pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; +pub const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; +pub const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeploymentData { + pub bundles: Vec, + pub mod_folders: Vec, + #[serde(with = "time::serde::iso8601")] + pub timestamp: OffsetDateTime, +} + +#[tracing::instrument] +async fn read_file_with_backup

(path: P) -> Result> +where + P: AsRef + std::fmt::Debug, +{ + let path = path.as_ref(); + let backup_path = { + let mut p = PathBuf::from(path); + let ext = if let Some(ext) = p.extension() { + ext.to_string_lossy().to_string() + ".bak" + } else { + String::from("bak") + }; + p.set_extension(ext); + p + }; + + let file_name = path + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| String::from("file")); + + let bin = match fs::read(&backup_path).await { + Ok(bin) => bin, + Err(err) if err.kind() == ErrorKind::NotFound => { + // TODO: This doesn't need to be awaited here, yet. + // I only need to make sure it has finished before writing the changed bundle. + tracing::debug!( + "Backup does not exist. Backing up original {} to '{}'", + file_name, + backup_path.display() + ); + fs::copy(path, &backup_path).await.wrap_err_with(|| { + format!( + "Failed to back up {} '{}' to '{}'", + file_name, + path.display(), + backup_path.display() + ) + })?; + + tracing::debug!("Reading {} from original '{}'", file_name, path.display()); + fs::read(path).await.wrap_err_with(|| { + format!("Failed to read {} file: {}", file_name, path.display()) + })? + } + Err(err) => { + return Err(err).wrap_err_with(|| { + format!( + "Failed to read {} from backup '{}'", + file_name, + backup_path.display() + ) + }); + } + }; + Ok(bin) +} + +#[tracing::instrument(skip_all)] +async fn patch_game_settings(state: Arc) -> Result<()> { + let settings_path = state.game_dir.join("bundle").join(SETTINGS_FILE_PATH); + + let settings = read_file_with_backup(&settings_path) + .await + .wrap_err("Failed to read settings.ini")?; + let settings = String::from_utf8(settings).wrap_err("Settings.ini is not valid UTF-8")?; + + let mut f = fs::File::create(&settings_path) + .await + .wrap_err_with(|| format!("Failed to open {}", settings_path.display()))?; + + let Some(i) = settings.find("boot_script =") else { + eyre::bail!("couldn't find 'boot_script' field"); + }; + + f.write_all(settings[0..i].as_bytes()).await?; + f.write_all(b"boot_script = \"scripts/mod_main\"").await?; + + let Some(j) = settings[i..].find('\n') else { + eyre::bail!("couldn't find end of 'boot_script' field"); + }; + + f.write_all(settings[(i + j)..].as_bytes()).await?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(package = info.name))] +fn make_package(info: &PackageInfo) -> Result { + let mut pkg = Package::new(info.name.clone(), PathBuf::new()); + + for f in &info.files { + let mut it = f.rsplit('.'); + let file_type = it + .next() + .ok_or_else(|| eyre::eyre!("missing file extension")) + .and_then(BundleFileType::from_str) + .wrap_err("Invalid file name in package info")?; + let name: String = it.collect(); + pkg.add_file(file_type, name); + } + + Ok(pkg) +} + +#[tracing::instrument] +async fn copy_recursive( + from: impl Into + std::fmt::Debug, + to: impl AsRef + std::fmt::Debug, +) -> Result<()> { + let to = to.as_ref(); + + #[tracing::instrument] + async fn handle_dir(from: PathBuf) -> Result> { + let mut dir = fs::read_dir(&from) + .await + .wrap_err("Failed to read directory")?; + let mut entries = Vec::new(); + + while let Some(entry) = dir.next_entry().await? { + let meta = entry.metadata().await.wrap_err_with(|| { + format!("Failed to get metadata for '{}'", entry.path().display()) + })?; + entries.push((meta.is_dir(), entry)); + } + + Ok(entries) + } + + let base = from.into(); + stream::unfold(vec![base.clone()], |mut state| async { + let from = state.pop()?; + let inner = match handle_dir(from).await { + Ok(entries) => { + for (is_dir, entry) in &entries { + if *is_dir { + state.push(entry.path()); + } + } + stream::iter(entries).map(Ok).left_stream() + } + Err(e) => stream::once(async { Err(e) }).right_stream(), + }; + + Some((inner, state)) + }) + .flatten() + .try_for_each(|(is_dir, entry)| { + let path = entry.path(); + let dest = path + .strip_prefix(&base) + .map(|suffix| to.join(suffix)) + .expect("all entries are relative to the directory we are walking"); + + async move { + if is_dir { + tracing::trace!("Creating directory '{}'", dest.display()); + // Instead of trying to filter "already exists" errors out explicitly, + // we just ignore all. It'll fail eventually with the next copy operation. + let _ = fs::create_dir(&dest).await; + Ok(()) + } else { + tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display()); + fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| { + format!( + "Failed to copy file '{}' -> '{}'", + path.display(), + dest.display() + ) + }) + } + } + }) + .await + .map(|_| ()) +} + +#[tracing::instrument(skip(state))] +async fn copy_mod_folders(state: Arc) -> Result> { + let game_dir = Arc::clone(&state.game_dir); + + let mut tasks = Vec::new(); + + for mod_info in state + .mods + .iter() + .filter(|m| m.id != "dml" && m.enabled && !m.bundled) + { + let span = tracing::trace_span!("copying legacy mod", name = mod_info.name); + let _enter = span.enter(); + + let mod_id = mod_info.id.clone(); + let mod_dir = Arc::clone(&state.mod_dir); + let game_dir = Arc::clone(&game_dir); + + let task = async move { + let from = mod_dir.join(&mod_id); + let to = game_dir.join("mods").join(&mod_id); + + tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id); + let _ = fs::create_dir_all(&to).await; + copy_recursive(&from, &to).await.wrap_err_with(|| { + format!( + "Failed to copy legacy mod from '{}' to '{}'", + from.display(), + to.display() + ) + })?; + + Ok::<_, Report>(mod_id) + }; + tasks.push(task); + } + + let ids = futures::future::try_join_all(tasks).await?; + Ok(ids) +} + +fn build_mod_data_lua(state: Arc) -> Result { + #[derive(Serialize)] + struct TemplateDataMod { + id: String, + name: String, + bundled: bool, + init: String, + data: Option, + localization: Option, + packages: Vec, + } + + let mut env = Environment::new(); + env.add_template("mod_data.lua", include_str!("../../assets/mod_data.lua.j2")) + .wrap_err("Failed to compile template for `mod_data.lua`")?; + let tmpl = env + .get_template("mod_data.lua") + .wrap_err("Failed to get template `mod_data.lua`")?; + + let data: Vec = state + .mods + .iter() + .filter_map(|m| { + if m.id == "dml" || !m.enabled { + return None; + } + + Some(TemplateDataMod { + id: m.id.clone(), + name: m.name.clone(), + bundled: m.bundled, + init: m.resources.init.to_string_lossy().to_string(), + data: m + .resources + .data + .as_ref() + .map(|p| p.to_string_lossy().to_string()), + localization: m + .resources + .localization + .as_ref() + .map(|p| p.to_string_lossy().to_string()), + packages: m.packages.iter().map(|p| p.name.clone()).collect(), + }) + }) + .collect(); + + let lua = tmpl + .render(minijinja::context!(mods => data)) + .wrap_err("Failed to render template `mod_data.lua`")?; + + tracing::debug!("mod_data.lua:\n{}", lua); + + Ok(lua) +} + +#[tracing::instrument(skip_all)] +async fn build_bundles(state: Arc) -> Result> { + let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); + let mut tasks = Vec::new(); + + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + + let mut bundles = Vec::new(); + + { + tracing::trace!("Building mod data script"); + + let span = tracing::debug_span!("Building mod data script"); + let _enter = span.enter(); + + let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?; + + tracing::trace!("Compiling mod data script"); + + let file = + lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?; + + tracing::trace!("Compile mod data script"); + + mod_bundle.add_file(file); + } + + tracing::trace!("Preparing tasks to deploy bundle files"); + + for mod_info in state + .mods + .iter() + .filter(|m| m.id != "dml" && m.enabled && m.bundled) + { + let span = tracing::trace_span!("building mod packages", name = mod_info.name); + let _enter = span.enter(); + + let mod_dir = state.mod_dir.join(&mod_info.id); + for pkg_info in &mod_info.packages { + let span = tracing::trace_span!("building package", name = pkg_info.name); + let _enter = span.enter(); + + tracing::trace!( + "Building package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; + let mut variant = BundleFileVariant::new(); + let bin = pkg + .to_binary() + .wrap_err("Failed to serialize package to binary")?; + variant.set_data(bin); + let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); + file.add_variant(variant); + + tracing::trace!( + "Compiled package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + mod_bundle.add_file(file); + + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = mod_dir.join(&bundle_name); + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + // Explicitely drop the guard, so that we can move the span + // into the async operation + drop(_enter); + + let ctx = state.ctx.clone(); + + let task = async move { + let bundle = { + let bin = fs::read(&src).await.wrap_err_with(|| { + format!("Failed to read bundle file '{}'", src.display()) + })?; + let name = Bundle::get_name_from_path(&ctx, &src); + Bundle::from_binary(&ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? + }; + + tracing::debug!( + src = %src.display(), + dest = %dest.display(), + "Copying bundle '{}' for mod '{}'", + pkg_name, + mod_name, + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + + Ok::(bundle) + } + .instrument(span); + + tasks.push(task); + } + } + + tracing::debug!("Copying {} mod bundles", tasks.len()); + + let mut tasks = stream::iter(tasks).buffer_unordered(10); + + while let Some(res) = tasks.next().await { + let bundle = res?; + bundles.push(bundle); + } + + { + let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); + tracing::trace!("Writing mod bundle to '{}'", path.display()); + fs::write(&path, mod_bundle.to_binary()?) + .await + .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; + } + + bundles.push(mod_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all)] +async fn patch_boot_bundle(state: Arc) -> Result> { + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); + + let mut bundles = Vec::with_capacity(2); + + let mut boot_bundle = async { + let bin = read_file_with_backup(&bundle_path) + .await + .wrap_err("Failed to read boot bundle")?; + + Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) + .wrap_err("Failed to parse boot bundle") + } + .instrument(tracing::trace_span!("read boot bundle")) + .await + .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; + + { + tracing::trace!("Adding mod package file to boot bundle"); + let span = tracing::trace_span!("create mod package file"); + let _enter = span.enter(); + + let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); + + for mod_info in &state.mods { + for pkg_info in &mod_info.packages { + pkg.add_file(BundleFileType::Package, &pkg_info.name); + } + } + + pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); + + let mut variant = BundleFileVariant::new(); + variant.set_data(pkg.to_binary()?); + let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + tracing::trace!("Handling DML packages and bundle"); + let span = tracing::trace_span!("handle DML"); + let _enter = span.enter(); + + let mut variant = BundleFileVariant::new(); + + let mod_info = state + .mods + .iter() + .find(|m| m.id == "dml") + .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; + let pkg_info = mod_info + .packages + .get(0) + .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) + .with_suggestion(|| "Re-download and import the newest version.".to_string())?; + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); + + { + let bin = fs::read(&src) + .await + .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; + let name = Bundle::get_name_from_path(&state.ctx, &src); + + let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; + + bundles.push(dml_bundle); + }; + + { + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + tracing::debug!( + "Copying bundle {} for mod {}: {} -> {}", + pkg_name, + mod_name, + src.display(), + dest.display() + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + } + + let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; + variant.set_data(pkg.to_binary()?); + + let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + let span = tracing::debug_span!("Importing mod main script"); + let _enter = span.enter(); + + let is_io_enabled = format!("{}", state.is_io_enabled); + let mut data = HashMap::new(); + data.insert("is_io_enabled", is_io_enabled.as_str()); + + let tmpl = include_str!("../../assets/mod_main.lua"); + let lua = Template::new(tmpl).render(&data); + tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); + let file = + lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; + + boot_bundle.add_file(file); + } + + async { + let bin = boot_bundle + .to_binary() + .wrap_err("Failed to serialize boot bundle")?; + fs::write(&bundle_path, bin) + .await + .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) + } + .instrument(tracing::trace_span!("write boot bundle")) + .await?; + + bundles.push(boot_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); + + let mut db = { + let bin = read_file_with_backup(&database_path) + .await + .wrap_err("Failed to read bundle database")?; + let mut r = Cursor::new(bin); + let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; + tracing::trace!("Finished parsing bundle database"); + db + }; + + for bundle in bundles.as_ref() { + tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); + db.add_bundle(bundle); + } + + { + let bin = db + .to_binary() + .wrap_err("Failed to serialize bundle database")?; + fs::write(&database_path, bin).await.wrap_err_with(|| { + format!( + "failed to write bundle database to '{}'", + database_path.display() + ) + })?; + } + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn write_deployment_data( + state: Arc, + bundles: B, + mod_folders: Vec, +) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let info = DeploymentData { + timestamp: OffsetDateTime::now_utc(), + bundles: bundles + .as_ref() + .iter() + .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) + .collect(), + // TODO: + mod_folders, + }; + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; + + fs::write(&path, &data) + .await + .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields( + game_dir = %state.game_dir.display(), + mods = state.mods.len() +))] +pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { + let state = Arc::new(state); + let bundle_dir = state.game_dir.join("bundle"); + let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); + + if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) + .await + .is_ok() + { + let err = eyre::eyre!("Found dtkit-patch-based mod installation."); + return Err(err) + .with_suggestion(|| { + "If you're a mod author and saved projects directly in 'mods/', \ + use DTMT to migrate them to the new project structure." + .to_string() + }) + .with_suggestion(|| { + "Click 'Reset Game' to remove the previous mod installation.".to_string() + }); + } + + let (_, game_info, deployment_info) = tokio::try_join!( + async { + fs::metadata(&bundle_dir) + .await + .wrap_err("Failed to open game bundle directory") + .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") + }, + async { + tokio::task::spawn_blocking(dtmt_shared::collect_game_info) + .await + .map_err(Report::new) + }, + async { + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + match read_sjson_file::<_, DeploymentData>(path).await { + Ok(data) => Ok(Some(data)), + Err(err) => { + if let Some(err) = err.downcast_ref::() + && err.kind() == ErrorKind::NotFound + { + Ok(None) + } else { + Err(err).wrap_err("Failed to read deployment data") + } + } + } + } + ) + .wrap_err("Failed to gather deployment information")?; + + tracing::debug!(?game_info, ?deployment_info); + + if let Some(game_info) = game_info { + if deployment_info + .as_ref() + .map(|i| game_info.last_updated > i.timestamp) + .unwrap_or(false) + { + tracing::warn!( + "Game was updated since last mod deployment. \ + Attempting to reconcile game files." + ); + + tokio::try_join!( + async { + let path = bundle_dir.join(BUNDLE_DATABASE_NAME); + let backup_path = path.with_extension("data.bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for bundle database.") + }, + async { + let path = bundle_dir.join(boot_bundle_path); + let backup_path = path.with_extension("bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for boot bundle") + } + ) + .with_suggestion(|| { + "Reset the game using 'Reset Game', then verify game files.".to_string() + })?; + + tracing::info!( + "Successfully re-created game file backups. \ + Continuing mod deployment." + ); + } + } + + check_mod_order(&state)?; + + tracing::info!( + "Deploying {} mods to '{}'.", + state.mods.iter().filter(|i| i.enabled).count(), + bundle_dir.display() + ); + + tracing::info!("Copy legacy mod folders"); + let mod_folders = copy_mod_folders(state.clone()) + .await + .wrap_err("Failed to copy mod folders")?; + + tracing::info!("Build mod bundles"); + let mut bundles = build_bundles(state.clone()) + .await + .wrap_err("Failed to build mod bundles")?; + + tracing::info!("Patch boot bundle"); + let mut boot_bundles = patch_boot_bundle(state.clone()) + .await + .wrap_err("Failed to patch boot bundle")?; + bundles.append(&mut boot_bundles); + + if let Some(info) = &deployment_info { + let bundle_dir = Arc::new(bundle_dir); + // Remove bundles from the previous deployment that don't match the current one. + // I.e. mods that used to be installed/enabled but aren't anymore. + { + let tasks = info.bundles.iter().cloned().filter_map(|file_name| { + let is_being_deployed = bundles.iter().any(|b2| { + let name = format!("{:016x}", b2.name()); + file_name == name + }); + + if !is_being_deployed { + let bundle_dir = bundle_dir.clone(); + let task = async move { + let path = bundle_dir.join(&file_name); + + tracing::debug!("Removing unused bundle '{}'", file_name); + + if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { + format!("Failed to remove unused bundle '{}'", path.display()) + }) { + tracing::error!("{:?}", err); + } + }; + Some(task) + } else { + None + } + }); + + futures::future::join_all(tasks).await; + } + + // Do the same thing for mod folders + { + let tasks = info.mod_folders.iter().filter_map(|mod_id| { + let is_being_deployed = mod_folders.iter().any(|id| id == mod_id); + + if !is_being_deployed { + let path = bundle_dir.join("mods").join(mod_id); + tracing::debug!("Removing unused mod folder '{}'", path.display()); + + let task = async move { + if let Err(err) = fs::remove_dir_all(&path).await.wrap_err_with(|| { + format!("Failed to remove unused legacy mod '{}'", path.display()) + }) { + tracing::error!("{:?}", err); + } + }; + + Some(task) + } else { + None + } + }); + futures::future::join_all(tasks).await; + } + } + + tracing::info!("Patch game settings"); + patch_game_settings(state.clone()) + .await + .wrap_err("Failed to patch game settings")?; + + tracing::info!("Patching bundle database"); + patch_bundle_database(state.clone(), &bundles) + .await + .wrap_err("Failed to patch bundle database")?; + + tracing::info!("Writing deployment data"); + write_deployment_data(state.clone(), &bundles, mod_folders) + .await + .wrap_err("Failed to write deployment data")?; + + tracing::info!("Finished deploying mods"); + Ok(()) +} diff --git a/crates/dtmm/src/controller/game.rs b/crates/dtmm/src/controller/game.rs index a5f6fa8..6b91169 100644 --- a/crates/dtmm/src/controller/game.rs +++ b/crates/dtmm/src/controller/game.rs @@ -1,46 +1,19 @@ -use std::collections::HashMap; -use std::io::{self, Cursor, ErrorKind}; +use std::io::{self, ErrorKind}; use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::Arc; use color_eyre::eyre::Context; -use color_eyre::{eyre, Help, Report, Result}; -use futures::stream; -use futures::StreamExt; -use path_slash::PathBufExt; -use sdk::filetype::lua; -use sdk::filetype::package::Package; +use color_eyre::{eyre, Result}; use sdk::murmur::Murmur64; -use sdk::{ - Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, -}; -use serde::{Deserialize, Serialize}; -use string_template::Template; -use time::OffsetDateTime; -use tokio::fs; +use tokio::fs::{self}; use tokio::io::AsyncWriteExt; -use tracing::Instrument; -use super::read_sjson_file; -use crate::controller::app::check_mod_order; -use crate::state::{ActionState, PackageInfo}; +use crate::controller::deploy::{ + DeploymentData, BOOT_BUNDLE_NAME, BUNDLE_DATABASE_NAME, DEPLOYMENT_DATA_PATH, +}; +use crate::state::ActionState; -const MOD_BUNDLE_NAME: &str = "packages/mods"; -const BOOT_BUNDLE_NAME: &str = "packages/boot"; -const DML_BUNDLE_NAME: &str = "packages/dml"; -const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; -const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; -const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; -const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; -const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; - -#[derive(Debug, Serialize, Deserialize)] -struct DeploymentData { - bundles: Vec, - #[serde(with = "time::serde::iso8601")] - timestamp: OffsetDateTime, -} +use super::deploy::SETTINGS_FILE_PATH; #[tracing::instrument] async fn read_file_with_backup

(path: P) -> Result> @@ -130,585 +103,6 @@ async fn patch_game_settings(state: Arc) -> Result<()> { Ok(()) } -#[tracing::instrument(skip_all, fields(package = info.name))] -fn make_package(info: &PackageInfo) -> Result { - let mut pkg = Package::new(info.name.clone(), PathBuf::new()); - - for f in &info.files { - let mut it = f.rsplit('.'); - let file_type = it - .next() - .ok_or_else(|| eyre::eyre!("missing file extension")) - .and_then(BundleFileType::from_str) - .wrap_err("Invalid file name in package info")?; - let name: String = it.collect(); - pkg.add_file(file_type, name); - } - - Ok(pkg) -} - -fn build_mod_data_lua(state: Arc) -> String { - let mut lua = String::from("return {\n"); - - // DMF is handled explicitely by the loading procedures, as it actually drives most of that - // and should therefore not show up in the load order. - for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { - lua.push_str(" {\n name = \""); - lua.push_str(&mod_info.name); - - lua.push_str("\",\n id = \""); - lua.push_str(&mod_info.id); - - lua.push_str("\",\n run = function()\n"); - - let resources = &mod_info.resources; - if resources.data.is_some() || resources.localization.is_some() { - lua.push_str(" new_mod(\""); - lua.push_str(&mod_info.id); - lua.push_str("\", {\n mod_script = \""); - lua.push_str(&resources.init.to_slash_lossy()); - - if let Some(data) = resources.data.as_ref() { - lua.push_str("\",\n mod_data = \""); - lua.push_str(&data.to_slash_lossy()); - } - - if let Some(localization) = &resources.localization { - lua.push_str("\",\n mod_localization = \""); - lua.push_str(&localization.to_slash_lossy()); - } - - lua.push_str("\",\n })\n"); - } else { - lua.push_str(" return dofile(\""); - lua.push_str(&resources.init.to_slash_lossy()); - lua.push_str("\")\n"); - } - - lua.push_str(" end,\n packages = {\n"); - - for pkg_info in &mod_info.packages { - lua.push_str(" \""); - lua.push_str(&pkg_info.name); - lua.push_str("\",\n"); - } - - lua.push_str(" },\n },\n"); - } - - lua.push('}'); - - tracing::debug!("mod_data_lua:\n{}", lua); - - lua -} - -#[tracing::instrument(skip_all)] -async fn build_bundles(state: Arc) -> Result> { - let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); - let mut tasks = Vec::new(); - - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - - let mut bundles = Vec::new(); - - { - tracing::trace!("Building mod data script"); - - let span = tracing::debug_span!("Building mod data script"); - let _enter = span.enter(); - - let lua = build_mod_data_lua(state.clone()); - - tracing::trace!("Compiling mod data script"); - - let file = - lua::compile(MOD_DATA_SCRIPT, &lua).wrap_err("Failed to compile mod data Lua file")?; - - tracing::trace!("Compile mod data script"); - - mod_bundle.add_file(file); - } - - tracing::trace!("Preparing tasks to deploy bundle files"); - - for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { - let span = tracing::trace_span!("building mod packages", name = mod_info.name); - let _enter = span.enter(); - - let mod_dir = state.mod_dir.join(&mod_info.id); - for pkg_info in &mod_info.packages { - let span = tracing::trace_span!("building package", name = pkg_info.name); - let _enter = span.enter(); - - tracing::trace!( - "Building package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; - let mut variant = BundleFileVariant::new(); - let bin = pkg - .to_binary() - .wrap_err("Failed to serialize package to binary")?; - variant.set_data(bin); - let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); - file.add_variant(variant); - - tracing::trace!( - "Compiled package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - mod_bundle.add_file(file); - - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = mod_dir.join(&bundle_name); - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - // Explicitely drop the guard, so that we can move the span - // into the async operation - drop(_enter); - - let ctx = state.ctx.clone(); - - let task = async move { - let bundle = { - let bin = fs::read(&src).await.wrap_err_with(|| { - format!("Failed to read bundle file '{}'", src.display()) - })?; - let name = Bundle::get_name_from_path(&ctx, &src); - Bundle::from_binary(&ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? - }; - - tracing::debug!( - src = %src.display(), - dest = %dest.display(), - "Copying bundle '{}' for mod '{}'", - pkg_name, - mod_name, - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - - Ok::(bundle) - } - .instrument(span); - - tasks.push(task); - } - } - - tracing::debug!("Copying {} mod bundles", tasks.len()); - - let mut tasks = stream::iter(tasks).buffer_unordered(10); - - while let Some(res) = tasks.next().await { - let bundle = res?; - bundles.push(bundle); - } - - { - let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); - tracing::trace!("Writing mod bundle to '{}'", path.display()); - fs::write(&path, mod_bundle.to_binary()?) - .await - .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; - } - - bundles.push(mod_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all)] -async fn patch_boot_bundle(state: Arc) -> Result> { - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); - - let mut bundles = Vec::with_capacity(2); - - let mut boot_bundle = async { - let bin = read_file_with_backup(&bundle_path) - .await - .wrap_err("Failed to read boot bundle")?; - - Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) - .wrap_err("Failed to parse boot bundle") - } - .instrument(tracing::trace_span!("read boot bundle")) - .await - .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; - - { - tracing::trace!("Adding mod package file to boot bundle"); - let span = tracing::trace_span!("create mod package file"); - let _enter = span.enter(); - - let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); - - for mod_info in &state.mods { - for pkg_info in &mod_info.packages { - pkg.add_file(BundleFileType::Package, &pkg_info.name); - } - } - - pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); - - let mut variant = BundleFileVariant::new(); - variant.set_data(pkg.to_binary()?); - let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - tracing::trace!("Handling DML packages and bundle"); - let span = tracing::trace_span!("handle DML"); - let _enter = span.enter(); - - let mut variant = BundleFileVariant::new(); - - let mod_info = state - .mods - .iter() - .find(|m| m.id == "dml") - .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; - let pkg_info = mod_info - .packages - .get(0) - .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) - .with_suggestion(|| "Re-download and import the newest version.".to_string())?; - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); - - { - let bin = fs::read(&src) - .await - .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; - let name = Bundle::get_name_from_path(&state.ctx, &src); - - let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; - - bundles.push(dml_bundle); - }; - - { - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - tracing::debug!( - "Copying bundle {} for mod {}: {} -> {}", - pkg_name, - mod_name, - src.display(), - dest.display() - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - } - - let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; - variant.set_data(pkg.to_binary()?); - - let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - let span = tracing::debug_span!("Importing mod main script"); - let _enter = span.enter(); - - let is_io_enabled = format!("{}", state.is_io_enabled); - let mut data = HashMap::new(); - data.insert("is_io_enabled", is_io_enabled.as_str()); - - let tmpl = include_str!("../../assets/mod_main.lua"); - let lua = Template::new(tmpl).render(&data); - tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); - let file = - lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; - - boot_bundle.add_file(file); - } - - async { - let bin = boot_bundle - .to_binary() - .wrap_err("Failed to serialize boot bundle")?; - fs::write(&bundle_path, bin) - .await - .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) - } - .instrument(tracing::trace_span!("write boot bundle")) - .await?; - - bundles.push(boot_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); - - let mut db = { - let bin = read_file_with_backup(&database_path) - .await - .wrap_err("Failed to read bundle database")?; - let mut r = Cursor::new(bin); - let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; - tracing::trace!("Finished parsing bundle database"); - db - }; - - for bundle in bundles.as_ref() { - tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); - db.add_bundle(bundle); - } - - { - let bin = db - .to_binary() - .wrap_err("Failed to serialize bundle database")?; - fs::write(&database_path, bin).await.wrap_err_with(|| { - format!( - "failed to write bundle database to '{}'", - database_path.display() - ) - })?; - } - - Ok(()) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn write_deployment_data(state: Arc, bundles: B) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let info = DeploymentData { - timestamp: OffsetDateTime::now_utc(), - bundles: bundles - .as_ref() - .iter() - .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) - .collect(), - }; - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; - - fs::write(&path, &data) - .await - .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; - - Ok(()) -} - -#[tracing::instrument(skip_all, fields( - game_dir = %state.game_dir.display(), - mods = state.mods.len() -))] -pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { - let state = Arc::new(state); - let bundle_dir = state.game_dir.join("bundle"); - let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); - - if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) - .await - .is_ok() - { - let err = eyre::eyre!("Found dtkit-patch-based mod installation."); - return Err(err) - .with_suggestion(|| { - "If you're a mod author and saved projects directly in 'mods/', \ - use DTMT to migrate them to the new project structure." - .to_string() - }) - .with_suggestion(|| { - "Click 'Reset Game' to remove the previous mod installation.".to_string() - }); - } - - let (_, game_info, deployment_info) = tokio::try_join!( - async { - fs::metadata(&bundle_dir) - .await - .wrap_err("Failed to open game bundle directory") - .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") - }, - async { - tokio::task::spawn_blocking(dtmt_shared::collect_game_info) - .await - .map_err(Report::new) - }, - async { - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - match read_sjson_file::<_, DeploymentData>(path) - .await - { - Ok(data) => Ok(Some(data)), - Err(err) => { - if let Some(err) = err.downcast_ref::() && err.kind() == ErrorKind::NotFound { - Ok(None) - } else { - Err(err).wrap_err("Failed to read deployment data") - } - } - } - } - ) - .wrap_err("Failed to gather deployment information")?; - - tracing::debug!(?game_info, ?deployment_info); - - if let Some(game_info) = game_info { - if deployment_info - .as_ref() - .map(|i| game_info.last_updated > i.timestamp) - .unwrap_or(false) - { - tracing::warn!( - "Game was updated since last mod deployment. \ - Attempting to reconcile game files." - ); - - tokio::try_join!( - async { - let path = bundle_dir.join(BUNDLE_DATABASE_NAME); - let backup_path = path.with_extension("data.bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for bundle database.") - }, - async { - let path = bundle_dir.join(boot_bundle_path); - let backup_path = path.with_extension("bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for boot bundle") - } - ) - .with_suggestion(|| { - "Reset the game using 'Reset Game', then verify game files.".to_string() - })?; - - tracing::info!( - "Successfully re-created game file backups. \ - Continuing mod deployment." - ); - } - } - - check_mod_order(&state)?; - - tracing::info!( - "Deploying {} mods to '{}'.", - state.mods.iter().filter(|i| i.enabled).count(), - bundle_dir.display() - ); - - tracing::info!("Build mod bundles"); - let mut bundles = build_bundles(state.clone()) - .await - .wrap_err("Failed to build mod bundles")?; - - tracing::info!("Patch boot bundle"); - let mut more_bundles = patch_boot_bundle(state.clone()) - .await - .wrap_err("Failed to patch boot bundle")?; - bundles.append(&mut more_bundles); - - if let Some(info) = &deployment_info { - let bundle_dir = Arc::new(bundle_dir); - let tasks = info.bundles.iter().cloned().filter_map(|file_name| { - let contains = bundles.iter().any(|b2| { - let name = format!("{:016x}", b2.name()); - file_name == name - }); - - if !contains { - let bundle_dir = bundle_dir.clone(); - let task = async move { - let path = bundle_dir.join(&file_name); - - tracing::debug!("Removing unused bundle '{}'", file_name); - - if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { - format!("Failed to remove unused bundle '{}'", path.display()) - }) { - tracing::error!("{:?}", err); - } - }; - Some(task) - } else { - None - } - }); - - futures::future::join_all(tasks).await; - } - - tracing::info!("Patch game settings"); - patch_game_settings(state.clone()) - .await - .wrap_err("Failed to patch game settings")?; - - tracing::info!("Patching bundle database"); - patch_bundle_database(state.clone(), &bundles) - .await - .wrap_err("Failed to patch bundle database")?; - - tracing::info!("Writing deployment data"); - write_deployment_data(state.clone(), &bundles) - .await - .wrap_err("Failed to write deployment data")?; - - tracing::info!("Finished deploying mods"); - Ok(()) -} - #[tracing::instrument(skip_all)] async fn reset_dtkit_patch(state: ActionState) -> Result<()> { let bundle_dir = state.game_dir.join("bundle"); diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs new file mode 100644 index 0000000..131d01f --- /dev/null +++ b/crates/dtmm/src/controller/import.rs @@ -0,0 +1,495 @@ +use std::collections::HashMap; +use std::ffi::CStr; +use std::io::{Cursor, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use color_eyre::eyre::{self, Context}; +use color_eyre::{Help, Report, Result}; +use druid::im::Vector; +use druid::{FileInfo, ImageBuf}; +use dtmt_shared::{ModConfig, ModConfigResources}; +use luajit2_sys as lua; +use nexusmods::Api as NexusApi; +use tokio::fs; +use zip::ZipArchive; + +use crate::state::{ActionState, ModInfo, NexusInfo, PackageInfo}; + +fn find_archive_file( + archive: &ZipArchive, + name: impl AsRef, +) -> Option { + let path = archive + .file_names() + .find(|path| path.ends_with(name.as_ref())) + .map(|s| s.to_string()); + path +} + +// Runs the content of a `.mod` file to extract what data we can get +// from legacy mods. +// 1. Create a global function `new_mod` that stores +// the relevant bits in global variables. +// 2. Run the `.mod` file, which will return a table. +// 3. Run the `run` function from that table. +// 4. Access the global variables from #1. +#[tracing::instrument] +fn parse_mod_id_file(data: &str) -> Result<(String, ModConfigResources)> { + tracing::debug!("Parsing mod file:\n{}", data); + + let ret = unsafe { + let state = lua::luaL_newstate(); + lua::luaL_openlibs(state); + + let run = b" +function fassert() end +function new_mod(id, resources) + _G.id = id + _G.script = resources.mod_script + _G.data = resources.mod_data + _G.localization = resources.mod_localization +end +\0"; + match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRSYNTAX => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Invalid syntax: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to create `new_mod`") + } + _ => unreachable!(), + } + + match lua::lua_pcall(state, 0, 0, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run buffer: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run buffer") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let name = b".mod\0"; + match lua::luaL_loadbuffer( + state, + data.as_ptr() as _, + data.len() as _, + name.as_ptr() as _, + ) as u32 + { + lua::LUA_OK => {} + lua::LUA_ERRSYNTAX => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Invalid syntax: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to load `.mod` file buffer") + } + _ => unreachable!(), + } + + match lua::lua_pcall(state, 0, 1, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run `.mod` file: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run `.mod` file") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let key = b"run\0"; + lua::lua_pushstring(state, key.as_ptr() as _); + lua::lua_gettable(state, -2); + + match lua::lua_pcall(state, 0, 0, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run `.mod.run`: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run `.mod.run`") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let get_global = |state, key: &[u8]| { + lua::lua_getglobal(state, key.as_ptr() as _); + + if lua::lua_isnil(state, -1) != 0 { + return Ok(None); + } + + let s = lua::lua_tostring(state, -1); + + if s.is_null() { + eyre::bail!("Expected string, got NULL"); + } + + let ret = CStr::from_ptr(s).to_string_lossy().to_string(); + lua::lua_pop(state, 1); + Ok(Some(ret)) + }; + + let mod_id = get_global(state, b"id\0") + .and_then(|s| s.ok_or_else(|| eyre::eyre!("Got `nil`"))) + .wrap_err("Failed to get `id`")?; + + let resources = ModConfigResources { + init: get_global(state, b"script\0") + .and_then(|s| s.map(PathBuf::from).ok_or_else(|| eyre::eyre!("Got `nil`"))) + .wrap_err("Failed to get `script`.")?, + data: get_global(state, b"data\0") + .wrap_err("Failed to get `data`.")? + .map(PathBuf::from), + localization: get_global(state, b"localization\0") + .wrap_err("Failed to get `localization`")? + .map(PathBuf::from), + }; + + lua::lua_close(state); + + (mod_id, resources) + }; + + Ok(ret) +} + +// Extracts the mod configuration from the mod archive. +// This may either be a proper `dtmt.cfg`, or the legacy `.mod` ID file. +// +// It also returns the directory where this file was found, used as root path. This +// allows flexibility in what the directory structure is exactly, since many people +// still end up creating tarbombs and Nexus does its own re-packaging. +#[tracing::instrument(skip(archive))] +fn extract_mod_config(archive: &mut ZipArchive) -> Result<(ModConfig, String)> { + let legacy_mod_data = if let Some(name) = find_archive_file(archive, ".mod") { + let (mod_id, resources) = { + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read `.mod` file from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read `.mod` file from archive")?; + + let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?; + parse_mod_id_file(&data) + .wrap_err("Invalid `.mod` file") + .note( + "The `.mod` file's `run` function may not contain any additional logic \ + besides the default.", + ) + .suggestion("Contact the mod author to fix this.")? + }; + + let root = if let Some(index) = name.rfind('/') { + name[..index].to_string() + } else { + String::new() + }; + + Some((mod_id, resources, root)) + } else { + None + }; + + if let Some(name) = find_archive_file(archive, "dtmt.cfg") { + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read mod config from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read mod config from archive")?; + + let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; + + let mut cfg: ModConfig = serde_sjson::from_str(&data) + .wrap_err("Failed to deserialize mod config") + .suggestion("Contact the mod author to fix this.")?; + + if let Some((mod_id, resources, root)) = legacy_mod_data { + if cfg.id != mod_id { + let err = eyre::eyre!("Mod ID in `dtmt.cfg` does not match mod ID in `.mod` file"); + return Err(err).suggestion("Contact the mod author to fix this."); + } + + cfg.resources = resources; + + Ok((cfg, root)) + } else { + let root = name + .strip_suffix("dtmt.cfg") + .expect("String must end with that suffix") + .to_string(); + + Ok((cfg, root)) + } + } else { + eyre::bail!( + "Mod needs a config file or `.mod` file. \ + Please get in touch with the author to provide a properly packaged mod." + ); + } +} + +#[tracing::instrument(skip(archive))] +fn extract_bundled_mod( + archive: &mut ZipArchive, + root: String, + dest: impl AsRef + std::fmt::Debug, +) -> Result>> { + let files: HashMap> = { + let name = archive + .file_names() + .find(|name| name.ends_with("files.sjson")) + .map(|s| s.to_string()) + .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; + + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read file index from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; + serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? + }; + + tracing::trace!(?files); + + let dest = dest.as_ref(); + tracing::trace!("Extracting mod archive to {}", dest.display()); + archive + .extract(dest) + .wrap_err_with(|| format!("Failed to extract archive to {}", dest.display()))?; + + let packages = files + .into_iter() + .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) + .collect(); + + tracing::trace!(?packages); + + Ok(packages) +} + +#[tracing::instrument(skip(archive))] +fn extract_legacy_mod( + archive: &mut ZipArchive, + root: String, + dest: impl Into + std::fmt::Debug, +) -> Result<()> { + let dest = dest.into(); + let file_count = archive.len(); + + for i in 0..file_count { + let mut f = archive + .by_index(i) + .wrap_err_with(|| format!("Failed to get file at index {}", i))?; + + let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else { + let err = eyre::eyre!("File name in archive is not a safe path value.").suggestion( + "Only use well-known applications to create the ZIP archive, \ + and don't create paths that point outside the archive directory.", + ); + return Err(err); + }; + + let Ok(suffix) = name.strip_prefix(&root) else { + tracing::warn!( + "Skipping file outside of the mod root directory: {}", + name.display() + ); + continue; + }; + let name = dest.join(suffix); + + if f.is_dir() { + // The majority of errors will actually be "X already exists". + // But rather than filter them invidually, we just ignore all of them. + // If there is a legitimate error of "couldn't create X", it will eventually fail when + // we try to put a file in there. + tracing::trace!("Creating directory '{}'", name.display()); + let _ = std::fs::create_dir_all(&name); + } else { + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err_with(|| format!("Failed to read file '{}'", name.display()))?; + + tracing::trace!("Writing file '{}'", name.display()); + let mut out = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&name) + .wrap_err_with(|| format!("Failed to open file '{}'", name.display()))?; + + out.write_all(&buf) + .wrap_err_with(|| format!("Failed to write to '{}'", name.display()))?; + } + } + + Ok(()) +} + +#[tracing::instrument(skip(state))] +pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { + let data = fs::read(&info.path) + .await + .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; + let data = Cursor::new(data); + + let nexus = if let Some((_, id, version, _)) = info + .path + .file_name() + .and_then(|s| s.to_str()) + .and_then(NexusApi::parse_file_name) + { + if !state.nexus_api_key.is_empty() { + let api = NexusApi::new(state.nexus_api_key.to_string())?; + let mod_info = api + .mods_id(id) + .await + .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; + let info = NexusInfo::from(mod_info); + + tracing::debug!("{:?}", info); + Some((info, version)) + } else { + None + } + } else { + None + }; + + let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; + + if tracing::enabled!(tracing::Level::DEBUG) { + let names = archive.file_names().fold(String::new(), |mut s, name| { + s.push('\n'); + s.push_str(name); + s + }); + tracing::debug!("Archive contents:{}", names); + } + + let (mut mod_cfg, root) = + extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?; + tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id); + tracing::debug!(root, ?mod_cfg); + + let image = if let Some(path) = &mod_cfg.image { + let name = archive + .file_names() + .find(|name| name.ends_with(&path.display().to_string())) + .map(|s| s.to_string()) + .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; + + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read image file from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + // Druid somehow doesn't return an error compatible with eyre, here. + // So we have to wrap through `Display` manually. + let img = match ImageBuf::from_data(&buf) { + Ok(img) => img, + Err(err) => { + let err = Report::msg(err.to_string()) + .wrap_err("Invalid image data") + .note("Supported formats are: PNG, JPEG, Bitmap and WebP") + .suggestion("Contact the mod author to fix this"); + return Err(err); + } + }; + + Some(img) + } else { + None + }; + + tracing::trace!(?image); + + let mod_dir = state.data_dir.join(state.mod_dir.as_ref()); + let dest = mod_dir.join(&mod_cfg.id); + + tracing::trace!("Creating mods directory {}", dest.display()); + fs::create_dir_all(&dest) + .await + .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?; + + let packages = if mod_cfg.bundled { + extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")? + } else { + extract_legacy_mod(&mut archive, root, &dest).wrap_err("Failed to extract legacy mod")?; + + if let Some((_, version)) = &nexus { + // We use the version number stored in the `ModInfo` to compare against the `NexusInfo` + // for version checks. So for this one, we can't actually rely on merely shadowing, + // like with the other fields. + mod_cfg.version = version.clone(); + } + + let data = serde_sjson::to_string(&mod_cfg).wrap_err("Failed to serialize mod config")?; + fs::write(dest.join("dtmt.cfg"), &data) + .await + .wrap_err("Failed to write mod config")?; + + Default::default() + }; + + if let Some((nexus, _)) = &nexus { + let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; + let path = dest.join("nexus.sjson"); + fs::write(&path, data.as_bytes()) + .await + .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; + } + + let info = ModInfo::new(mod_cfg, packages, image, nexus.map(|(info, _)| info)); + Ok(info) +} diff --git a/crates/dtmm/src/controller/mod.rs b/crates/dtmm/src/controller/mod.rs index eacc3ef..9c75e84 100644 --- a/crates/dtmm/src/controller/mod.rs +++ b/crates/dtmm/src/controller/mod.rs @@ -5,7 +5,9 @@ use serde::Deserialize; use tokio::fs; pub mod app; +pub mod deploy; pub mod game; +pub mod import; pub mod worker; #[tracing::instrument] diff --git a/crates/dtmm/src/controller/worker.rs b/crates/dtmm/src/controller/worker.rs index 238b3f3..152030f 100644 --- a/crates/dtmm/src/controller/worker.rs +++ b/crates/dtmm/src/controller/worker.rs @@ -13,7 +13,9 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::RwLock; use crate::controller::app::*; +use crate::controller::deploy::deploy_mods; use crate::controller::game::*; +use crate::controller::import::import_mod; use crate::state::AsyncAction; use crate::state::ACTION_FINISH_CHECK_UPDATE; use crate::state::ACTION_FINISH_LOAD_INITIAL; @@ -36,7 +38,9 @@ async fn handle_action( action_queue: Arc>>, ) { while let Some(action) = action_queue.write().await.recv().await { - tracing::debug!(?action); + if cfg!(debug_assertions) && !matches!(action, AsyncAction::Log(_)) { + tracing::debug!(?action); + } let event_sink = event_sink.clone(); match action { diff --git a/crates/dtmm/src/main.rs b/crates/dtmm/src/main.rs index 9ba838c..8069e6d 100644 --- a/crates/dtmm/src/main.rs +++ b/crates/dtmm/src/main.rs @@ -1,6 +1,7 @@ #![recursion_limit = "256"] #![feature(let_chains)] #![feature(arc_unwrap_or_clone)] +#![feature(iterator_try_collect)] #![windows_subsystem = "windows"] use std::path::PathBuf; diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index 55774aa..64fdd28 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -73,6 +73,7 @@ impl From for ModDependency { #[derive(Clone, Data, Debug, Lens, serde::Serialize, serde::Deserialize)] pub(crate) struct NexusInfo { pub id: u64, + pub name: String, pub version: String, pub author: String, pub summary: Arc, @@ -83,6 +84,7 @@ impl From for NexusInfo { fn from(value: NexusMod) -> Self { Self { id: value.mod_id, + name: value.name, version: value.version, author: value.author, summary: Arc::new(value.summary), @@ -109,6 +111,7 @@ pub(crate) struct ModInfo { #[data(ignore)] pub resources: ModResourceInfo, pub depends: Vector, + pub bundled: bool, #[data(ignore)] pub nexus: Option, } @@ -129,6 +132,7 @@ impl ModInfo { version: cfg.version, enabled: false, packages, + bundled: cfg.bundled, image, categories: cfg.categories.into_iter().collect(), resources: ModResourceInfo { diff --git a/crates/dtmm/src/state/delegate.rs b/crates/dtmm/src/state/delegate.rs index 97c6e6b..47d56e8 100644 --- a/crates/dtmm/src/state/delegate.rs +++ b/crates/dtmm/src/state/delegate.rs @@ -411,6 +411,7 @@ impl AppDelegate for Delegate { state.config_path = Arc::new(config.path); state.data_dir = Arc::new(config.data_dir); state.game_dir = Arc::new(config.game_dir.unwrap_or_default()); + state.nexus_api_key = Arc::new(config.nexus_api_key.unwrap_or_default()); state.is_io_enabled = config.unsafe_io; } diff --git a/crates/dtmm/src/ui/window/main.rs b/crates/dtmm/src/ui/window/main.rs index 7aa2819..baa8e22 100644 --- a/crates/dtmm/src/ui/window/main.rs +++ b/crates/dtmm/src/ui/window/main.rs @@ -135,8 +135,13 @@ fn build_mod_list() -> impl Widget { }) .lens(lens!((usize, Arc, bool), 1).then(ModInfo::enabled.in_arc())); - let name = - Label::raw().lens(lens!((usize, Arc, bool), 1).then(ModInfo::name.in_arc())); + let name = Label::dynamic(|info: &Arc, _| { + info.nexus + .as_ref() + .map(|n| n.name.clone()) + .unwrap_or_else(|| info.name.clone()) + }) + .lens(lens!((usize, Arc, bool), 1)); let version = { let icon = { @@ -145,7 +150,7 @@ fn build_mod_list() -> impl Widget { let tree = theme::icons::recolor_icon(tree, true, COLOR_YELLOW_LIGHT); - Svg::new(Arc::new(tree)).fix_height(druid::theme::TEXT_SIZE_NORMAL) + Svg::new(tree).fix_height(druid::theme::TEXT_SIZE_NORMAL) }; Either::new( diff --git a/crates/dtmm/src/util/config.rs b/crates/dtmm/src/util/config.rs index 3a0d0b2..9affbb6 100644 --- a/crates/dtmm/src/util/config.rs +++ b/crates/dtmm/src/util/config.rs @@ -125,6 +125,9 @@ where .wrap_err_with(|| format!("Invalid config file {}", path.display()))?; cfg.path = path; + + tracing::debug!("Read config file '{}': {:?}", cfg.path.display(), cfg); + Ok(cfg) } Err(err) if err.kind() == ErrorKind::NotFound => { @@ -133,6 +136,11 @@ where .wrap_err_with(|| format!("Failed to read config file {}", path.display()))?; } + tracing::debug!( + "Config file not found at '{}', creating default.", + path.display() + ); + { let parent = default_path .parent() diff --git a/crates/dtmt/src/cmd/dictionary.rs b/crates/dtmt/src/cmd/dictionary.rs index 0400519..62a5295 100644 --- a/crates/dtmt/src/cmd/dictionary.rs +++ b/crates/dtmt/src/cmd/dictionary.rs @@ -145,7 +145,10 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( .get_one::("group") .expect("required argument not found"); - let r: BufReader> = if let Some(name) = path.file_name() && name == "-" { + let r: BufReader> = if let Some(name) = + path.file_name() + && name == "-" + { let f = tokio::io::stdin(); BufReader::new(Box::new(f)) } else { diff --git a/crates/dtmt/src/cmd/migrate.rs b/crates/dtmt/src/cmd/migrate.rs index 1a4d605..2eacded 100644 --- a/crates/dtmt/src/cmd/migrate.rs +++ b/crates/dtmt/src/cmd/migrate.rs @@ -350,6 +350,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> localization: mod_file.localization, }, depends: vec![ModDependency::ID(String::from("DMF"))], + bundled: true, }; tracing::debug!(?dtmt_cfg); diff --git a/lib/dtmt-shared/src/lib.rs b/lib/dtmt-shared/src/lib.rs index 3907928..28e4694 100644 --- a/lib/dtmt-shared/src/lib.rs +++ b/lib/dtmt-shared/src/lib.rs @@ -30,6 +30,17 @@ pub enum ModDependency { Config { id: String, order: ModOrder }, } +// A bit dumb, but serde doesn't support literal values with the +// `default` attribute, only paths. +fn default_true() -> bool { + true +} + +// Similarly dumb, as the `skip_serializing_if` attribute needs a function +fn is_true(val: &bool) -> bool { + *val +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ModConfig { #[serde(skip)] @@ -51,6 +62,8 @@ pub struct ModConfig { pub resources: ModConfigResources, #[serde(default)] pub depends: Vec, + #[serde(default = "default_true", skip_serializing_if = "is_true")] + pub bundled: bool, } pub const STEAMAPP_ID: u32 = 1361210; diff --git a/lib/nexusmods/src/lib.rs b/lib/nexusmods/src/lib.rs index a0700ae..145435d 100644 --- a/lib/nexusmods/src/lib.rs +++ b/lib/nexusmods/src/lib.rs @@ -112,7 +112,7 @@ impl Api { RE.captures(name.as_ref()).and_then(|cap| { let name = cap.name("name").map(|s| s.as_str().to_string())?; let mod_id = cap.name("mod_id").and_then(|s| s.as_str().parse().ok())?; - let version = cap.name("version").map(|s| s.as_str().to_string())?; + let version = cap.name("version").map(|s| s.as_str().replace('-', "."))?; let updated = cap .name("updated") .and_then(|s| s.as_str().parse().ok()) @@ -154,7 +154,7 @@ impl Api { self.mods_download_link(nxm.mod_id, nxm.file_id, nxm.key, nxm.expires) )?; - let Some(download_url) = download_info.get(0).map(|i| i.uri.clone()) else { + let Some(download_url) = download_info.first().map(|i| i.uri.clone()) else { return Err(Error::InvalidNXM("no download link", url)); }; @@ -215,7 +215,7 @@ impl Api { }; let user_id = query.get("user_id").and_then(|id| id.parse().ok()); - let Some(user_id) = user_id else { + let Some(user_id) = user_id else { return Err(Error::InvalidNXM("Missing 'user_id'", nxm)); }; diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 96a2ec2..813df06 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -227,13 +227,10 @@ impl Bundle { let _enter = span.enter(); tracing::trace!(num_files = self.files.len()); - self.files - .iter() - .fold(Ok::, Report>(Vec::new()), |data, file| { - let mut data = data?; - data.append(&mut file.to_binary()?); - Ok(data) - })? + self.files.iter().try_fold(Vec::new(), |mut data, file| { + data.append(&mut file.to_binary()?); + Ok::<_, Report>(data) + })? }; // Ceiling division (or division toward infinity) to calculate diff --git a/lib/sdk/src/filetype/lua.rs b/lib/sdk/src/filetype/lua.rs index 2458dec..bfe6de7 100644 --- a/lib/sdk/src/filetype/lua.rs +++ b/lib/sdk/src/filetype/lua.rs @@ -53,8 +53,8 @@ where let mut buf = vec![0u8; length]; r.read_exact(&mut buf)?; - let mut s = String::from_utf8(buf) - .wrap_err_with(|| format!("Invalid byte sequence for LuaJIT bytecode name"))?; + let mut s = + String::from_utf8(buf).wrap_err("Invalid byte sequence for LuaJIT bytecode name")?; // Remove the leading `@` s.remove(0); s