diff --git a/Cargo.lock b/Cargo.lock index 8385e67..a7885f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,10 +39,12 @@ dependencies = [ [[package]] name = "ansi-parser" -version = "0.9.0" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127" dependencies = [ "heapless", - "nom", + "nom 4.2.3", ] [[package]] @@ -379,7 +381,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -894,7 +896,6 @@ name = "dtmm" version = "0.1.0" dependencies = [ "ansi-parser", - "async-recursion", "bitflags 1.3.2", "clap", "color-eyre", @@ -905,8 +906,6 @@ dependencies = [ "dtmt-shared", "futures", "lazy_static", - "luajit2-sys", - "minijinja", "nexusmods", "oodle", "path-slash", @@ -1391,7 +1390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", - "version_check", + "version_check 0.9.4", ] [[package]] @@ -1615,12 +1614,12 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heapless" -version = "0.6.1" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422" +checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1" dependencies = [ "as-slice", - "generic-array 0.14.7", + "generic-array 0.13.3", "hash32", "stable_deref_trait", ] @@ -1748,7 +1747,7 @@ dependencies = [ "serde", "sized-chunks", "typenum", - "version_check", + "version_check 0.9.4", ] [[package]] @@ -2085,15 +2084,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minijinja" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" -dependencies = [ - "serde", -] - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2185,6 +2175,16 @@ dependencies = [ "memoffset 0.6.5", ] +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check 0.1.5", +] + [[package]] name = "nom" version = "7.1.3" @@ -2203,7 +2203,7 @@ checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3" dependencies = [ "bytecount", "memchr", - "nom", + "nom 7.1.3", ] [[package]] @@ -2673,7 +2673,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "version_check", + "version_check 0.9.4", ] [[package]] @@ -2684,7 +2684,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "version_check", + "version_check 0.9.4", ] [[package]] @@ -3102,7 +3102,7 @@ dependencies = [ name = "serde_sjson" version = "1.0.0" dependencies = [ - "nom", + "nom 7.1.3", "nom_locate", "serde", ] @@ -3832,7 +3832,7 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ - "version_check", + "version_check 0.9.4", ] [[package]] @@ -3966,6 +3966,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + [[package]] name = "version_check" version = "0.9.4" @@ -4378,3 +4384,7 @@ dependencies = [ "cc", "pkg-config", ] + +[[patch.unused]] +name = "ansi-parser" +version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 451cd09..123b2ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,10 @@ members = [ "lib/oodle", "lib/sdk", "lib/serde_sjson", - "lib/luajit2-sys", +] +exclude = [ + "lib/color-eyre", + "lib/ansi-parser", ] [patch.crates-io] diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 5f60220..86444be 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -31,8 +31,5 @@ lazy_static = "1.4.0" colors-transform = "0.2.11" usvg = "0.25.0" druid-widget-nursery = "0.1" -ansi-parser = "0.9.0" +ansi-parser = "0.8.0" string_template = "0.2.1" -luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } -async-recursion = "1.0.5" -minijinja = "1.0.10" diff --git a/crates/dtmm/assets/mod_data.lua.j2 b/crates/dtmm/assets/mod_data.lua.j2 deleted file mode 100644 index 9f87ad1..0000000 --- a/crates/dtmm/assets/mod_data.lua.j2 +++ /dev/null @@ -1,27 +0,0 @@ -return { -{% for mod in mods %} -{ - id = "{{ mod.id }}", - name = "{{ mod.name }}", - bundled = {{ mod.bundled }}, - packages = { - {% for pkg in mod.packages %} - "{{ pkg }}", - {% endfor %} - }, - run = function() - {% if mod.data is none %} - return dofile("{{ mod.init }}") - {% else %} - new_mod("{{ mod.id }}", { - mod_script = "{{ mod.init }}", - mod_data = "{{ mod.data }}", - {% if not mod.localization is none %} - mod_localization = "{{ mod.localization }}", - {% endif %} - }) - {% endif %} - end, -}, -{% endfor %} -} diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua index e4006f6..6dbf3e2 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua @@ -11,6 +11,100 @@ local log = function(category, format, ...) end end +-- Patch `GameStateMachine.init` to add our own state for loading mods. +-- In the future, Fatshark might provide us with a dedicated way to do this. +local function patch_mod_loading_state() + local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") + + -- A necessary override. + -- The original does not proxy `dt` to `_state_update`, but we need that. + StateBootSubStateBase.update = function(self, dt) + local done, error = self:_state_update(dt) + local params = self._params + + if error then + return StateError, { error } + elseif done then + local next_index = params.sub_state_index + 1 + params.sub_state_index = next_index + local next_state_data = params.states[next_index] + + if next_state_data then + return next_state_data[1], self._params + else + self._parent:sub_states_done() + end + end + end + + local StateBootLoadMods = class("StateBootLoadMods", "StateBootSubStateBase") + + StateBootLoadMods.on_enter = function(self, parent, params) + log("StateBootLoadMods", "Entered") + StateBootLoadMods.super.on_enter(self, parent, params) + + local state_params = self:_state_params() + local package_manager = state_params.package_manager + + self._state = "load_package" + self._package_manager = package_manager + self._package_handles = { + ["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadMods", nil), + ["packages/dml"] = package_manager:load("packages/dml", "StateBootLoadMods", nil), + } + end + + StateBootLoadMods._state_update = function(self, dt) + local state = self._state + local package_manager = self._package_manager + + if state == "load_package" and package_manager:update() then + log("StateBootLoadMods", "Packages loaded, loading mods") + self._state = "load_mods" + local ModLoader = require("scripts/mods/dml/init") + + local mod_data = require("scripts/mods/mod_data") + local mod_loader = ModLoader:new(mod_data, self._parent:gui()) + + self._mod_loader = mod_loader + Managers.mod = mod_loader + elseif state == "load_mods" and self._mod_loader:update(dt) then + log("StateBootLoadMods", "Mods loaded, exiting") + return true, false + end + + return false, false + end + + local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") + + local patched = false + + local GameStateMachine_init = GameStateMachine.init + GameStateMachine.init = function(self, parent, start_state, params, ...) + if not patched then + log("mod_main", "Injecting mod loading state") + patched = true + + -- Hardcoded position after `StateRequireScripts`. + -- We do want to wait until then, so that most of the game's core + -- systems are at least loaded and can be hooked, even if they aren't + -- running, yet. + local pos = 4 + table.insert(params.states, pos, { + StateBootLoadMods, + { + package_manager = params.package_manager, + }, + }) + end + + GameStateMachine_init(self, parent, start_state, params, ...) + end + + log("mod_main", "Mod patching complete") +end + log("mod_main", "Initializing mods...") local require_store = {} @@ -18,7 +112,7 @@ local require_store = {} -- This token is treated as a string template and filled by DTMM during deployment. -- This allows hiding unsafe I/O functions behind a setting. -- It's also a valid table definition, thereby degrading gracefully when not replaced. -local is_io_enabled = {{ is_io_enabled }} -- luacheck: ignore 113 +local is_io_enabled = { { is_io_enabled } } -- luacheck: ignore 113 local lua_libs = { debug = debug, os = { @@ -44,8 +138,7 @@ Mods = { -- Fatshark's code scrubs them. -- The loader can then decide to pass them on to mods, or ignore them lua = setmetatable({}, { __index = lua_libs }), - require_store = require_store, - original_require = require, + require_store = require_store } local can_insert = function(filepath, new_result) @@ -105,98 +198,6 @@ end require("scripts/main") log("mod_main", "'scripts/main' loaded") --- Inject our state into the game. The state needs to run after `StateGame._init_managers`, --- since some parts of DMF, and presumably other mods, depend on some of those managers to exist. -local function patch_mod_loading_state() - local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") - local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase") - local StateGameLoadMods = class("StateGameLoadMods") - - StateBootLoadDML.on_enter = function(self, parent, params) - log("StateBootLoadDML", "Entered") - StateBootLoadDML.super.on_enter(self, parent, params) - - local state_params = self:_state_params() - local package_manager = state_params.package_manager - - self._package_manager = package_manager - self._package_handles = { - ["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil), - ["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil), - } - end - - StateBootLoadDML._state_update = function(self, dt) - local package_manager = self._package_manager - - if package_manager:update() then - local DML = require("scripts/mods/dml/init") - local mod_data = require("scripts/mods/mod_data") - local mod_loader = DML.create_loader(mod_data) - Managers.mod = mod_loader - log("StateBootLoadDML", "DML loaded, exiting") - return true, false - end - - return false, false - end - - - function StateGameLoadMods:on_enter(_, params) - log("StateGameLoadMods", "Entered") - self._next_state = require("scripts/game_states/game/state_splash") - self._next_state_params = params - end - - function StateGameLoadMods:update(main_dt) - local state = self._loading_state - - -- We're relying on the fact that DML internally makes sure - -- that `Managers.mod:update()` is being called appropriately. - -- The implementation as of this writing is to hook `StateGame.update`. - if Managers.mod:all_mods_loaded() then - Log.info("StateGameLoadMods", "Mods loaded, exiting") - return self._next_state, self._next_state_params - end - end - - local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") - local GameStateMachine_init = GameStateMachine.init - GameStateMachine.init = function(self, parent, start_state, params, creation_context, state_change_callbacks, name) - if name == "Main" then - log("mod_main", "Injecting StateBootLoadDML") - - -- Hardcoded position after `StateRequireScripts`. - -- We need to wait until then to even begin most of our stuff, - -- so that most of the game's core systems are at least loaded and can be hooked, - -- even if they aren't running, yet. - local pos = 4 - table.insert(params.states, pos, { - StateBootLoadDML, - { - package_manager = params.package_manager, - }, - }) - - GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) - elseif name == "Game" then - log("mod_main", "Injection StateGameLoadMods") - -- The second time around, we want to be the first, so we pass our own - -- 'start_state'. - -- We can't just have the state machine be initialized and then change its `_next_state`, as by the end of - -- `init`, a bunch of stuff will already be initialized. - GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name) - -- And since we're done now, we can revert the function to its original - GameStateMachine.init = GameStateMachine_init - - return - else - -- In all other cases, simply call the original - GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) - end - end -end - -- Override `init` to run our injection function init() patch_mod_loading_state() diff --git a/crates/dtmm/src/controller/app.rs b/crates/dtmm/src/controller/app.rs index cb6e2f0..5c5d980 100644 --- a/crates/dtmm/src/controller/app.rs +++ b/crates/dtmm/src/controller/app.rs @@ -1,17 +1,18 @@ use std::collections::HashMap; -use std::io::ErrorKind; +use std::io::{Cursor, ErrorKind, Read}; use std::path::{Path, PathBuf}; use std::sync::Arc; use color_eyre::eyre::{self, Context}; use color_eyre::{Help, Report, Result}; use druid::im::Vector; -use druid::ImageBuf; +use druid::{FileInfo, ImageBuf}; use dtmt_shared::ModConfig; use nexusmods::Api as NexusApi; use tokio::fs::{self, DirEntry, File}; use tokio_stream::wrappers::ReadDirStream; use tokio_stream::StreamExt; +use zip::ZipArchive; use crate::state::{ActionState, InitialLoadResult, ModInfo, ModOrder, NexusInfo, PackageInfo}; use crate::util; @@ -19,6 +20,161 @@ use crate::util::config::{ConfigSerialize, LoadOrderEntry}; use super::read_sjson_file; +#[tracing::instrument(skip(state))] +pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { + let data = fs::read(&info.path) + .await + .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; + let data = Cursor::new(data); + + let nexus = if let Some((_, id, _, _)) = info + .path + .file_name() + .and_then(|s| s.to_str()) + .and_then(NexusApi::parse_file_name) + { + if !state.nexus_api_key.is_empty() { + let api = NexusApi::new(state.nexus_api_key.to_string())?; + let mod_info = api + .mods_id(id) + .await + .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; + Some(NexusInfo::from(mod_info)) + } else { + None + } + } else { + None + }; + + let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; + + if tracing::enabled!(tracing::Level::DEBUG) { + let names = archive.file_names().fold(String::new(), |mut s, name| { + s.push('\n'); + s.push_str(name); + s + }); + tracing::debug!("Archive contents:{}", names); + } + + let dir_name = { + let f = archive.by_index(0).wrap_err("Archive is empty")?; + + if !f.is_dir() { + let err = eyre::eyre!("archive does not have a top-level directory"); + return Err(err).with_suggestion(|| "Use 'dtmt build' to create the mod archive."); + } + + let name = f.name(); + // The directory name is returned with a trailing slash, which we don't want + name[..(name.len().saturating_sub(1))].to_string() + }; + + tracing::info!("Importing mod {}", dir_name); + + let names: Vec<_> = archive.file_names().map(|s| s.to_string()).collect(); + + let mod_cfg: ModConfig = { + let name = names + .iter() + .find(|name| name.ends_with("dtmt.cfg")) + .ok_or_else(|| eyre::eyre!("archive does not contain mod config"))?; + + let mut f = archive + .by_name(name) + .wrap_err("Failed to read mod config from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read mod config from archive")?; + + let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; + + serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")? + }; + + tracing::debug!(?mod_cfg); + + let files: HashMap> = { + let name = names + .iter() + .find(|name| name.ends_with("files.sjson")) + .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; + + let mut f = archive + .by_name(name) + .wrap_err("Failed to read file index from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; + + serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? + }; + + tracing::trace!(?files); + + let image = if let Some(path) = &mod_cfg.image { + let name = names + .iter() + .find(|name| name.ends_with(&path.display().to_string())) + .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; + + let mut f = archive + .by_name(name) + .wrap_err("Failed to read image file from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + // Druid somehow doesn't return an error compatible with eyre, here. + // So we have to wrap through `Display` manually. + let img = match ImageBuf::from_data(&buf) { + Ok(img) => img, + Err(err) => { + let err = Report::msg(err.to_string()).wrap_err("Invalid image data"); + return Err(err).with_suggestion(|| { + "Supported formats are: PNG, JPEG, Bitmap and WebP".to_string() + }); + } + }; + + Some(img) + } else { + None + }; + + let mod_dir = state.mod_dir; + + tracing::trace!("Creating mods directory {}", mod_dir.display()); + fs::create_dir_all(Arc::as_ref(&mod_dir)) + .await + .wrap_err_with(|| format!("Failed to create data directory {}", mod_dir.display()))?; + + tracing::trace!("Extracting mod archive to {}", mod_dir.display()); + archive + .extract(Arc::as_ref(&mod_dir)) + .wrap_err_with(|| format!("Failed to extract archive to {}", mod_dir.display()))?; + + if let Some(nexus) = &nexus { + let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; + let path = mod_dir.join(&mod_cfg.id).join("nexus.sjson"); + fs::write(&path, data.as_bytes()) + .await + .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; + } + + let packages = files + .into_iter() + .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) + .collect(); + let info = ModInfo::new(mod_cfg, packages, image, nexus); + + Ok(info) +} + #[tracing::instrument(skip(state))] pub(crate) async fn delete_mod(state: ActionState, info: &ModInfo) -> Result<()> { let mod_dir = state.mod_dir.join(&info.id); @@ -73,13 +229,9 @@ async fn read_mod_dir_entry(res: Result) -> Result { Err(err) => return Err(err), }; - let files: HashMap> = if cfg.bundled { - read_sjson_file(&index_path) - .await - .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))? - } else { - Default::default() - }; + let files: HashMap> = read_sjson_file(&index_path) + .await + .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?; let image = if let Some(path) = &cfg.image { let path = entry.path().join(path); diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs deleted file mode 100644 index 53c4ef1..0000000 --- a/crates/dtmm/src/controller/deploy.rs +++ /dev/null @@ -1,868 +0,0 @@ -use std::collections::HashMap; -use std::io::{Cursor, ErrorKind}; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use std::sync::Arc; - -use color_eyre::eyre::Context; -use color_eyre::{eyre, Help, Report, Result}; -use futures::StreamExt; -use futures::{stream, TryStreamExt}; -use minijinja::Environment; -use sdk::filetype::lua; -use sdk::filetype::package::Package; -use sdk::murmur::Murmur64; -use sdk::{ - Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, -}; -use serde::{Deserialize, Serialize}; -use string_template::Template; -use time::OffsetDateTime; -use tokio::fs::{self, DirEntry}; -use tokio::io::AsyncWriteExt; -use tracing::Instrument; - -use super::read_sjson_file; -use crate::controller::app::check_mod_order; -use crate::state::{ActionState, PackageInfo}; - -pub const MOD_BUNDLE_NAME: &str = "packages/mods"; -pub const BOOT_BUNDLE_NAME: &str = "packages/boot"; -pub const DML_BUNDLE_NAME: &str = "packages/dml"; -pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; -pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; -pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; -pub const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; -pub const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; - -#[derive(Debug, Serialize, Deserialize)] -pub struct DeploymentData { - pub bundles: Vec, - pub mod_folders: Vec, - #[serde(with = "time::serde::iso8601")] - pub timestamp: OffsetDateTime, -} - -#[tracing::instrument] -async fn read_file_with_backup

(path: P) -> Result> -where - P: AsRef + std::fmt::Debug, -{ - let path = path.as_ref(); - let backup_path = { - let mut p = PathBuf::from(path); - let ext = if let Some(ext) = p.extension() { - ext.to_string_lossy().to_string() + ".bak" - } else { - String::from("bak") - }; - p.set_extension(ext); - p - }; - - let file_name = path - .file_name() - .map(|s| s.to_string_lossy().to_string()) - .unwrap_or_else(|| String::from("file")); - - let bin = match fs::read(&backup_path).await { - Ok(bin) => bin, - Err(err) if err.kind() == ErrorKind::NotFound => { - // TODO: This doesn't need to be awaited here, yet. - // I only need to make sure it has finished before writing the changed bundle. - tracing::debug!( - "Backup does not exist. Backing up original {} to '{}'", - file_name, - backup_path.display() - ); - fs::copy(path, &backup_path).await.wrap_err_with(|| { - format!( - "Failed to back up {} '{}' to '{}'", - file_name, - path.display(), - backup_path.display() - ) - })?; - - tracing::debug!("Reading {} from original '{}'", file_name, path.display()); - fs::read(path).await.wrap_err_with(|| { - format!("Failed to read {} file: {}", file_name, path.display()) - })? - } - Err(err) => { - return Err(err).wrap_err_with(|| { - format!( - "Failed to read {} from backup '{}'", - file_name, - backup_path.display() - ) - }); - } - }; - Ok(bin) -} - -#[tracing::instrument(skip_all)] -async fn patch_game_settings(state: Arc) -> Result<()> { - let settings_path = state.game_dir.join("bundle").join(SETTINGS_FILE_PATH); - - let settings = read_file_with_backup(&settings_path) - .await - .wrap_err("Failed to read settings.ini")?; - let settings = String::from_utf8(settings).wrap_err("Settings.ini is not valid UTF-8")?; - - let mut f = fs::File::create(&settings_path) - .await - .wrap_err_with(|| format!("Failed to open {}", settings_path.display()))?; - - let Some(i) = settings.find("boot_script =") else { - eyre::bail!("couldn't find 'boot_script' field"); - }; - - f.write_all(settings[0..i].as_bytes()).await?; - f.write_all(b"boot_script = \"scripts/mod_main\"").await?; - - let Some(j) = settings[i..].find('\n') else { - eyre::bail!("couldn't find end of 'boot_script' field"); - }; - - f.write_all(settings[(i + j)..].as_bytes()).await?; - - Ok(()) -} - -#[tracing::instrument(skip_all, fields(package = info.name))] -fn make_package(info: &PackageInfo) -> Result { - let mut pkg = Package::new(info.name.clone(), PathBuf::new()); - - for f in &info.files { - let mut it = f.rsplit('.'); - let file_type = it - .next() - .ok_or_else(|| eyre::eyre!("missing file extension")) - .and_then(BundleFileType::from_str) - .wrap_err("Invalid file name in package info")?; - let name: String = it.collect(); - pkg.add_file(file_type, name); - } - - Ok(pkg) -} - -#[tracing::instrument] -async fn copy_recursive( - from: impl Into + std::fmt::Debug, - to: impl AsRef + std::fmt::Debug, -) -> Result<()> { - let to = to.as_ref(); - - #[tracing::instrument] - async fn handle_dir(from: PathBuf) -> Result> { - let mut dir = fs::read_dir(&from) - .await - .wrap_err("Failed to read directory")?; - let mut entries = Vec::new(); - - while let Some(entry) = dir.next_entry().await? { - let meta = entry.metadata().await.wrap_err_with(|| { - format!("Failed to get metadata for '{}'", entry.path().display()) - })?; - entries.push((meta.is_dir(), entry)); - } - - Ok(entries) - } - - let base = from.into(); - stream::unfold(vec![base.clone()], |mut state| async { - let from = state.pop()?; - let inner = match handle_dir(from).await { - Ok(entries) => { - for (is_dir, entry) in &entries { - if *is_dir { - state.push(entry.path()); - } - } - stream::iter(entries).map(Ok).left_stream() - } - Err(e) => stream::once(async { Err(e) }).right_stream(), - }; - - Some((inner, state)) - }) - .flatten() - .try_for_each(|(is_dir, entry)| { - let path = entry.path(); - let dest = path - .strip_prefix(&base) - .map(|suffix| to.join(suffix)) - .expect("all entries are relative to the directory we are walking"); - - async move { - if is_dir { - tracing::trace!("Creating directory '{}'", dest.display()); - // Instead of trying to filter "already exists" errors out explicitly, - // we just ignore all. It'll fail eventually with the next copy operation. - let _ = fs::create_dir(&dest).await; - Ok(()) - } else { - tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display()); - fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| { - format!( - "Failed to copy file '{}' -> '{}'", - path.display(), - dest.display() - ) - }) - } - } - }) - .await - .map(|_| ()) -} - -#[tracing::instrument(skip(state))] -async fn copy_mod_folders(state: Arc) -> Result> { - let game_dir = Arc::clone(&state.game_dir); - - let mut tasks = Vec::new(); - - for mod_info in state - .mods - .iter() - .filter(|m| m.id != "dml" && m.enabled && !m.bundled) - { - let span = tracing::trace_span!("copying legacy mod", name = mod_info.name); - let _enter = span.enter(); - - let mod_id = mod_info.id.clone(); - let mod_dir = Arc::clone(&state.mod_dir); - let game_dir = Arc::clone(&game_dir); - - let task = async move { - let from = mod_dir.join(&mod_id); - let to = game_dir.join("mods").join(&mod_id); - - tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id); - let _ = fs::create_dir_all(&to).await; - copy_recursive(&from, &to).await.wrap_err_with(|| { - format!( - "Failed to copy legacy mod from '{}' to '{}'", - from.display(), - to.display() - ) - })?; - - Ok::<_, Report>(mod_id) - }; - tasks.push(task); - } - - let ids = futures::future::try_join_all(tasks).await?; - Ok(ids) -} - -fn build_mod_data_lua(state: Arc) -> Result { - #[derive(Serialize)] - struct TemplateDataMod { - id: String, - name: String, - bundled: bool, - init: String, - data: Option, - localization: Option, - packages: Vec, - } - - let mut env = Environment::new(); - env.add_template("mod_data.lua", include_str!("../../assets/mod_data.lua.j2")) - .wrap_err("Failed to compile template for `mod_data.lua`")?; - let tmpl = env - .get_template("mod_data.lua") - .wrap_err("Failed to get template `mod_data.lua`")?; - - let data: Vec = state - .mods - .iter() - .filter_map(|m| { - if m.id == "dml" || !m.enabled { - return None; - } - - Some(TemplateDataMod { - id: m.id.clone(), - name: m.name.clone(), - bundled: m.bundled, - init: m.resources.init.to_string_lossy().to_string(), - data: m - .resources - .data - .as_ref() - .map(|p| p.to_string_lossy().to_string()), - localization: m - .resources - .localization - .as_ref() - .map(|p| p.to_string_lossy().to_string()), - packages: m.packages.iter().map(|p| p.name.clone()).collect(), - }) - }) - .collect(); - - let lua = tmpl - .render(minijinja::context!(mods => data)) - .wrap_err("Failed to render template `mod_data.lua`")?; - - tracing::debug!("mod_data.lua:\n{}", lua); - - Ok(lua) -} - -#[tracing::instrument(skip_all)] -async fn build_bundles(state: Arc) -> Result> { - let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); - let mut tasks = Vec::new(); - - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - - let mut bundles = Vec::new(); - - { - tracing::trace!("Building mod data script"); - - let span = tracing::debug_span!("Building mod data script"); - let _enter = span.enter(); - - let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?; - - tracing::trace!("Compiling mod data script"); - - let file = - lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?; - - tracing::trace!("Compile mod data script"); - - mod_bundle.add_file(file); - } - - tracing::trace!("Preparing tasks to deploy bundle files"); - - for mod_info in state - .mods - .iter() - .filter(|m| m.id != "dml" && m.enabled && m.bundled) - { - let span = tracing::trace_span!("building mod packages", name = mod_info.name); - let _enter = span.enter(); - - let mod_dir = state.mod_dir.join(&mod_info.id); - for pkg_info in &mod_info.packages { - let span = tracing::trace_span!("building package", name = pkg_info.name); - let _enter = span.enter(); - - tracing::trace!( - "Building package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; - let mut variant = BundleFileVariant::new(); - let bin = pkg - .to_binary() - .wrap_err("Failed to serialize package to binary")?; - variant.set_data(bin); - let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); - file.add_variant(variant); - - tracing::trace!( - "Compiled package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - mod_bundle.add_file(file); - - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = mod_dir.join(&bundle_name); - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - // Explicitely drop the guard, so that we can move the span - // into the async operation - drop(_enter); - - let ctx = state.ctx.clone(); - - let task = async move { - let bundle = { - let bin = fs::read(&src).await.wrap_err_with(|| { - format!("Failed to read bundle file '{}'", src.display()) - })?; - let name = Bundle::get_name_from_path(&ctx, &src); - Bundle::from_binary(&ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? - }; - - tracing::debug!( - src = %src.display(), - dest = %dest.display(), - "Copying bundle '{}' for mod '{}'", - pkg_name, - mod_name, - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - - Ok::(bundle) - } - .instrument(span); - - tasks.push(task); - } - } - - tracing::debug!("Copying {} mod bundles", tasks.len()); - - let mut tasks = stream::iter(tasks).buffer_unordered(10); - - while let Some(res) = tasks.next().await { - let bundle = res?; - bundles.push(bundle); - } - - { - let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); - tracing::trace!("Writing mod bundle to '{}'", path.display()); - fs::write(&path, mod_bundle.to_binary()?) - .await - .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; - } - - bundles.push(mod_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all)] -async fn patch_boot_bundle(state: Arc) -> Result> { - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); - - let mut bundles = Vec::with_capacity(2); - - let mut boot_bundle = async { - let bin = read_file_with_backup(&bundle_path) - .await - .wrap_err("Failed to read boot bundle")?; - - Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) - .wrap_err("Failed to parse boot bundle") - } - .instrument(tracing::trace_span!("read boot bundle")) - .await - .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; - - { - tracing::trace!("Adding mod package file to boot bundle"); - let span = tracing::trace_span!("create mod package file"); - let _enter = span.enter(); - - let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); - - for mod_info in &state.mods { - for pkg_info in &mod_info.packages { - pkg.add_file(BundleFileType::Package, &pkg_info.name); - } - } - - pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); - - let mut variant = BundleFileVariant::new(); - variant.set_data(pkg.to_binary()?); - let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - tracing::trace!("Handling DML packages and bundle"); - let span = tracing::trace_span!("handle DML"); - let _enter = span.enter(); - - let mut variant = BundleFileVariant::new(); - - let mod_info = state - .mods - .iter() - .find(|m| m.id == "dml") - .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; - let pkg_info = mod_info - .packages - .get(0) - .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) - .with_suggestion(|| "Re-download and import the newest version.".to_string())?; - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); - - { - let bin = fs::read(&src) - .await - .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; - let name = Bundle::get_name_from_path(&state.ctx, &src); - - let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; - - bundles.push(dml_bundle); - }; - - { - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - tracing::debug!( - "Copying bundle {} for mod {}: {} -> {}", - pkg_name, - mod_name, - src.display(), - dest.display() - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - } - - let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; - variant.set_data(pkg.to_binary()?); - - let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - let span = tracing::debug_span!("Importing mod main script"); - let _enter = span.enter(); - - let is_io_enabled = format!("{}", state.is_io_enabled); - let mut data = HashMap::new(); - data.insert("is_io_enabled", is_io_enabled.as_str()); - - let tmpl = include_str!("../../assets/mod_main.lua"); - let lua = Template::new(tmpl).render(&data); - tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); - let file = - lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; - - boot_bundle.add_file(file); - } - - async { - let bin = boot_bundle - .to_binary() - .wrap_err("Failed to serialize boot bundle")?; - fs::write(&bundle_path, bin) - .await - .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) - } - .instrument(tracing::trace_span!("write boot bundle")) - .await?; - - bundles.push(boot_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); - - let mut db = { - let bin = read_file_with_backup(&database_path) - .await - .wrap_err("Failed to read bundle database")?; - let mut r = Cursor::new(bin); - let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; - tracing::trace!("Finished parsing bundle database"); - db - }; - - for bundle in bundles.as_ref() { - tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); - db.add_bundle(bundle); - } - - { - let bin = db - .to_binary() - .wrap_err("Failed to serialize bundle database")?; - fs::write(&database_path, bin).await.wrap_err_with(|| { - format!( - "failed to write bundle database to '{}'", - database_path.display() - ) - })?; - } - - Ok(()) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn write_deployment_data( - state: Arc, - bundles: B, - mod_folders: Vec, -) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let info = DeploymentData { - timestamp: OffsetDateTime::now_utc(), - bundles: bundles - .as_ref() - .iter() - .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) - .collect(), - // TODO: - mod_folders, - }; - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; - - fs::write(&path, &data) - .await - .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; - - Ok(()) -} - -#[tracing::instrument(skip_all, fields( - game_dir = %state.game_dir.display(), - mods = state.mods.len() -))] -pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { - let state = Arc::new(state); - let bundle_dir = state.game_dir.join("bundle"); - let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); - - if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) - .await - .is_ok() - { - let err = eyre::eyre!("Found dtkit-patch-based mod installation."); - return Err(err) - .with_suggestion(|| { - "If you're a mod author and saved projects directly in 'mods/', \ - use DTMT to migrate them to the new project structure." - .to_string() - }) - .with_suggestion(|| { - "Click 'Reset Game' to remove the previous mod installation.".to_string() - }); - } - - let (_, game_info, deployment_info) = tokio::try_join!( - async { - fs::metadata(&bundle_dir) - .await - .wrap_err("Failed to open game bundle directory") - .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") - }, - async { - tokio::task::spawn_blocking(dtmt_shared::collect_game_info) - .await - .map_err(Report::new) - }, - async { - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - match read_sjson_file::<_, DeploymentData>(path).await { - Ok(data) => Ok(Some(data)), - Err(err) => { - if let Some(err) = err.downcast_ref::() - && err.kind() == ErrorKind::NotFound - { - Ok(None) - } else { - Err(err).wrap_err("Failed to read deployment data") - } - } - } - } - ) - .wrap_err("Failed to gather deployment information")?; - - tracing::debug!(?game_info, ?deployment_info); - - if let Some(game_info) = game_info { - if deployment_info - .as_ref() - .map(|i| game_info.last_updated > i.timestamp) - .unwrap_or(false) - { - tracing::warn!( - "Game was updated since last mod deployment. \ - Attempting to reconcile game files." - ); - - tokio::try_join!( - async { - let path = bundle_dir.join(BUNDLE_DATABASE_NAME); - let backup_path = path.with_extension("data.bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for bundle database.") - }, - async { - let path = bundle_dir.join(boot_bundle_path); - let backup_path = path.with_extension("bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for boot bundle") - } - ) - .with_suggestion(|| { - "Reset the game using 'Reset Game', then verify game files.".to_string() - })?; - - tracing::info!( - "Successfully re-created game file backups. \ - Continuing mod deployment." - ); - } - } - - check_mod_order(&state)?; - - tracing::info!( - "Deploying {} mods to '{}'.", - state.mods.iter().filter(|i| i.enabled).count(), - bundle_dir.display() - ); - - tracing::info!("Copy legacy mod folders"); - let mod_folders = copy_mod_folders(state.clone()) - .await - .wrap_err("Failed to copy mod folders")?; - - tracing::info!("Build mod bundles"); - let mut bundles = build_bundles(state.clone()) - .await - .wrap_err("Failed to build mod bundles")?; - - tracing::info!("Patch boot bundle"); - let mut boot_bundles = patch_boot_bundle(state.clone()) - .await - .wrap_err("Failed to patch boot bundle")?; - bundles.append(&mut boot_bundles); - - if let Some(info) = &deployment_info { - let bundle_dir = Arc::new(bundle_dir); - // Remove bundles from the previous deployment that don't match the current one. - // I.e. mods that used to be installed/enabled but aren't anymore. - { - let tasks = info.bundles.iter().cloned().filter_map(|file_name| { - let is_being_deployed = bundles.iter().any(|b2| { - let name = format!("{:016x}", b2.name()); - file_name == name - }); - - if !is_being_deployed { - let bundle_dir = bundle_dir.clone(); - let task = async move { - let path = bundle_dir.join(&file_name); - - tracing::debug!("Removing unused bundle '{}'", file_name); - - if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { - format!("Failed to remove unused bundle '{}'", path.display()) - }) { - tracing::error!("{:?}", err); - } - }; - Some(task) - } else { - None - } - }); - - futures::future::join_all(tasks).await; - } - - // Do the same thing for mod folders - { - let tasks = info.mod_folders.iter().filter_map(|mod_id| { - let is_being_deployed = mod_folders.iter().any(|id| id == mod_id); - - if !is_being_deployed { - let path = bundle_dir.join("mods").join(mod_id); - tracing::debug!("Removing unused mod folder '{}'", path.display()); - - let task = async move { - if let Err(err) = fs::remove_dir_all(&path).await.wrap_err_with(|| { - format!("Failed to remove unused legacy mod '{}'", path.display()) - }) { - tracing::error!("{:?}", err); - } - }; - - Some(task) - } else { - None - } - }); - futures::future::join_all(tasks).await; - } - } - - tracing::info!("Patch game settings"); - patch_game_settings(state.clone()) - .await - .wrap_err("Failed to patch game settings")?; - - tracing::info!("Patching bundle database"); - patch_bundle_database(state.clone(), &bundles) - .await - .wrap_err("Failed to patch bundle database")?; - - tracing::info!("Writing deployment data"); - write_deployment_data(state.clone(), &bundles, mod_folders) - .await - .wrap_err("Failed to write deployment data")?; - - tracing::info!("Finished deploying mods"); - Ok(()) -} diff --git a/crates/dtmm/src/controller/game.rs b/crates/dtmm/src/controller/game.rs index 6b91169..a5f6fa8 100644 --- a/crates/dtmm/src/controller/game.rs +++ b/crates/dtmm/src/controller/game.rs @@ -1,19 +1,46 @@ -use std::io::{self, ErrorKind}; +use std::collections::HashMap; +use std::io::{self, Cursor, ErrorKind}; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use color_eyre::eyre::Context; -use color_eyre::{eyre, Result}; +use color_eyre::{eyre, Help, Report, Result}; +use futures::stream; +use futures::StreamExt; +use path_slash::PathBufExt; +use sdk::filetype::lua; +use sdk::filetype::package::Package; use sdk::murmur::Murmur64; -use tokio::fs::{self}; -use tokio::io::AsyncWriteExt; - -use crate::controller::deploy::{ - DeploymentData, BOOT_BUNDLE_NAME, BUNDLE_DATABASE_NAME, DEPLOYMENT_DATA_PATH, +use sdk::{ + Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, }; -use crate::state::ActionState; +use serde::{Deserialize, Serialize}; +use string_template::Template; +use time::OffsetDateTime; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use tracing::Instrument; -use super::deploy::SETTINGS_FILE_PATH; +use super::read_sjson_file; +use crate::controller::app::check_mod_order; +use crate::state::{ActionState, PackageInfo}; + +const MOD_BUNDLE_NAME: &str = "packages/mods"; +const BOOT_BUNDLE_NAME: &str = "packages/boot"; +const DML_BUNDLE_NAME: &str = "packages/dml"; +const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; +const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; +const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; +const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; +const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; + +#[derive(Debug, Serialize, Deserialize)] +struct DeploymentData { + bundles: Vec, + #[serde(with = "time::serde::iso8601")] + timestamp: OffsetDateTime, +} #[tracing::instrument] async fn read_file_with_backup

(path: P) -> Result> @@ -103,6 +130,585 @@ async fn patch_game_settings(state: Arc) -> Result<()> { Ok(()) } +#[tracing::instrument(skip_all, fields(package = info.name))] +fn make_package(info: &PackageInfo) -> Result { + let mut pkg = Package::new(info.name.clone(), PathBuf::new()); + + for f in &info.files { + let mut it = f.rsplit('.'); + let file_type = it + .next() + .ok_or_else(|| eyre::eyre!("missing file extension")) + .and_then(BundleFileType::from_str) + .wrap_err("Invalid file name in package info")?; + let name: String = it.collect(); + pkg.add_file(file_type, name); + } + + Ok(pkg) +} + +fn build_mod_data_lua(state: Arc) -> String { + let mut lua = String::from("return {\n"); + + // DMF is handled explicitely by the loading procedures, as it actually drives most of that + // and should therefore not show up in the load order. + for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { + lua.push_str(" {\n name = \""); + lua.push_str(&mod_info.name); + + lua.push_str("\",\n id = \""); + lua.push_str(&mod_info.id); + + lua.push_str("\",\n run = function()\n"); + + let resources = &mod_info.resources; + if resources.data.is_some() || resources.localization.is_some() { + lua.push_str(" new_mod(\""); + lua.push_str(&mod_info.id); + lua.push_str("\", {\n mod_script = \""); + lua.push_str(&resources.init.to_slash_lossy()); + + if let Some(data) = resources.data.as_ref() { + lua.push_str("\",\n mod_data = \""); + lua.push_str(&data.to_slash_lossy()); + } + + if let Some(localization) = &resources.localization { + lua.push_str("\",\n mod_localization = \""); + lua.push_str(&localization.to_slash_lossy()); + } + + lua.push_str("\",\n })\n"); + } else { + lua.push_str(" return dofile(\""); + lua.push_str(&resources.init.to_slash_lossy()); + lua.push_str("\")\n"); + } + + lua.push_str(" end,\n packages = {\n"); + + for pkg_info in &mod_info.packages { + lua.push_str(" \""); + lua.push_str(&pkg_info.name); + lua.push_str("\",\n"); + } + + lua.push_str(" },\n },\n"); + } + + lua.push('}'); + + tracing::debug!("mod_data_lua:\n{}", lua); + + lua +} + +#[tracing::instrument(skip_all)] +async fn build_bundles(state: Arc) -> Result> { + let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); + let mut tasks = Vec::new(); + + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + + let mut bundles = Vec::new(); + + { + tracing::trace!("Building mod data script"); + + let span = tracing::debug_span!("Building mod data script"); + let _enter = span.enter(); + + let lua = build_mod_data_lua(state.clone()); + + tracing::trace!("Compiling mod data script"); + + let file = + lua::compile(MOD_DATA_SCRIPT, &lua).wrap_err("Failed to compile mod data Lua file")?; + + tracing::trace!("Compile mod data script"); + + mod_bundle.add_file(file); + } + + tracing::trace!("Preparing tasks to deploy bundle files"); + + for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { + let span = tracing::trace_span!("building mod packages", name = mod_info.name); + let _enter = span.enter(); + + let mod_dir = state.mod_dir.join(&mod_info.id); + for pkg_info in &mod_info.packages { + let span = tracing::trace_span!("building package", name = pkg_info.name); + let _enter = span.enter(); + + tracing::trace!( + "Building package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; + let mut variant = BundleFileVariant::new(); + let bin = pkg + .to_binary() + .wrap_err("Failed to serialize package to binary")?; + variant.set_data(bin); + let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); + file.add_variant(variant); + + tracing::trace!( + "Compiled package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + mod_bundle.add_file(file); + + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = mod_dir.join(&bundle_name); + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + // Explicitely drop the guard, so that we can move the span + // into the async operation + drop(_enter); + + let ctx = state.ctx.clone(); + + let task = async move { + let bundle = { + let bin = fs::read(&src).await.wrap_err_with(|| { + format!("Failed to read bundle file '{}'", src.display()) + })?; + let name = Bundle::get_name_from_path(&ctx, &src); + Bundle::from_binary(&ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? + }; + + tracing::debug!( + src = %src.display(), + dest = %dest.display(), + "Copying bundle '{}' for mod '{}'", + pkg_name, + mod_name, + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + + Ok::(bundle) + } + .instrument(span); + + tasks.push(task); + } + } + + tracing::debug!("Copying {} mod bundles", tasks.len()); + + let mut tasks = stream::iter(tasks).buffer_unordered(10); + + while let Some(res) = tasks.next().await { + let bundle = res?; + bundles.push(bundle); + } + + { + let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); + tracing::trace!("Writing mod bundle to '{}'", path.display()); + fs::write(&path, mod_bundle.to_binary()?) + .await + .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; + } + + bundles.push(mod_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all)] +async fn patch_boot_bundle(state: Arc) -> Result> { + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); + + let mut bundles = Vec::with_capacity(2); + + let mut boot_bundle = async { + let bin = read_file_with_backup(&bundle_path) + .await + .wrap_err("Failed to read boot bundle")?; + + Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) + .wrap_err("Failed to parse boot bundle") + } + .instrument(tracing::trace_span!("read boot bundle")) + .await + .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; + + { + tracing::trace!("Adding mod package file to boot bundle"); + let span = tracing::trace_span!("create mod package file"); + let _enter = span.enter(); + + let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); + + for mod_info in &state.mods { + for pkg_info in &mod_info.packages { + pkg.add_file(BundleFileType::Package, &pkg_info.name); + } + } + + pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); + + let mut variant = BundleFileVariant::new(); + variant.set_data(pkg.to_binary()?); + let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + tracing::trace!("Handling DML packages and bundle"); + let span = tracing::trace_span!("handle DML"); + let _enter = span.enter(); + + let mut variant = BundleFileVariant::new(); + + let mod_info = state + .mods + .iter() + .find(|m| m.id == "dml") + .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; + let pkg_info = mod_info + .packages + .get(0) + .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) + .with_suggestion(|| "Re-download and import the newest version.".to_string())?; + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); + + { + let bin = fs::read(&src) + .await + .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; + let name = Bundle::get_name_from_path(&state.ctx, &src); + + let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; + + bundles.push(dml_bundle); + }; + + { + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + tracing::debug!( + "Copying bundle {} for mod {}: {} -> {}", + pkg_name, + mod_name, + src.display(), + dest.display() + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + } + + let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; + variant.set_data(pkg.to_binary()?); + + let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + let span = tracing::debug_span!("Importing mod main script"); + let _enter = span.enter(); + + let is_io_enabled = format!("{}", state.is_io_enabled); + let mut data = HashMap::new(); + data.insert("is_io_enabled", is_io_enabled.as_str()); + + let tmpl = include_str!("../../assets/mod_main.lua"); + let lua = Template::new(tmpl).render(&data); + tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); + let file = + lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; + + boot_bundle.add_file(file); + } + + async { + let bin = boot_bundle + .to_binary() + .wrap_err("Failed to serialize boot bundle")?; + fs::write(&bundle_path, bin) + .await + .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) + } + .instrument(tracing::trace_span!("write boot bundle")) + .await?; + + bundles.push(boot_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); + + let mut db = { + let bin = read_file_with_backup(&database_path) + .await + .wrap_err("Failed to read bundle database")?; + let mut r = Cursor::new(bin); + let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; + tracing::trace!("Finished parsing bundle database"); + db + }; + + for bundle in bundles.as_ref() { + tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); + db.add_bundle(bundle); + } + + { + let bin = db + .to_binary() + .wrap_err("Failed to serialize bundle database")?; + fs::write(&database_path, bin).await.wrap_err_with(|| { + format!( + "failed to write bundle database to '{}'", + database_path.display() + ) + })?; + } + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn write_deployment_data(state: Arc, bundles: B) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let info = DeploymentData { + timestamp: OffsetDateTime::now_utc(), + bundles: bundles + .as_ref() + .iter() + .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) + .collect(), + }; + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; + + fs::write(&path, &data) + .await + .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields( + game_dir = %state.game_dir.display(), + mods = state.mods.len() +))] +pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { + let state = Arc::new(state); + let bundle_dir = state.game_dir.join("bundle"); + let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); + + if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) + .await + .is_ok() + { + let err = eyre::eyre!("Found dtkit-patch-based mod installation."); + return Err(err) + .with_suggestion(|| { + "If you're a mod author and saved projects directly in 'mods/', \ + use DTMT to migrate them to the new project structure." + .to_string() + }) + .with_suggestion(|| { + "Click 'Reset Game' to remove the previous mod installation.".to_string() + }); + } + + let (_, game_info, deployment_info) = tokio::try_join!( + async { + fs::metadata(&bundle_dir) + .await + .wrap_err("Failed to open game bundle directory") + .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") + }, + async { + tokio::task::spawn_blocking(dtmt_shared::collect_game_info) + .await + .map_err(Report::new) + }, + async { + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + match read_sjson_file::<_, DeploymentData>(path) + .await + { + Ok(data) => Ok(Some(data)), + Err(err) => { + if let Some(err) = err.downcast_ref::() && err.kind() == ErrorKind::NotFound { + Ok(None) + } else { + Err(err).wrap_err("Failed to read deployment data") + } + } + } + } + ) + .wrap_err("Failed to gather deployment information")?; + + tracing::debug!(?game_info, ?deployment_info); + + if let Some(game_info) = game_info { + if deployment_info + .as_ref() + .map(|i| game_info.last_updated > i.timestamp) + .unwrap_or(false) + { + tracing::warn!( + "Game was updated since last mod deployment. \ + Attempting to reconcile game files." + ); + + tokio::try_join!( + async { + let path = bundle_dir.join(BUNDLE_DATABASE_NAME); + let backup_path = path.with_extension("data.bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for bundle database.") + }, + async { + let path = bundle_dir.join(boot_bundle_path); + let backup_path = path.with_extension("bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for boot bundle") + } + ) + .with_suggestion(|| { + "Reset the game using 'Reset Game', then verify game files.".to_string() + })?; + + tracing::info!( + "Successfully re-created game file backups. \ + Continuing mod deployment." + ); + } + } + + check_mod_order(&state)?; + + tracing::info!( + "Deploying {} mods to '{}'.", + state.mods.iter().filter(|i| i.enabled).count(), + bundle_dir.display() + ); + + tracing::info!("Build mod bundles"); + let mut bundles = build_bundles(state.clone()) + .await + .wrap_err("Failed to build mod bundles")?; + + tracing::info!("Patch boot bundle"); + let mut more_bundles = patch_boot_bundle(state.clone()) + .await + .wrap_err("Failed to patch boot bundle")?; + bundles.append(&mut more_bundles); + + if let Some(info) = &deployment_info { + let bundle_dir = Arc::new(bundle_dir); + let tasks = info.bundles.iter().cloned().filter_map(|file_name| { + let contains = bundles.iter().any(|b2| { + let name = format!("{:016x}", b2.name()); + file_name == name + }); + + if !contains { + let bundle_dir = bundle_dir.clone(); + let task = async move { + let path = bundle_dir.join(&file_name); + + tracing::debug!("Removing unused bundle '{}'", file_name); + + if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { + format!("Failed to remove unused bundle '{}'", path.display()) + }) { + tracing::error!("{:?}", err); + } + }; + Some(task) + } else { + None + } + }); + + futures::future::join_all(tasks).await; + } + + tracing::info!("Patch game settings"); + patch_game_settings(state.clone()) + .await + .wrap_err("Failed to patch game settings")?; + + tracing::info!("Patching bundle database"); + patch_bundle_database(state.clone(), &bundles) + .await + .wrap_err("Failed to patch bundle database")?; + + tracing::info!("Writing deployment data"); + write_deployment_data(state.clone(), &bundles) + .await + .wrap_err("Failed to write deployment data")?; + + tracing::info!("Finished deploying mods"); + Ok(()) +} + #[tracing::instrument(skip_all)] async fn reset_dtkit_patch(state: ActionState) -> Result<()> { let bundle_dir = state.game_dir.join("bundle"); diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs deleted file mode 100644 index 131d01f..0000000 --- a/crates/dtmm/src/controller/import.rs +++ /dev/null @@ -1,495 +0,0 @@ -use std::collections::HashMap; -use std::ffi::CStr; -use std::io::{Cursor, Read, Seek, Write}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use color_eyre::eyre::{self, Context}; -use color_eyre::{Help, Report, Result}; -use druid::im::Vector; -use druid::{FileInfo, ImageBuf}; -use dtmt_shared::{ModConfig, ModConfigResources}; -use luajit2_sys as lua; -use nexusmods::Api as NexusApi; -use tokio::fs; -use zip::ZipArchive; - -use crate::state::{ActionState, ModInfo, NexusInfo, PackageInfo}; - -fn find_archive_file( - archive: &ZipArchive, - name: impl AsRef, -) -> Option { - let path = archive - .file_names() - .find(|path| path.ends_with(name.as_ref())) - .map(|s| s.to_string()); - path -} - -// Runs the content of a `.mod` file to extract what data we can get -// from legacy mods. -// 1. Create a global function `new_mod` that stores -// the relevant bits in global variables. -// 2. Run the `.mod` file, which will return a table. -// 3. Run the `run` function from that table. -// 4. Access the global variables from #1. -#[tracing::instrument] -fn parse_mod_id_file(data: &str) -> Result<(String, ModConfigResources)> { - tracing::debug!("Parsing mod file:\n{}", data); - - let ret = unsafe { - let state = lua::luaL_newstate(); - lua::luaL_openlibs(state); - - let run = b" -function fassert() end -function new_mod(id, resources) - _G.id = id - _G.script = resources.mod_script - _G.data = resources.mod_data - _G.localization = resources.mod_localization -end -\0"; - match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 { - lua::LUA_OK => {} - lua::LUA_ERRSYNTAX => { - let err = lua::lua_tostring(state, -1); - let err = CStr::from_ptr(err).to_string_lossy().to_string(); - - lua::lua_close(state); - - eyre::bail!("Invalid syntax: {}", err); - } - lua::LUA_ERRMEM => { - lua::lua_close(state); - eyre::bail!("Failed to allocate sufficient memory to create `new_mod`") - } - _ => unreachable!(), - } - - match lua::lua_pcall(state, 0, 0, 0) as u32 { - lua::LUA_OK => {} - lua::LUA_ERRRUN => { - let err = lua::lua_tostring(state, -1); - let err = CStr::from_ptr(err).to_string_lossy().to_string(); - - lua::lua_close(state); - - eyre::bail!("Failed to run buffer: {}", err); - } - lua::LUA_ERRMEM => { - lua::lua_close(state); - eyre::bail!("Failed to allocate sufficient memory to run buffer") - } - // We don't use an error handler function, so this should be unreachable - lua::LUA_ERRERR => unreachable!(), - _ => unreachable!(), - } - - let name = b".mod\0"; - match lua::luaL_loadbuffer( - state, - data.as_ptr() as _, - data.len() as _, - name.as_ptr() as _, - ) as u32 - { - lua::LUA_OK => {} - lua::LUA_ERRSYNTAX => { - let err = lua::lua_tostring(state, -1); - let err = CStr::from_ptr(err).to_string_lossy().to_string(); - - lua::lua_close(state); - - eyre::bail!("Invalid syntax: {}", err); - } - lua::LUA_ERRMEM => { - lua::lua_close(state); - eyre::bail!("Failed to allocate sufficient memory to load `.mod` file buffer") - } - _ => unreachable!(), - } - - match lua::lua_pcall(state, 0, 1, 0) as u32 { - lua::LUA_OK => {} - lua::LUA_ERRRUN => { - let err = lua::lua_tostring(state, -1); - let err = CStr::from_ptr(err).to_string_lossy().to_string(); - - lua::lua_close(state); - - eyre::bail!("Failed to run `.mod` file: {}", err); - } - lua::LUA_ERRMEM => { - lua::lua_close(state); - eyre::bail!("Failed to allocate sufficient memory to run `.mod` file") - } - // We don't use an error handler function, so this should be unreachable - lua::LUA_ERRERR => unreachable!(), - _ => unreachable!(), - } - - let key = b"run\0"; - lua::lua_pushstring(state, key.as_ptr() as _); - lua::lua_gettable(state, -2); - - match lua::lua_pcall(state, 0, 0, 0) as u32 { - lua::LUA_OK => {} - lua::LUA_ERRRUN => { - let err = lua::lua_tostring(state, -1); - let err = CStr::from_ptr(err).to_string_lossy().to_string(); - - lua::lua_close(state); - - eyre::bail!("Failed to run `.mod.run`: {}", err); - } - lua::LUA_ERRMEM => { - lua::lua_close(state); - eyre::bail!("Failed to allocate sufficient memory to run `.mod.run`") - } - // We don't use an error handler function, so this should be unreachable - lua::LUA_ERRERR => unreachable!(), - _ => unreachable!(), - } - - let get_global = |state, key: &[u8]| { - lua::lua_getglobal(state, key.as_ptr() as _); - - if lua::lua_isnil(state, -1) != 0 { - return Ok(None); - } - - let s = lua::lua_tostring(state, -1); - - if s.is_null() { - eyre::bail!("Expected string, got NULL"); - } - - let ret = CStr::from_ptr(s).to_string_lossy().to_string(); - lua::lua_pop(state, 1); - Ok(Some(ret)) - }; - - let mod_id = get_global(state, b"id\0") - .and_then(|s| s.ok_or_else(|| eyre::eyre!("Got `nil`"))) - .wrap_err("Failed to get `id`")?; - - let resources = ModConfigResources { - init: get_global(state, b"script\0") - .and_then(|s| s.map(PathBuf::from).ok_or_else(|| eyre::eyre!("Got `nil`"))) - .wrap_err("Failed to get `script`.")?, - data: get_global(state, b"data\0") - .wrap_err("Failed to get `data`.")? - .map(PathBuf::from), - localization: get_global(state, b"localization\0") - .wrap_err("Failed to get `localization`")? - .map(PathBuf::from), - }; - - lua::lua_close(state); - - (mod_id, resources) - }; - - Ok(ret) -} - -// Extracts the mod configuration from the mod archive. -// This may either be a proper `dtmt.cfg`, or the legacy `.mod` ID file. -// -// It also returns the directory where this file was found, used as root path. This -// allows flexibility in what the directory structure is exactly, since many people -// still end up creating tarbombs and Nexus does its own re-packaging. -#[tracing::instrument(skip(archive))] -fn extract_mod_config(archive: &mut ZipArchive) -> Result<(ModConfig, String)> { - let legacy_mod_data = if let Some(name) = find_archive_file(archive, ".mod") { - let (mod_id, resources) = { - let mut f = archive - .by_name(&name) - .wrap_err("Failed to read `.mod` file from archive")?; - - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read `.mod` file from archive")?; - - let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?; - parse_mod_id_file(&data) - .wrap_err("Invalid `.mod` file") - .note( - "The `.mod` file's `run` function may not contain any additional logic \ - besides the default.", - ) - .suggestion("Contact the mod author to fix this.")? - }; - - let root = if let Some(index) = name.rfind('/') { - name[..index].to_string() - } else { - String::new() - }; - - Some((mod_id, resources, root)) - } else { - None - }; - - if let Some(name) = find_archive_file(archive, "dtmt.cfg") { - let mut f = archive - .by_name(&name) - .wrap_err("Failed to read mod config from archive")?; - - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read mod config from archive")?; - - let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; - - let mut cfg: ModConfig = serde_sjson::from_str(&data) - .wrap_err("Failed to deserialize mod config") - .suggestion("Contact the mod author to fix this.")?; - - if let Some((mod_id, resources, root)) = legacy_mod_data { - if cfg.id != mod_id { - let err = eyre::eyre!("Mod ID in `dtmt.cfg` does not match mod ID in `.mod` file"); - return Err(err).suggestion("Contact the mod author to fix this."); - } - - cfg.resources = resources; - - Ok((cfg, root)) - } else { - let root = name - .strip_suffix("dtmt.cfg") - .expect("String must end with that suffix") - .to_string(); - - Ok((cfg, root)) - } - } else { - eyre::bail!( - "Mod needs a config file or `.mod` file. \ - Please get in touch with the author to provide a properly packaged mod." - ); - } -} - -#[tracing::instrument(skip(archive))] -fn extract_bundled_mod( - archive: &mut ZipArchive, - root: String, - dest: impl AsRef + std::fmt::Debug, -) -> Result>> { - let files: HashMap> = { - let name = archive - .file_names() - .find(|name| name.ends_with("files.sjson")) - .map(|s| s.to_string()) - .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; - - let mut f = archive - .by_name(&name) - .wrap_err("Failed to read file index from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; - serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? - }; - - tracing::trace!(?files); - - let dest = dest.as_ref(); - tracing::trace!("Extracting mod archive to {}", dest.display()); - archive - .extract(dest) - .wrap_err_with(|| format!("Failed to extract archive to {}", dest.display()))?; - - let packages = files - .into_iter() - .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) - .collect(); - - tracing::trace!(?packages); - - Ok(packages) -} - -#[tracing::instrument(skip(archive))] -fn extract_legacy_mod( - archive: &mut ZipArchive, - root: String, - dest: impl Into + std::fmt::Debug, -) -> Result<()> { - let dest = dest.into(); - let file_count = archive.len(); - - for i in 0..file_count { - let mut f = archive - .by_index(i) - .wrap_err_with(|| format!("Failed to get file at index {}", i))?; - - let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else { - let err = eyre::eyre!("File name in archive is not a safe path value.").suggestion( - "Only use well-known applications to create the ZIP archive, \ - and don't create paths that point outside the archive directory.", - ); - return Err(err); - }; - - let Ok(suffix) = name.strip_prefix(&root) else { - tracing::warn!( - "Skipping file outside of the mod root directory: {}", - name.display() - ); - continue; - }; - let name = dest.join(suffix); - - if f.is_dir() { - // The majority of errors will actually be "X already exists". - // But rather than filter them invidually, we just ignore all of them. - // If there is a legitimate error of "couldn't create X", it will eventually fail when - // we try to put a file in there. - tracing::trace!("Creating directory '{}'", name.display()); - let _ = std::fs::create_dir_all(&name); - } else { - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err_with(|| format!("Failed to read file '{}'", name.display()))?; - - tracing::trace!("Writing file '{}'", name.display()); - let mut out = std::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&name) - .wrap_err_with(|| format!("Failed to open file '{}'", name.display()))?; - - out.write_all(&buf) - .wrap_err_with(|| format!("Failed to write to '{}'", name.display()))?; - } - } - - Ok(()) -} - -#[tracing::instrument(skip(state))] -pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { - let data = fs::read(&info.path) - .await - .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; - let data = Cursor::new(data); - - let nexus = if let Some((_, id, version, _)) = info - .path - .file_name() - .and_then(|s| s.to_str()) - .and_then(NexusApi::parse_file_name) - { - if !state.nexus_api_key.is_empty() { - let api = NexusApi::new(state.nexus_api_key.to_string())?; - let mod_info = api - .mods_id(id) - .await - .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; - let info = NexusInfo::from(mod_info); - - tracing::debug!("{:?}", info); - Some((info, version)) - } else { - None - } - } else { - None - }; - - let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; - - if tracing::enabled!(tracing::Level::DEBUG) { - let names = archive.file_names().fold(String::new(), |mut s, name| { - s.push('\n'); - s.push_str(name); - s - }); - tracing::debug!("Archive contents:{}", names); - } - - let (mut mod_cfg, root) = - extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?; - tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id); - tracing::debug!(root, ?mod_cfg); - - let image = if let Some(path) = &mod_cfg.image { - let name = archive - .file_names() - .find(|name| name.ends_with(&path.display().to_string())) - .map(|s| s.to_string()) - .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; - - let mut f = archive - .by_name(&name) - .wrap_err("Failed to read image file from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - // Druid somehow doesn't return an error compatible with eyre, here. - // So we have to wrap through `Display` manually. - let img = match ImageBuf::from_data(&buf) { - Ok(img) => img, - Err(err) => { - let err = Report::msg(err.to_string()) - .wrap_err("Invalid image data") - .note("Supported formats are: PNG, JPEG, Bitmap and WebP") - .suggestion("Contact the mod author to fix this"); - return Err(err); - } - }; - - Some(img) - } else { - None - }; - - tracing::trace!(?image); - - let mod_dir = state.data_dir.join(state.mod_dir.as_ref()); - let dest = mod_dir.join(&mod_cfg.id); - - tracing::trace!("Creating mods directory {}", dest.display()); - fs::create_dir_all(&dest) - .await - .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?; - - let packages = if mod_cfg.bundled { - extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")? - } else { - extract_legacy_mod(&mut archive, root, &dest).wrap_err("Failed to extract legacy mod")?; - - if let Some((_, version)) = &nexus { - // We use the version number stored in the `ModInfo` to compare against the `NexusInfo` - // for version checks. So for this one, we can't actually rely on merely shadowing, - // like with the other fields. - mod_cfg.version = version.clone(); - } - - let data = serde_sjson::to_string(&mod_cfg).wrap_err("Failed to serialize mod config")?; - fs::write(dest.join("dtmt.cfg"), &data) - .await - .wrap_err("Failed to write mod config")?; - - Default::default() - }; - - if let Some((nexus, _)) = &nexus { - let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; - let path = dest.join("nexus.sjson"); - fs::write(&path, data.as_bytes()) - .await - .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; - } - - let info = ModInfo::new(mod_cfg, packages, image, nexus.map(|(info, _)| info)); - Ok(info) -} diff --git a/crates/dtmm/src/controller/mod.rs b/crates/dtmm/src/controller/mod.rs index 9c75e84..eacc3ef 100644 --- a/crates/dtmm/src/controller/mod.rs +++ b/crates/dtmm/src/controller/mod.rs @@ -5,9 +5,7 @@ use serde::Deserialize; use tokio::fs; pub mod app; -pub mod deploy; pub mod game; -pub mod import; pub mod worker; #[tracing::instrument] diff --git a/crates/dtmm/src/controller/worker.rs b/crates/dtmm/src/controller/worker.rs index 152030f..238b3f3 100644 --- a/crates/dtmm/src/controller/worker.rs +++ b/crates/dtmm/src/controller/worker.rs @@ -13,9 +13,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::RwLock; use crate::controller::app::*; -use crate::controller::deploy::deploy_mods; use crate::controller::game::*; -use crate::controller::import::import_mod; use crate::state::AsyncAction; use crate::state::ACTION_FINISH_CHECK_UPDATE; use crate::state::ACTION_FINISH_LOAD_INITIAL; @@ -38,9 +36,7 @@ async fn handle_action( action_queue: Arc>>, ) { while let Some(action) = action_queue.write().await.recv().await { - if cfg!(debug_assertions) && !matches!(action, AsyncAction::Log(_)) { - tracing::debug!(?action); - } + tracing::debug!(?action); let event_sink = event_sink.clone(); match action { diff --git a/crates/dtmm/src/main.rs b/crates/dtmm/src/main.rs index 8069e6d..9ba838c 100644 --- a/crates/dtmm/src/main.rs +++ b/crates/dtmm/src/main.rs @@ -1,7 +1,6 @@ #![recursion_limit = "256"] #![feature(let_chains)] #![feature(arc_unwrap_or_clone)] -#![feature(iterator_try_collect)] #![windows_subsystem = "windows"] use std::path::PathBuf; diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index 64fdd28..55774aa 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -73,7 +73,6 @@ impl From for ModDependency { #[derive(Clone, Data, Debug, Lens, serde::Serialize, serde::Deserialize)] pub(crate) struct NexusInfo { pub id: u64, - pub name: String, pub version: String, pub author: String, pub summary: Arc, @@ -84,7 +83,6 @@ impl From for NexusInfo { fn from(value: NexusMod) -> Self { Self { id: value.mod_id, - name: value.name, version: value.version, author: value.author, summary: Arc::new(value.summary), @@ -111,7 +109,6 @@ pub(crate) struct ModInfo { #[data(ignore)] pub resources: ModResourceInfo, pub depends: Vector, - pub bundled: bool, #[data(ignore)] pub nexus: Option, } @@ -132,7 +129,6 @@ impl ModInfo { version: cfg.version, enabled: false, packages, - bundled: cfg.bundled, image, categories: cfg.categories.into_iter().collect(), resources: ModResourceInfo { diff --git a/crates/dtmm/src/state/delegate.rs b/crates/dtmm/src/state/delegate.rs index 47d56e8..97c6e6b 100644 --- a/crates/dtmm/src/state/delegate.rs +++ b/crates/dtmm/src/state/delegate.rs @@ -411,7 +411,6 @@ impl AppDelegate for Delegate { state.config_path = Arc::new(config.path); state.data_dir = Arc::new(config.data_dir); state.game_dir = Arc::new(config.game_dir.unwrap_or_default()); - state.nexus_api_key = Arc::new(config.nexus_api_key.unwrap_or_default()); state.is_io_enabled = config.unsafe_io; } diff --git a/crates/dtmm/src/ui/window/main.rs b/crates/dtmm/src/ui/window/main.rs index baa8e22..7aa2819 100644 --- a/crates/dtmm/src/ui/window/main.rs +++ b/crates/dtmm/src/ui/window/main.rs @@ -135,13 +135,8 @@ fn build_mod_list() -> impl Widget { }) .lens(lens!((usize, Arc, bool), 1).then(ModInfo::enabled.in_arc())); - let name = Label::dynamic(|info: &Arc, _| { - info.nexus - .as_ref() - .map(|n| n.name.clone()) - .unwrap_or_else(|| info.name.clone()) - }) - .lens(lens!((usize, Arc, bool), 1)); + let name = + Label::raw().lens(lens!((usize, Arc, bool), 1).then(ModInfo::name.in_arc())); let version = { let icon = { @@ -150,7 +145,7 @@ fn build_mod_list() -> impl Widget { let tree = theme::icons::recolor_icon(tree, true, COLOR_YELLOW_LIGHT); - Svg::new(tree).fix_height(druid::theme::TEXT_SIZE_NORMAL) + Svg::new(Arc::new(tree)).fix_height(druid::theme::TEXT_SIZE_NORMAL) }; Either::new( diff --git a/crates/dtmm/src/util/config.rs b/crates/dtmm/src/util/config.rs index 9affbb6..3a0d0b2 100644 --- a/crates/dtmm/src/util/config.rs +++ b/crates/dtmm/src/util/config.rs @@ -125,9 +125,6 @@ where .wrap_err_with(|| format!("Invalid config file {}", path.display()))?; cfg.path = path; - - tracing::debug!("Read config file '{}': {:?}", cfg.path.display(), cfg); - Ok(cfg) } Err(err) if err.kind() == ErrorKind::NotFound => { @@ -136,11 +133,6 @@ where .wrap_err_with(|| format!("Failed to read config file {}", path.display()))?; } - tracing::debug!( - "Config file not found at '{}', creating default.", - path.display() - ); - { let parent = default_path .parent() diff --git a/crates/dtmt/src/cmd/dictionary.rs b/crates/dtmt/src/cmd/dictionary.rs index 62a5295..0400519 100644 --- a/crates/dtmt/src/cmd/dictionary.rs +++ b/crates/dtmt/src/cmd/dictionary.rs @@ -145,10 +145,7 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( .get_one::("group") .expect("required argument not found"); - let r: BufReader> = if let Some(name) = - path.file_name() - && name == "-" - { + let r: BufReader> = if let Some(name) = path.file_name() && name == "-" { let f = tokio::io::stdin(); BufReader::new(Box::new(f)) } else { diff --git a/crates/dtmt/src/cmd/migrate.rs b/crates/dtmt/src/cmd/migrate.rs index 2eacded..1a4d605 100644 --- a/crates/dtmt/src/cmd/migrate.rs +++ b/crates/dtmt/src/cmd/migrate.rs @@ -350,7 +350,6 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> localization: mod_file.localization, }, depends: vec![ModDependency::ID(String::from("DMF"))], - bundled: true, }; tracing::debug!(?dtmt_cfg); diff --git a/lib/dtmt-shared/src/lib.rs b/lib/dtmt-shared/src/lib.rs index 28e4694..3907928 100644 --- a/lib/dtmt-shared/src/lib.rs +++ b/lib/dtmt-shared/src/lib.rs @@ -30,17 +30,6 @@ pub enum ModDependency { Config { id: String, order: ModOrder }, } -// A bit dumb, but serde doesn't support literal values with the -// `default` attribute, only paths. -fn default_true() -> bool { - true -} - -// Similarly dumb, as the `skip_serializing_if` attribute needs a function -fn is_true(val: &bool) -> bool { - *val -} - #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ModConfig { #[serde(skip)] @@ -62,8 +51,6 @@ pub struct ModConfig { pub resources: ModConfigResources, #[serde(default)] pub depends: Vec, - #[serde(default = "default_true", skip_serializing_if = "is_true")] - pub bundled: bool, } pub const STEAMAPP_ID: u32 = 1361210; diff --git a/lib/nexusmods/src/lib.rs b/lib/nexusmods/src/lib.rs index 145435d..a0700ae 100644 --- a/lib/nexusmods/src/lib.rs +++ b/lib/nexusmods/src/lib.rs @@ -112,7 +112,7 @@ impl Api { RE.captures(name.as_ref()).and_then(|cap| { let name = cap.name("name").map(|s| s.as_str().to_string())?; let mod_id = cap.name("mod_id").and_then(|s| s.as_str().parse().ok())?; - let version = cap.name("version").map(|s| s.as_str().replace('-', "."))?; + let version = cap.name("version").map(|s| s.as_str().to_string())?; let updated = cap .name("updated") .and_then(|s| s.as_str().parse().ok()) @@ -154,7 +154,7 @@ impl Api { self.mods_download_link(nxm.mod_id, nxm.file_id, nxm.key, nxm.expires) )?; - let Some(download_url) = download_info.first().map(|i| i.uri.clone()) else { + let Some(download_url) = download_info.get(0).map(|i| i.uri.clone()) else { return Err(Error::InvalidNXM("no download link", url)); }; @@ -215,7 +215,7 @@ impl Api { }; let user_id = query.get("user_id").and_then(|id| id.parse().ok()); - let Some(user_id) = user_id else { + let Some(user_id) = user_id else { return Err(Error::InvalidNXM("Missing 'user_id'", nxm)); }; diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 813df06..96a2ec2 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -227,10 +227,13 @@ impl Bundle { let _enter = span.enter(); tracing::trace!(num_files = self.files.len()); - self.files.iter().try_fold(Vec::new(), |mut data, file| { - data.append(&mut file.to_binary()?); - Ok::<_, Report>(data) - })? + self.files + .iter() + .fold(Ok::, Report>(Vec::new()), |data, file| { + let mut data = data?; + data.append(&mut file.to_binary()?); + Ok(data) + })? }; // Ceiling division (or division toward infinity) to calculate diff --git a/lib/sdk/src/filetype/lua.rs b/lib/sdk/src/filetype/lua.rs index bfe6de7..2458dec 100644 --- a/lib/sdk/src/filetype/lua.rs +++ b/lib/sdk/src/filetype/lua.rs @@ -53,8 +53,8 @@ where let mut buf = vec![0u8; length]; r.read_exact(&mut buf)?; - let mut s = - String::from_utf8(buf).wrap_err("Invalid byte sequence for LuaJIT bytecode name")?; + let mut s = String::from_utf8(buf) + .wrap_err_with(|| format!("Invalid byte sequence for LuaJIT bytecode name"))?; // Remove the leading `@` s.remove(0); s