From e52c2b4cff0e10ddfcfc785f6ff399f97d8c23b8 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 8 Nov 2023 15:04:32 +0100 Subject: [PATCH 01/13] Add mod config option for loose files Just the field in the config file, for now. --- crates/dtmm/src/state/data.rs | 2 ++ crates/dtmt/src/cmd/migrate.rs | 1 + lib/dtmt-shared/src/lib.rs | 13 +++++++++++++ 3 files changed, 16 insertions(+) diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index 55774aa..c43e973 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -109,6 +109,7 @@ pub(crate) struct ModInfo { #[data(ignore)] pub resources: ModResourceInfo, pub depends: Vector, + pub bundle: bool, #[data(ignore)] pub nexus: Option, } @@ -129,6 +130,7 @@ impl ModInfo { version: cfg.version, enabled: false, packages, + bundle: cfg.bundle, image, categories: cfg.categories.into_iter().collect(), resources: ModResourceInfo { diff --git a/crates/dtmt/src/cmd/migrate.rs b/crates/dtmt/src/cmd/migrate.rs index 1a4d605..8ecd648 100644 --- a/crates/dtmt/src/cmd/migrate.rs +++ b/crates/dtmt/src/cmd/migrate.rs @@ -350,6 +350,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> localization: mod_file.localization, }, depends: vec![ModDependency::ID(String::from("DMF"))], + bundle: true, }; tracing::debug!(?dtmt_cfg); diff --git a/lib/dtmt-shared/src/lib.rs b/lib/dtmt-shared/src/lib.rs index 3907928..a162d3b 100644 --- a/lib/dtmt-shared/src/lib.rs +++ b/lib/dtmt-shared/src/lib.rs @@ -30,6 +30,17 @@ pub enum ModDependency { Config { id: String, order: ModOrder }, } +// A bit dumb, but serde doesn't support literal values with the +// `default` attribute, only paths. +fn default_true() -> bool { + true +} + +// Similarly dumb, as the `skip_serializing_if` attribute needs a function +fn is_true(val: &bool) -> bool { + *val +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ModConfig { #[serde(skip)] @@ -51,6 +62,8 @@ pub struct ModConfig { pub resources: ModConfigResources, #[serde(default)] pub depends: Vec, + #[serde(default = "default_true", skip_serializing_if = "is_true")] + pub bundle: bool, } pub const STEAMAPP_ID: u32 = 1361210; -- 2.45.3 From 8715cf5309c83c1fd34ae360d5a2cca63b3f91d3 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 8 Nov 2023 15:06:53 +0100 Subject: [PATCH 02/13] Apply formatting --- crates/dtmm/src/controller/game.rs | 8 ++++---- crates/dtmt/src/cmd/dictionary.rs | 5 ++++- lib/nexusmods/src/lib.rs | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/dtmm/src/controller/game.rs b/crates/dtmm/src/controller/game.rs index a5f6fa8..5520c4a 100644 --- a/crates/dtmm/src/controller/game.rs +++ b/crates/dtmm/src/controller/game.rs @@ -583,12 +583,12 @@ pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { }, async { let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - match read_sjson_file::<_, DeploymentData>(path) - .await - { + match read_sjson_file::<_, DeploymentData>(path).await { Ok(data) => Ok(Some(data)), Err(err) => { - if let Some(err) = err.downcast_ref::() && err.kind() == ErrorKind::NotFound { + if let Some(err) = err.downcast_ref::() + && err.kind() == ErrorKind::NotFound + { Ok(None) } else { Err(err).wrap_err("Failed to read deployment data") diff --git a/crates/dtmt/src/cmd/dictionary.rs b/crates/dtmt/src/cmd/dictionary.rs index 0400519..62a5295 100644 --- a/crates/dtmt/src/cmd/dictionary.rs +++ b/crates/dtmt/src/cmd/dictionary.rs @@ -145,7 +145,10 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<( .get_one::("group") .expect("required argument not found"); - let r: BufReader> = if let Some(name) = path.file_name() && name == "-" { + let r: BufReader> = if let Some(name) = + path.file_name() + && name == "-" + { let f = tokio::io::stdin(); BufReader::new(Box::new(f)) } else { diff --git a/lib/nexusmods/src/lib.rs b/lib/nexusmods/src/lib.rs index a0700ae..3dbbecd 100644 --- a/lib/nexusmods/src/lib.rs +++ b/lib/nexusmods/src/lib.rs @@ -215,7 +215,7 @@ impl Api { }; let user_id = query.get("user_id").and_then(|id| id.parse().ok()); - let Some(user_id) = user_id else { + let Some(user_id) = user_id else { return Err(Error::InvalidNXM("Missing 'user_id'", nxm)); }; -- 2.45.3 From 266d63c20d4eb9865e75ccc1cf75ff84375e4136 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 9 Nov 2023 19:48:05 +0100 Subject: [PATCH 03/13] Apply clippy lints --- lib/nexusmods/src/lib.rs | 2 +- lib/sdk/src/bundle/mod.rs | 11 ++++------- lib/sdk/src/filetype/lua.rs | 4 ++-- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/nexusmods/src/lib.rs b/lib/nexusmods/src/lib.rs index 3dbbecd..c305db0 100644 --- a/lib/nexusmods/src/lib.rs +++ b/lib/nexusmods/src/lib.rs @@ -154,7 +154,7 @@ impl Api { self.mods_download_link(nxm.mod_id, nxm.file_id, nxm.key, nxm.expires) )?; - let Some(download_url) = download_info.get(0).map(|i| i.uri.clone()) else { + let Some(download_url) = download_info.first().map(|i| i.uri.clone()) else { return Err(Error::InvalidNXM("no download link", url)); }; diff --git a/lib/sdk/src/bundle/mod.rs b/lib/sdk/src/bundle/mod.rs index 96a2ec2..813df06 100644 --- a/lib/sdk/src/bundle/mod.rs +++ b/lib/sdk/src/bundle/mod.rs @@ -227,13 +227,10 @@ impl Bundle { let _enter = span.enter(); tracing::trace!(num_files = self.files.len()); - self.files - .iter() - .fold(Ok::, Report>(Vec::new()), |data, file| { - let mut data = data?; - data.append(&mut file.to_binary()?); - Ok(data) - })? + self.files.iter().try_fold(Vec::new(), |mut data, file| { + data.append(&mut file.to_binary()?); + Ok::<_, Report>(data) + })? }; // Ceiling division (or division toward infinity) to calculate diff --git a/lib/sdk/src/filetype/lua.rs b/lib/sdk/src/filetype/lua.rs index 2458dec..bfe6de7 100644 --- a/lib/sdk/src/filetype/lua.rs +++ b/lib/sdk/src/filetype/lua.rs @@ -53,8 +53,8 @@ where let mut buf = vec![0u8; length]; r.read_exact(&mut buf)?; - let mut s = String::from_utf8(buf) - .wrap_err_with(|| format!("Invalid byte sequence for LuaJIT bytecode name"))?; + let mut s = + String::from_utf8(buf).wrap_err("Invalid byte sequence for LuaJIT bytecode name")?; // Remove the leading `@` s.remove(0); s -- 2.45.3 From 3d7d30162798f2431d8c578536c9bbd4e7c5da2a Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 9 Nov 2023 20:10:54 +0100 Subject: [PATCH 04/13] Implement deploying non-bundled mods Closes #113. --- Cargo.lock | 50 +- Cargo.toml | 5 +- crates/dtmm/Cargo.toml | 4 +- crates/dtmm/src/controller/app.rs | 170 +----- crates/dtmm/src/controller/deploy.rs | 875 +++++++++++++++++++++++++++ crates/dtmm/src/controller/game.rs | 622 +------------------ crates/dtmm/src/controller/import.rs | 475 +++++++++++++++ crates/dtmm/src/controller/mod.rs | 2 + crates/dtmm/src/controller/worker.rs | 2 + crates/dtmm/src/main.rs | 1 + crates/dtmm/src/state/data.rs | 4 +- crates/dtmm/src/ui/window/main.rs | 2 +- crates/dtmt/src/cmd/migrate.rs | 2 +- lib/dtmt-shared/src/lib.rs | 2 +- 14 files changed, 1396 insertions(+), 820 deletions(-) create mode 100644 crates/dtmm/src/controller/deploy.rs create mode 100644 crates/dtmm/src/controller/import.rs diff --git a/Cargo.lock b/Cargo.lock index a7885f2..011fa6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,12 +39,10 @@ dependencies = [ [[package]] name = "ansi-parser" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127" +version = "0.9.0" dependencies = [ "heapless", - "nom 4.2.3", + "nom", ] [[package]] @@ -381,7 +379,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom 7.1.3", + "nom", ] [[package]] @@ -896,6 +894,7 @@ name = "dtmm" version = "0.1.0" dependencies = [ "ansi-parser", + "async-recursion", "bitflags 1.3.2", "clap", "color-eyre", @@ -906,6 +905,7 @@ dependencies = [ "dtmt-shared", "futures", "lazy_static", + "luajit2-sys", "nexusmods", "oodle", "path-slash", @@ -1390,7 +1390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -1614,12 +1614,12 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "heapless" -version = "0.5.6" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1" +checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422" dependencies = [ "as-slice", - "generic-array 0.13.3", + "generic-array 0.14.7", "hash32", "stable_deref_trait", ] @@ -1747,7 +1747,7 @@ dependencies = [ "serde", "sized-chunks", "typenum", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -2175,16 +2175,6 @@ dependencies = [ "memoffset 0.6.5", ] -[[package]] -name = "nom" -version = "4.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" -dependencies = [ - "memchr", - "version_check 0.1.5", -] - [[package]] name = "nom" version = "7.1.3" @@ -2203,7 +2193,7 @@ checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3" dependencies = [ "bytecount", "memchr", - "nom 7.1.3", + "nom", ] [[package]] @@ -2673,7 +2663,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -2684,7 +2674,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2", "quote", - "version_check 0.9.4", + "version_check", ] [[package]] @@ -3102,7 +3092,7 @@ dependencies = [ name = "serde_sjson" version = "1.0.0" dependencies = [ - "nom 7.1.3", + "nom", "nom_locate", "serde", ] @@ -3832,7 +3822,7 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ - "version_check 0.9.4", + "version_check", ] [[package]] @@ -3966,12 +3956,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29" -[[package]] -name = "version_check" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" - [[package]] name = "version_check" version = "0.9.4" @@ -4384,7 +4368,3 @@ dependencies = [ "cc", "pkg-config", ] - -[[patch.unused]] -name = "ansi-parser" -version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index 123b2ef..451cd09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,10 +7,7 @@ members = [ "lib/oodle", "lib/sdk", "lib/serde_sjson", -] -exclude = [ - "lib/color-eyre", - "lib/ansi-parser", + "lib/luajit2-sys", ] [patch.crates-io] diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 86444be..1842a62 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -31,5 +31,7 @@ lazy_static = "1.4.0" colors-transform = "0.2.11" usvg = "0.25.0" druid-widget-nursery = "0.1" -ansi-parser = "0.8.0" +ansi-parser = "0.9.0" string_template = "0.2.1" +luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } +async-recursion = "1.0.5" diff --git a/crates/dtmm/src/controller/app.rs b/crates/dtmm/src/controller/app.rs index 5c5d980..cb6e2f0 100644 --- a/crates/dtmm/src/controller/app.rs +++ b/crates/dtmm/src/controller/app.rs @@ -1,18 +1,17 @@ use std::collections::HashMap; -use std::io::{Cursor, ErrorKind, Read}; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::Arc; use color_eyre::eyre::{self, Context}; use color_eyre::{Help, Report, Result}; use druid::im::Vector; -use druid::{FileInfo, ImageBuf}; +use druid::ImageBuf; use dtmt_shared::ModConfig; use nexusmods::Api as NexusApi; use tokio::fs::{self, DirEntry, File}; use tokio_stream::wrappers::ReadDirStream; use tokio_stream::StreamExt; -use zip::ZipArchive; use crate::state::{ActionState, InitialLoadResult, ModInfo, ModOrder, NexusInfo, PackageInfo}; use crate::util; @@ -20,161 +19,6 @@ use crate::util::config::{ConfigSerialize, LoadOrderEntry}; use super::read_sjson_file; -#[tracing::instrument(skip(state))] -pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { - let data = fs::read(&info.path) - .await - .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; - let data = Cursor::new(data); - - let nexus = if let Some((_, id, _, _)) = info - .path - .file_name() - .and_then(|s| s.to_str()) - .and_then(NexusApi::parse_file_name) - { - if !state.nexus_api_key.is_empty() { - let api = NexusApi::new(state.nexus_api_key.to_string())?; - let mod_info = api - .mods_id(id) - .await - .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; - Some(NexusInfo::from(mod_info)) - } else { - None - } - } else { - None - }; - - let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; - - if tracing::enabled!(tracing::Level::DEBUG) { - let names = archive.file_names().fold(String::new(), |mut s, name| { - s.push('\n'); - s.push_str(name); - s - }); - tracing::debug!("Archive contents:{}", names); - } - - let dir_name = { - let f = archive.by_index(0).wrap_err("Archive is empty")?; - - if !f.is_dir() { - let err = eyre::eyre!("archive does not have a top-level directory"); - return Err(err).with_suggestion(|| "Use 'dtmt build' to create the mod archive."); - } - - let name = f.name(); - // The directory name is returned with a trailing slash, which we don't want - name[..(name.len().saturating_sub(1))].to_string() - }; - - tracing::info!("Importing mod {}", dir_name); - - let names: Vec<_> = archive.file_names().map(|s| s.to_string()).collect(); - - let mod_cfg: ModConfig = { - let name = names - .iter() - .find(|name| name.ends_with("dtmt.cfg")) - .ok_or_else(|| eyre::eyre!("archive does not contain mod config"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read mod config from archive")?; - - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read mod config from archive")?; - - let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; - - serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")? - }; - - tracing::debug!(?mod_cfg); - - let files: HashMap> = { - let name = names - .iter() - .find(|name| name.ends_with("files.sjson")) - .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read file index from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; - - serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? - }; - - tracing::trace!(?files); - - let image = if let Some(path) = &mod_cfg.image { - let name = names - .iter() - .find(|name| name.ends_with(&path.display().to_string())) - .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; - - let mut f = archive - .by_name(name) - .wrap_err("Failed to read image file from archive")?; - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read file index from archive")?; - - // Druid somehow doesn't return an error compatible with eyre, here. - // So we have to wrap through `Display` manually. - let img = match ImageBuf::from_data(&buf) { - Ok(img) => img, - Err(err) => { - let err = Report::msg(err.to_string()).wrap_err("Invalid image data"); - return Err(err).with_suggestion(|| { - "Supported formats are: PNG, JPEG, Bitmap and WebP".to_string() - }); - } - }; - - Some(img) - } else { - None - }; - - let mod_dir = state.mod_dir; - - tracing::trace!("Creating mods directory {}", mod_dir.display()); - fs::create_dir_all(Arc::as_ref(&mod_dir)) - .await - .wrap_err_with(|| format!("Failed to create data directory {}", mod_dir.display()))?; - - tracing::trace!("Extracting mod archive to {}", mod_dir.display()); - archive - .extract(Arc::as_ref(&mod_dir)) - .wrap_err_with(|| format!("Failed to extract archive to {}", mod_dir.display()))?; - - if let Some(nexus) = &nexus { - let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; - let path = mod_dir.join(&mod_cfg.id).join("nexus.sjson"); - fs::write(&path, data.as_bytes()) - .await - .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; - } - - let packages = files - .into_iter() - .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) - .collect(); - let info = ModInfo::new(mod_cfg, packages, image, nexus); - - Ok(info) -} - #[tracing::instrument(skip(state))] pub(crate) async fn delete_mod(state: ActionState, info: &ModInfo) -> Result<()> { let mod_dir = state.mod_dir.join(&info.id); @@ -229,9 +73,13 @@ async fn read_mod_dir_entry(res: Result) -> Result { Err(err) => return Err(err), }; - let files: HashMap> = read_sjson_file(&index_path) - .await - .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?; + let files: HashMap> = if cfg.bundled { + read_sjson_file(&index_path) + .await + .wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))? + } else { + Default::default() + }; let image = if let Some(path) = &cfg.image { let path = entry.path().join(path); diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs new file mode 100644 index 0000000..6a804e7 --- /dev/null +++ b/crates/dtmm/src/controller/deploy.rs @@ -0,0 +1,875 @@ +use std::collections::HashMap; +use std::io::{Cursor, ErrorKind}; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; + +use color_eyre::eyre::Context; +use color_eyre::{eyre, Help, Report, Result}; +use futures::StreamExt; +use futures::{stream, TryStreamExt}; +use path_slash::PathBufExt; +use sdk::filetype::lua; +use sdk::filetype::package::Package; +use sdk::murmur::Murmur64; +use sdk::{ + Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, +}; +use serde::{Deserialize, Serialize}; +use string_template::Template; +use time::OffsetDateTime; +use tokio::fs::{self, DirEntry}; +use tokio::io::AsyncWriteExt; +use tracing::Instrument; + +use super::read_sjson_file; +use crate::controller::app::check_mod_order; +use crate::state::{ActionState, PackageInfo}; + +pub const MOD_BUNDLE_NAME: &str = "packages/mods"; +pub const BOOT_BUNDLE_NAME: &str = "packages/boot"; +pub const DML_BUNDLE_NAME: &str = "packages/dml"; +pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; +pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; +pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; +pub const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; +pub const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeploymentData { + pub bundles: Vec, + pub mod_folders: Vec, + #[serde(with = "time::serde::iso8601")] + pub timestamp: OffsetDateTime, +} + +#[tracing::instrument] +async fn read_file_with_backup

(path: P) -> Result> +where + P: AsRef + std::fmt::Debug, +{ + let path = path.as_ref(); + let backup_path = { + let mut p = PathBuf::from(path); + let ext = if let Some(ext) = p.extension() { + ext.to_string_lossy().to_string() + ".bak" + } else { + String::from("bak") + }; + p.set_extension(ext); + p + }; + + let file_name = path + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| String::from("file")); + + let bin = match fs::read(&backup_path).await { + Ok(bin) => bin, + Err(err) if err.kind() == ErrorKind::NotFound => { + // TODO: This doesn't need to be awaited here, yet. + // I only need to make sure it has finished before writing the changed bundle. + tracing::debug!( + "Backup does not exist. Backing up original {} to '{}'", + file_name, + backup_path.display() + ); + fs::copy(path, &backup_path).await.wrap_err_with(|| { + format!( + "Failed to back up {} '{}' to '{}'", + file_name, + path.display(), + backup_path.display() + ) + })?; + + tracing::debug!("Reading {} from original '{}'", file_name, path.display()); + fs::read(path).await.wrap_err_with(|| { + format!("Failed to read {} file: {}", file_name, path.display()) + })? + } + Err(err) => { + return Err(err).wrap_err_with(|| { + format!( + "Failed to read {} from backup '{}'", + file_name, + backup_path.display() + ) + }); + } + }; + Ok(bin) +} + +#[tracing::instrument(skip_all)] +async fn patch_game_settings(state: Arc) -> Result<()> { + let settings_path = state.game_dir.join("bundle").join(SETTINGS_FILE_PATH); + + let settings = read_file_with_backup(&settings_path) + .await + .wrap_err("Failed to read settings.ini")?; + let settings = String::from_utf8(settings).wrap_err("Settings.ini is not valid UTF-8")?; + + let mut f = fs::File::create(&settings_path) + .await + .wrap_err_with(|| format!("Failed to open {}", settings_path.display()))?; + + let Some(i) = settings.find("boot_script =") else { + eyre::bail!("couldn't find 'boot_script' field"); + }; + + f.write_all(settings[0..i].as_bytes()).await?; + f.write_all(b"boot_script = \"scripts/mod_main\"").await?; + + let Some(j) = settings[i..].find('\n') else { + eyre::bail!("couldn't find end of 'boot_script' field"); + }; + + f.write_all(settings[(i + j)..].as_bytes()).await?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(package = info.name))] +fn make_package(info: &PackageInfo) -> Result { + let mut pkg = Package::new(info.name.clone(), PathBuf::new()); + + for f in &info.files { + let mut it = f.rsplit('.'); + let file_type = it + .next() + .ok_or_else(|| eyre::eyre!("missing file extension")) + .and_then(BundleFileType::from_str) + .wrap_err("Invalid file name in package info")?; + let name: String = it.collect(); + pkg.add_file(file_type, name); + } + + Ok(pkg) +} + +#[tracing::instrument] +async fn copy_recursive( + from: impl Into + std::fmt::Debug, + to: impl AsRef + std::fmt::Debug, +) -> Result<()> { + let to = to.as_ref(); + + #[tracing::instrument] + async fn handle_dir(from: PathBuf) -> Result> { + let mut dir = fs::read_dir(&from) + .await + .wrap_err("Failed to read directory")?; + let mut entries = Vec::new(); + + while let Some(entry) = dir.next_entry().await? { + let meta = entry.metadata().await.wrap_err_with(|| { + format!("Failed to get metadata for '{}'", entry.path().display()) + })?; + entries.push((meta.is_dir(), entry)); + } + + Ok(entries) + } + + let base = from.into(); + stream::unfold(vec![base.clone()], |mut state| async { + let from = state.pop()?; + let inner = match handle_dir(from).await { + Ok(entries) => { + for (is_dir, entry) in &entries { + if *is_dir { + state.push(entry.path()); + } + } + stream::iter(entries).map(Ok).left_stream() + } + Err(e) => stream::once(async { Err(e) }).right_stream(), + }; + + Some((inner, state)) + }) + .flatten() + .try_for_each(|(is_dir, entry)| { + let path = entry.path(); + let dest = path + .strip_prefix(&base) + .map(|suffix| to.join(suffix)) + .expect("all entries are relative to the directory we are walking"); + + async move { + if is_dir { + tracing::trace!("Creating directory '{}'", dest.display()); + fs::create_dir(&dest) + .await + .map(|_| ()) + .wrap_err_with(|| format!("Failed to create directory '{}'", dest.display())) + } else { + tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display()); + fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| { + format!( + "Failed to copy file '{}' -> '{}'", + path.display(), + dest.display() + ) + }) + } + } + }) + .await + .map(|_| ()) +} + +#[tracing::instrument(skip(state))] +async fn copy_mod_folders(state: Arc) -> Result> { + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + + let mut tasks = Vec::new(); + + for mod_info in state + .mods + .iter() + .filter(|m| m.id != "dml" && m.enabled && !m.bundled) + { + let span = tracing::trace_span!("copying legacy mod", name = mod_info.name); + let _enter = span.enter(); + + let mod_id = mod_info.id.clone(); + let mod_dir = Arc::clone(&state.mod_dir); + let bundle_dir = Arc::clone(&bundle_dir); + + let task = async move { + let from = mod_dir.join(&mod_id); + let to = bundle_dir.join("mods").join(&mod_id); + + tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id); + let _ = fs::create_dir_all(&to).await; + copy_recursive(&from, &to).await.wrap_err_with(|| { + format!( + "Failed to copy legacy mod from '{}' to '{}'", + from.display(), + to.display() + ) + })?; + + Ok::<_, Report>(mod_id) + }; + tasks.push(task); + } + + let ids = futures::future::try_join_all(tasks).await?; + Ok(ids) +} + +fn build_mod_data_lua(state: Arc) -> String { + let mut lua = String::from("return {\n"); + + // DMF is handled explicitely by the loading procedures, as it actually drives most of that + // and should therefore not show up in the load order. + for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { + lua.push_str(" {\n name = \""); + lua.push_str(&mod_info.name); + + lua.push_str("\",\n id = \""); + lua.push_str(&mod_info.id); + + lua.push_str("\",\n bundled = \""); + if mod_info.bundled { + lua.push_str("true"); + } else { + lua.push_str("false"); + } + + lua.push_str("\",\n run = function()\n"); + + let resources = &mod_info.resources; + if resources.data.is_some() || resources.localization.is_some() { + lua.push_str(" new_mod(\""); + lua.push_str(&mod_info.id); + lua.push_str("\", {\n mod_script = \""); + lua.push_str(&resources.init.to_slash_lossy()); + + if let Some(data) = resources.data.as_ref() { + lua.push_str("\",\n mod_data = \""); + lua.push_str(&data.to_slash_lossy()); + } + + if let Some(localization) = &resources.localization { + lua.push_str("\",\n mod_localization = \""); + lua.push_str(&localization.to_slash_lossy()); + } + + lua.push_str("\",\n })\n"); + } else { + lua.push_str(" return dofile(\""); + lua.push_str(&resources.init.to_slash_lossy()); + lua.push_str("\")\n"); + } + + lua.push_str(" end,\n packages = {\n"); + + for pkg_info in &mod_info.packages { + lua.push_str(" \""); + lua.push_str(&pkg_info.name); + lua.push_str("\",\n"); + } + + lua.push_str(" },\n },\n"); + } + + lua.push('}'); + + tracing::debug!("mod_data_lua:\n{}", lua); + + lua +} + +#[tracing::instrument(skip_all)] +async fn build_bundles(state: Arc) -> Result> { + let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); + let mut tasks = Vec::new(); + + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + + let mut bundles = Vec::new(); + + { + tracing::trace!("Building mod data script"); + + let span = tracing::debug_span!("Building mod data script"); + let _enter = span.enter(); + + let lua = build_mod_data_lua(state.clone()); + + tracing::trace!("Compiling mod data script"); + + let file = + lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?; + + tracing::trace!("Compile mod data script"); + + mod_bundle.add_file(file); + } + + tracing::trace!("Preparing tasks to deploy bundle files"); + + for mod_info in state + .mods + .iter() + .filter(|m| m.id != "dml" && m.enabled && m.bundled) + { + let span = tracing::trace_span!("building mod packages", name = mod_info.name); + let _enter = span.enter(); + + let mod_dir = state.mod_dir.join(&mod_info.id); + for pkg_info in &mod_info.packages { + let span = tracing::trace_span!("building package", name = pkg_info.name); + let _enter = span.enter(); + + tracing::trace!( + "Building package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; + let mut variant = BundleFileVariant::new(); + let bin = pkg + .to_binary() + .wrap_err("Failed to serialize package to binary")?; + variant.set_data(bin); + let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); + file.add_variant(variant); + + tracing::trace!( + "Compiled package {} for mod {}", + pkg_info.name, + mod_info.name + ); + + mod_bundle.add_file(file); + + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = mod_dir.join(&bundle_name); + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + // Explicitely drop the guard, so that we can move the span + // into the async operation + drop(_enter); + + let ctx = state.ctx.clone(); + + let task = async move { + let bundle = { + let bin = fs::read(&src).await.wrap_err_with(|| { + format!("Failed to read bundle file '{}'", src.display()) + })?; + let name = Bundle::get_name_from_path(&ctx, &src); + Bundle::from_binary(&ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? + }; + + tracing::debug!( + src = %src.display(), + dest = %dest.display(), + "Copying bundle '{}' for mod '{}'", + pkg_name, + mod_name, + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + + Ok::(bundle) + } + .instrument(span); + + tasks.push(task); + } + } + + tracing::debug!("Copying {} mod bundles", tasks.len()); + + let mut tasks = stream::iter(tasks).buffer_unordered(10); + + while let Some(res) = tasks.next().await { + let bundle = res?; + bundles.push(bundle); + } + + { + let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); + tracing::trace!("Writing mod bundle to '{}'", path.display()); + fs::write(&path, mod_bundle.to_binary()?) + .await + .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; + } + + bundles.push(mod_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all)] +async fn patch_boot_bundle(state: Arc) -> Result> { + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); + + let mut bundles = Vec::with_capacity(2); + + let mut boot_bundle = async { + let bin = read_file_with_backup(&bundle_path) + .await + .wrap_err("Failed to read boot bundle")?; + + Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) + .wrap_err("Failed to parse boot bundle") + } + .instrument(tracing::trace_span!("read boot bundle")) + .await + .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; + + { + tracing::trace!("Adding mod package file to boot bundle"); + let span = tracing::trace_span!("create mod package file"); + let _enter = span.enter(); + + let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); + + for mod_info in &state.mods { + for pkg_info in &mod_info.packages { + pkg.add_file(BundleFileType::Package, &pkg_info.name); + } + } + + pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); + + let mut variant = BundleFileVariant::new(); + variant.set_data(pkg.to_binary()?); + let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + tracing::trace!("Handling DML packages and bundle"); + let span = tracing::trace_span!("handle DML"); + let _enter = span.enter(); + + let mut variant = BundleFileVariant::new(); + + let mod_info = state + .mods + .iter() + .find(|m| m.id == "dml") + .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; + let pkg_info = mod_info + .packages + .get(0) + .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) + .with_suggestion(|| "Re-download and import the newest version.".to_string())?; + let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); + let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); + + { + let bin = fs::read(&src) + .await + .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; + let name = Bundle::get_name_from_path(&state.ctx, &src); + + let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) + .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; + + bundles.push(dml_bundle); + }; + + { + let dest = bundle_dir.join(&bundle_name); + let pkg_name = pkg_info.name.clone(); + let mod_name = mod_info.name.clone(); + + tracing::debug!( + "Copying bundle {} for mod {}: {} -> {}", + pkg_name, + mod_name, + src.display(), + dest.display() + ); + // We attempt to remove any previous file, so that the hard link can be created. + // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy + // may be possible despite an error here, or the error will be reported by it anyways. + // TODO: There is a chance that we delete an actual game bundle, but with 64bit + // hashes, it's low enough for now, and the setup required to detect + // "game bundle vs mod bundle" is non-trivial. + let _ = fs::remove_file(&dest).await; + fs::copy(&src, &dest).await.wrap_err_with(|| { + format!( + "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", + src.display(), + dest.display() + ) + })?; + } + + let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; + variant.set_data(pkg.to_binary()?); + + let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); + f.add_variant(variant); + + boot_bundle.add_file(f); + } + + { + let span = tracing::debug_span!("Importing mod main script"); + let _enter = span.enter(); + + let is_io_enabled = format!("{}", state.is_io_enabled); + let mut data = HashMap::new(); + data.insert("is_io_enabled", is_io_enabled.as_str()); + + let tmpl = include_str!("../../assets/mod_main.lua"); + let lua = Template::new(tmpl).render(&data); + tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); + let file = + lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; + + boot_bundle.add_file(file); + } + + async { + let bin = boot_bundle + .to_binary() + .wrap_err("Failed to serialize boot bundle")?; + fs::write(&bundle_path, bin) + .await + .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) + } + .instrument(tracing::trace_span!("write boot bundle")) + .await?; + + bundles.push(boot_bundle); + + Ok(bundles) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); + + let mut db = { + let bin = read_file_with_backup(&database_path) + .await + .wrap_err("Failed to read bundle database")?; + let mut r = Cursor::new(bin); + let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; + tracing::trace!("Finished parsing bundle database"); + db + }; + + for bundle in bundles.as_ref() { + tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); + db.add_bundle(bundle); + } + + { + let bin = db + .to_binary() + .wrap_err("Failed to serialize bundle database")?; + fs::write(&database_path, bin).await.wrap_err_with(|| { + format!( + "failed to write bundle database to '{}'", + database_path.display() + ) + })?; + } + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] +async fn write_deployment_data( + state: Arc, + bundles: B, + mod_folders: Vec, +) -> Result<()> +where + B: AsRef<[Bundle]>, +{ + let info = DeploymentData { + timestamp: OffsetDateTime::now_utc(), + bundles: bundles + .as_ref() + .iter() + .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) + .collect(), + // TODO: + mod_folders, + }; + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; + + fs::write(&path, &data) + .await + .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; + + Ok(()) +} + +#[tracing::instrument(skip_all, fields( + game_dir = %state.game_dir.display(), + mods = state.mods.len() +))] +pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { + let state = Arc::new(state); + let bundle_dir = state.game_dir.join("bundle"); + let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); + + if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) + .await + .is_ok() + { + let err = eyre::eyre!("Found dtkit-patch-based mod installation."); + return Err(err) + .with_suggestion(|| { + "If you're a mod author and saved projects directly in 'mods/', \ + use DTMT to migrate them to the new project structure." + .to_string() + }) + .with_suggestion(|| { + "Click 'Reset Game' to remove the previous mod installation.".to_string() + }); + } + + let (_, game_info, deployment_info) = tokio::try_join!( + async { + fs::metadata(&bundle_dir) + .await + .wrap_err("Failed to open game bundle directory") + .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") + }, + async { + tokio::task::spawn_blocking(dtmt_shared::collect_game_info) + .await + .map_err(Report::new) + }, + async { + let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); + match read_sjson_file::<_, DeploymentData>(path).await { + Ok(data) => Ok(Some(data)), + Err(err) => { + if let Some(err) = err.downcast_ref::() + && err.kind() == ErrorKind::NotFound + { + Ok(None) + } else { + Err(err).wrap_err("Failed to read deployment data") + } + } + } + } + ) + .wrap_err("Failed to gather deployment information")?; + + tracing::debug!(?game_info, ?deployment_info); + + if let Some(game_info) = game_info { + if deployment_info + .as_ref() + .map(|i| game_info.last_updated > i.timestamp) + .unwrap_or(false) + { + tracing::warn!( + "Game was updated since last mod deployment. \ + Attempting to reconcile game files." + ); + + tokio::try_join!( + async { + let path = bundle_dir.join(BUNDLE_DATABASE_NAME); + let backup_path = path.with_extension("data.bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for bundle database.") + }, + async { + let path = bundle_dir.join(boot_bundle_path); + let backup_path = path.with_extension("bak"); + + fs::copy(&path, &backup_path) + .await + .wrap_err("Failed to re-create backup for boot bundle") + } + ) + .with_suggestion(|| { + "Reset the game using 'Reset Game', then verify game files.".to_string() + })?; + + tracing::info!( + "Successfully re-created game file backups. \ + Continuing mod deployment." + ); + } + } + + check_mod_order(&state)?; + + tracing::info!( + "Deploying {} mods to '{}'.", + state.mods.iter().filter(|i| i.enabled).count(), + bundle_dir.display() + ); + + tracing::info!("Copy legacy mod folders"); + let mod_folders = copy_mod_folders(state.clone()) + .await + .wrap_err("Failed to copy mod folders")?; + + tracing::info!("Build mod bundles"); + let mut bundles = build_bundles(state.clone()) + .await + .wrap_err("Failed to build mod bundles")?; + + tracing::info!("Patch boot bundle"); + let mut boot_bundles = patch_boot_bundle(state.clone()) + .await + .wrap_err("Failed to patch boot bundle")?; + bundles.append(&mut boot_bundles); + + if let Some(info) = &deployment_info { + let bundle_dir = Arc::new(bundle_dir); + // Remove bundles from the previous deployment that don't match the current one. + // I.e. mods that used to be installed/enabled but aren't anymore. + { + let tasks = info.bundles.iter().cloned().filter_map(|file_name| { + let is_being_deployed = bundles.iter().any(|b2| { + let name = format!("{:016x}", b2.name()); + file_name == name + }); + + if !is_being_deployed { + let bundle_dir = bundle_dir.clone(); + let task = async move { + let path = bundle_dir.join(&file_name); + + tracing::debug!("Removing unused bundle '{}'", file_name); + + if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { + format!("Failed to remove unused bundle '{}'", path.display()) + }) { + tracing::error!("{:?}", err); + } + }; + Some(task) + } else { + None + } + }); + + futures::future::join_all(tasks).await; + } + + // Do the same thing for mod folders + { + let tasks = info.mod_folders.iter().filter_map(|mod_id| { + let is_being_deployed = mod_folders.iter().any(|id| id == mod_id); + + if !is_being_deployed { + let path = bundle_dir.join("mods").join(mod_id); + tracing::debug!("Removing unused mod folder '{}'", path.display()); + + let task = async move { + if let Err(err) = fs::remove_dir_all(&path).await.wrap_err_with(|| { + format!("Failed to remove unused legacy mod '{}'", path.display()) + }) { + tracing::error!("{:?}", err); + } + }; + + Some(task) + } else { + None + } + }); + futures::future::join_all(tasks).await; + } + } + + tracing::info!("Patch game settings"); + patch_game_settings(state.clone()) + .await + .wrap_err("Failed to patch game settings")?; + + tracing::info!("Patching bundle database"); + patch_bundle_database(state.clone(), &bundles) + .await + .wrap_err("Failed to patch bundle database")?; + + tracing::info!("Writing deployment data"); + write_deployment_data(state.clone(), &bundles, mod_folders) + .await + .wrap_err("Failed to write deployment data")?; + + tracing::info!("Finished deploying mods"); + Ok(()) +} diff --git a/crates/dtmm/src/controller/game.rs b/crates/dtmm/src/controller/game.rs index 5520c4a..6b91169 100644 --- a/crates/dtmm/src/controller/game.rs +++ b/crates/dtmm/src/controller/game.rs @@ -1,46 +1,19 @@ -use std::collections::HashMap; -use std::io::{self, Cursor, ErrorKind}; +use std::io::{self, ErrorKind}; use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::Arc; use color_eyre::eyre::Context; -use color_eyre::{eyre, Help, Report, Result}; -use futures::stream; -use futures::StreamExt; -use path_slash::PathBufExt; -use sdk::filetype::lua; -use sdk::filetype::package::Package; +use color_eyre::{eyre, Result}; use sdk::murmur::Murmur64; -use sdk::{ - Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, -}; -use serde::{Deserialize, Serialize}; -use string_template::Template; -use time::OffsetDateTime; -use tokio::fs; +use tokio::fs::{self}; use tokio::io::AsyncWriteExt; -use tracing::Instrument; -use super::read_sjson_file; -use crate::controller::app::check_mod_order; -use crate::state::{ActionState, PackageInfo}; +use crate::controller::deploy::{ + DeploymentData, BOOT_BUNDLE_NAME, BUNDLE_DATABASE_NAME, DEPLOYMENT_DATA_PATH, +}; +use crate::state::ActionState; -const MOD_BUNDLE_NAME: &str = "packages/mods"; -const BOOT_BUNDLE_NAME: &str = "packages/boot"; -const DML_BUNDLE_NAME: &str = "packages/dml"; -const BUNDLE_DATABASE_NAME: &str = "bundle_database.data"; -const MOD_BOOT_SCRIPT: &str = "scripts/mod_main"; -const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data"; -const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini"; -const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson"; - -#[derive(Debug, Serialize, Deserialize)] -struct DeploymentData { - bundles: Vec, - #[serde(with = "time::serde::iso8601")] - timestamp: OffsetDateTime, -} +use super::deploy::SETTINGS_FILE_PATH; #[tracing::instrument] async fn read_file_with_backup

(path: P) -> Result> @@ -130,585 +103,6 @@ async fn patch_game_settings(state: Arc) -> Result<()> { Ok(()) } -#[tracing::instrument(skip_all, fields(package = info.name))] -fn make_package(info: &PackageInfo) -> Result { - let mut pkg = Package::new(info.name.clone(), PathBuf::new()); - - for f in &info.files { - let mut it = f.rsplit('.'); - let file_type = it - .next() - .ok_or_else(|| eyre::eyre!("missing file extension")) - .and_then(BundleFileType::from_str) - .wrap_err("Invalid file name in package info")?; - let name: String = it.collect(); - pkg.add_file(file_type, name); - } - - Ok(pkg) -} - -fn build_mod_data_lua(state: Arc) -> String { - let mut lua = String::from("return {\n"); - - // DMF is handled explicitely by the loading procedures, as it actually drives most of that - // and should therefore not show up in the load order. - for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { - lua.push_str(" {\n name = \""); - lua.push_str(&mod_info.name); - - lua.push_str("\",\n id = \""); - lua.push_str(&mod_info.id); - - lua.push_str("\",\n run = function()\n"); - - let resources = &mod_info.resources; - if resources.data.is_some() || resources.localization.is_some() { - lua.push_str(" new_mod(\""); - lua.push_str(&mod_info.id); - lua.push_str("\", {\n mod_script = \""); - lua.push_str(&resources.init.to_slash_lossy()); - - if let Some(data) = resources.data.as_ref() { - lua.push_str("\",\n mod_data = \""); - lua.push_str(&data.to_slash_lossy()); - } - - if let Some(localization) = &resources.localization { - lua.push_str("\",\n mod_localization = \""); - lua.push_str(&localization.to_slash_lossy()); - } - - lua.push_str("\",\n })\n"); - } else { - lua.push_str(" return dofile(\""); - lua.push_str(&resources.init.to_slash_lossy()); - lua.push_str("\")\n"); - } - - lua.push_str(" end,\n packages = {\n"); - - for pkg_info in &mod_info.packages { - lua.push_str(" \""); - lua.push_str(&pkg_info.name); - lua.push_str("\",\n"); - } - - lua.push_str(" },\n },\n"); - } - - lua.push('}'); - - tracing::debug!("mod_data_lua:\n{}", lua); - - lua -} - -#[tracing::instrument(skip_all)] -async fn build_bundles(state: Arc) -> Result> { - let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string()); - let mut tasks = Vec::new(); - - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - - let mut bundles = Vec::new(); - - { - tracing::trace!("Building mod data script"); - - let span = tracing::debug_span!("Building mod data script"); - let _enter = span.enter(); - - let lua = build_mod_data_lua(state.clone()); - - tracing::trace!("Compiling mod data script"); - - let file = - lua::compile(MOD_DATA_SCRIPT, &lua).wrap_err("Failed to compile mod data Lua file")?; - - tracing::trace!("Compile mod data script"); - - mod_bundle.add_file(file); - } - - tracing::trace!("Preparing tasks to deploy bundle files"); - - for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { - let span = tracing::trace_span!("building mod packages", name = mod_info.name); - let _enter = span.enter(); - - let mod_dir = state.mod_dir.join(&mod_info.id); - for pkg_info in &mod_info.packages { - let span = tracing::trace_span!("building package", name = pkg_info.name); - let _enter = span.enter(); - - tracing::trace!( - "Building package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - let pkg = make_package(pkg_info).wrap_err("Failed to make package")?; - let mut variant = BundleFileVariant::new(); - let bin = pkg - .to_binary() - .wrap_err("Failed to serialize package to binary")?; - variant.set_data(bin); - let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package); - file.add_variant(variant); - - tracing::trace!( - "Compiled package {} for mod {}", - pkg_info.name, - mod_info.name - ); - - mod_bundle.add_file(file); - - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = mod_dir.join(&bundle_name); - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - // Explicitely drop the guard, so that we can move the span - // into the async operation - drop(_enter); - - let ctx = state.ctx.clone(); - - let task = async move { - let bundle = { - let bin = fs::read(&src).await.wrap_err_with(|| { - format!("Failed to read bundle file '{}'", src.display()) - })?; - let name = Bundle::get_name_from_path(&ctx, &src); - Bundle::from_binary(&ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))? - }; - - tracing::debug!( - src = %src.display(), - dest = %dest.display(), - "Copying bundle '{}' for mod '{}'", - pkg_name, - mod_name, - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - - Ok::(bundle) - } - .instrument(span); - - tasks.push(task); - } - } - - tracing::debug!("Copying {} mod bundles", tasks.len()); - - let mut tasks = stream::iter(tasks).buffer_unordered(10); - - while let Some(res) = tasks.next().await { - let bundle = res?; - bundles.push(bundle); - } - - { - let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64())); - tracing::trace!("Writing mod bundle to '{}'", path.display()); - fs::write(&path, mod_bundle.to_binary()?) - .await - .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; - } - - bundles.push(mod_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all)] -async fn patch_boot_bundle(state: Arc) -> Result> { - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()))); - - let mut bundles = Vec::with_capacity(2); - - let mut boot_bundle = async { - let bin = read_file_with_backup(&bundle_path) - .await - .wrap_err("Failed to read boot bundle")?; - - Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin) - .wrap_err("Failed to parse boot bundle") - } - .instrument(tracing::trace_span!("read boot bundle")) - .await - .wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?; - - { - tracing::trace!("Adding mod package file to boot bundle"); - let span = tracing::trace_span!("create mod package file"); - let _enter = span.enter(); - - let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new()); - - for mod_info in &state.mods { - for pkg_info in &mod_info.packages { - pkg.add_file(BundleFileType::Package, &pkg_info.name); - } - } - - pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT); - - let mut variant = BundleFileVariant::new(); - variant.set_data(pkg.to_binary()?); - let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - tracing::trace!("Handling DML packages and bundle"); - let span = tracing::trace_span!("handle DML"); - let _enter = span.enter(); - - let mut variant = BundleFileVariant::new(); - - let mod_info = state - .mods - .iter() - .find(|m| m.id == "dml") - .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?; - let pkg_info = mod_info - .packages - .get(0) - .ok_or_else(|| eyre::eyre!("invalid mod package for DML")) - .with_suggestion(|| "Re-download and import the newest version.".to_string())?; - let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name)); - let src = state.mod_dir.join(&mod_info.id).join(&bundle_name); - - { - let bin = fs::read(&src) - .await - .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?; - let name = Bundle::get_name_from_path(&state.ctx, &src); - - let dml_bundle = Bundle::from_binary(&state.ctx, name, bin) - .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?; - - bundles.push(dml_bundle); - }; - - { - let dest = bundle_dir.join(&bundle_name); - let pkg_name = pkg_info.name.clone(); - let mod_name = mod_info.name.clone(); - - tracing::debug!( - "Copying bundle {} for mod {}: {} -> {}", - pkg_name, - mod_name, - src.display(), - dest.display() - ); - // We attempt to remove any previous file, so that the hard link can be created. - // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy - // may be possible despite an error here, or the error will be reported by it anyways. - // TODO: There is a chance that we delete an actual game bundle, but with 64bit - // hashes, it's low enough for now, and the setup required to detect - // "game bundle vs mod bundle" is non-trivial. - let _ = fs::remove_file(&dest).await; - fs::copy(&src, &dest).await.wrap_err_with(|| { - format!( - "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}", - src.display(), - dest.display() - ) - })?; - } - - let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?; - variant.set_data(pkg.to_binary()?); - - let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package); - f.add_variant(variant); - - boot_bundle.add_file(f); - } - - { - let span = tracing::debug_span!("Importing mod main script"); - let _enter = span.enter(); - - let is_io_enabled = format!("{}", state.is_io_enabled); - let mut data = HashMap::new(); - data.insert("is_io_enabled", is_io_enabled.as_str()); - - let tmpl = include_str!("../../assets/mod_main.lua"); - let lua = Template::new(tmpl).render(&data); - tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); - let file = - lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; - - boot_bundle.add_file(file); - } - - async { - let bin = boot_bundle - .to_binary() - .wrap_err("Failed to serialize boot bundle")?; - fs::write(&bundle_path, bin) - .await - .wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display())) - } - .instrument(tracing::trace_span!("write boot bundle")) - .await?; - - bundles.push(boot_bundle); - - Ok(bundles) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn patch_bundle_database(state: Arc, bundles: B) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let bundle_dir = Arc::new(state.game_dir.join("bundle")); - let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME); - - let mut db = { - let bin = read_file_with_backup(&database_path) - .await - .wrap_err("Failed to read bundle database")?; - let mut r = Cursor::new(bin); - let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?; - tracing::trace!("Finished parsing bundle database"); - db - }; - - for bundle in bundles.as_ref() { - tracing::trace!("Adding '{}' to bundle database", bundle.name().display()); - db.add_bundle(bundle); - } - - { - let bin = db - .to_binary() - .wrap_err("Failed to serialize bundle database")?; - fs::write(&database_path, bin).await.wrap_err_with(|| { - format!( - "failed to write bundle database to '{}'", - database_path.display() - ) - })?; - } - - Ok(()) -} - -#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))] -async fn write_deployment_data(state: Arc, bundles: B) -> Result<()> -where - B: AsRef<[Bundle]>, -{ - let info = DeploymentData { - timestamp: OffsetDateTime::now_utc(), - bundles: bundles - .as_ref() - .iter() - .map(|bundle| format!("{:x}", bundle.name().to_murmur64())) - .collect(), - }; - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?; - - fs::write(&path, &data) - .await - .wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?; - - Ok(()) -} - -#[tracing::instrument(skip_all, fields( - game_dir = %state.game_dir.display(), - mods = state.mods.len() -))] -pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { - let state = Arc::new(state); - let bundle_dir = state.game_dir.join("bundle"); - let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())); - - if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999"))) - .await - .is_ok() - { - let err = eyre::eyre!("Found dtkit-patch-based mod installation."); - return Err(err) - .with_suggestion(|| { - "If you're a mod author and saved projects directly in 'mods/', \ - use DTMT to migrate them to the new project structure." - .to_string() - }) - .with_suggestion(|| { - "Click 'Reset Game' to remove the previous mod installation.".to_string() - }); - } - - let (_, game_info, deployment_info) = tokio::try_join!( - async { - fs::metadata(&bundle_dir) - .await - .wrap_err("Failed to open game bundle directory") - .with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.") - }, - async { - tokio::task::spawn_blocking(dtmt_shared::collect_game_info) - .await - .map_err(Report::new) - }, - async { - let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - match read_sjson_file::<_, DeploymentData>(path).await { - Ok(data) => Ok(Some(data)), - Err(err) => { - if let Some(err) = err.downcast_ref::() - && err.kind() == ErrorKind::NotFound - { - Ok(None) - } else { - Err(err).wrap_err("Failed to read deployment data") - } - } - } - } - ) - .wrap_err("Failed to gather deployment information")?; - - tracing::debug!(?game_info, ?deployment_info); - - if let Some(game_info) = game_info { - if deployment_info - .as_ref() - .map(|i| game_info.last_updated > i.timestamp) - .unwrap_or(false) - { - tracing::warn!( - "Game was updated since last mod deployment. \ - Attempting to reconcile game files." - ); - - tokio::try_join!( - async { - let path = bundle_dir.join(BUNDLE_DATABASE_NAME); - let backup_path = path.with_extension("data.bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for bundle database.") - }, - async { - let path = bundle_dir.join(boot_bundle_path); - let backup_path = path.with_extension("bak"); - - fs::copy(&path, &backup_path) - .await - .wrap_err("Failed to re-create backup for boot bundle") - } - ) - .with_suggestion(|| { - "Reset the game using 'Reset Game', then verify game files.".to_string() - })?; - - tracing::info!( - "Successfully re-created game file backups. \ - Continuing mod deployment." - ); - } - } - - check_mod_order(&state)?; - - tracing::info!( - "Deploying {} mods to '{}'.", - state.mods.iter().filter(|i| i.enabled).count(), - bundle_dir.display() - ); - - tracing::info!("Build mod bundles"); - let mut bundles = build_bundles(state.clone()) - .await - .wrap_err("Failed to build mod bundles")?; - - tracing::info!("Patch boot bundle"); - let mut more_bundles = patch_boot_bundle(state.clone()) - .await - .wrap_err("Failed to patch boot bundle")?; - bundles.append(&mut more_bundles); - - if let Some(info) = &deployment_info { - let bundle_dir = Arc::new(bundle_dir); - let tasks = info.bundles.iter().cloned().filter_map(|file_name| { - let contains = bundles.iter().any(|b2| { - let name = format!("{:016x}", b2.name()); - file_name == name - }); - - if !contains { - let bundle_dir = bundle_dir.clone(); - let task = async move { - let path = bundle_dir.join(&file_name); - - tracing::debug!("Removing unused bundle '{}'", file_name); - - if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| { - format!("Failed to remove unused bundle '{}'", path.display()) - }) { - tracing::error!("{:?}", err); - } - }; - Some(task) - } else { - None - } - }); - - futures::future::join_all(tasks).await; - } - - tracing::info!("Patch game settings"); - patch_game_settings(state.clone()) - .await - .wrap_err("Failed to patch game settings")?; - - tracing::info!("Patching bundle database"); - patch_bundle_database(state.clone(), &bundles) - .await - .wrap_err("Failed to patch bundle database")?; - - tracing::info!("Writing deployment data"); - write_deployment_data(state.clone(), &bundles) - .await - .wrap_err("Failed to write deployment data")?; - - tracing::info!("Finished deploying mods"); - Ok(()) -} - #[tracing::instrument(skip_all)] async fn reset_dtkit_patch(state: ActionState) -> Result<()> { let bundle_dir = state.game_dir.join("bundle"); diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs new file mode 100644 index 0000000..3e17d4c --- /dev/null +++ b/crates/dtmm/src/controller/import.rs @@ -0,0 +1,475 @@ +use std::collections::HashMap; +use std::ffi::CStr; +use std::io::{Cursor, Read, Seek, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use color_eyre::eyre::{self, Context}; +use color_eyre::{Help, Report, Result}; +use druid::im::Vector; +use druid::{FileInfo, ImageBuf}; +use dtmt_shared::{ModConfig, ModConfigResources}; +use luajit2_sys as lua; +use nexusmods::Api as NexusApi; +use tokio::fs; +use zip::ZipArchive; + +use crate::state::{ActionState, ModInfo, NexusInfo, PackageInfo}; + +fn find_archive_file( + archive: &ZipArchive, + name: impl AsRef, +) -> Option { + let path = archive + .file_names() + .find(|path| path.ends_with(name.as_ref())) + .map(|s| s.to_string()); + path +} + +// Runs the content of a `.mod` file to extract what data we can get +// from legacy mods. +// 1. Create a global function `new_mod` that stores +// the relevant bits in global variables. +// 2. Run the `.mod` file, which will merely return a table. +// 3. Run the `run` function from that table. +// 4. Access the global variables from #1. +#[tracing::instrument] +fn parse_mod_id_file(data: &str) -> Result<(String, ModConfigResources)> { + tracing::debug!("Parsing mod file:\n{}", data); + + let ret = unsafe { + let state = lua::luaL_newstate(); + lua::luaL_openlibs(state); + + let run = b" +function fassert() end +function new_mod(id, resources) + _G.id = id + _G.script = resources.mod_script + _G.data = resources.mod_data + _G.localization = resources.mod_localization +end +\0"; + match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRSYNTAX => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Invalid syntax: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to create `new_mod`") + } + _ => unreachable!(), + } + + match lua::lua_pcall(state, 0, 0, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run buffer: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run buffer") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let name = b".mod\0"; + match lua::luaL_loadbuffer( + state, + data.as_ptr() as _, + data.len() as _, + name.as_ptr() as _, + ) as u32 + { + lua::LUA_OK => {} + lua::LUA_ERRSYNTAX => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Invalid syntax: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to load `.mod` file buffer") + } + _ => unreachable!(), + } + + match lua::lua_pcall(state, 0, 1, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run `.mod` file: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run `.mod` file") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let key = b"run\0"; + lua::lua_pushstring(state, key.as_ptr() as _); + lua::lua_gettable(state, -2); + + match lua::lua_pcall(state, 0, 0, 0) as u32 { + lua::LUA_OK => {} + lua::LUA_ERRRUN => { + let err = lua::lua_tostring(state, -1); + let err = CStr::from_ptr(err).to_string_lossy().to_string(); + + lua::lua_close(state); + + eyre::bail!("Failed to run `.mod.run`: {}", err); + } + lua::LUA_ERRMEM => { + lua::lua_close(state); + eyre::bail!("Failed to allocate sufficient memory to run `.mod.run`") + } + // We don't use an error handler function, so this should be unreachable + lua::LUA_ERRERR => unreachable!(), + _ => unreachable!(), + } + + let get_global = |state, key: &[u8]| { + lua::lua_getglobal(state, key.as_ptr() as _); + + if lua::lua_isnil(state, -1) != 0 { + return Ok(None); + } + + let s = lua::lua_tostring(state, -1); + + if s.is_null() { + eyre::bail!("Expected string, got NULL"); + } + + let ret = CStr::from_ptr(s).to_string_lossy().to_string(); + lua::lua_pop(state, 1); + Ok(Some(ret)) + }; + + let mod_id = get_global(state, b"id\0") + .and_then(|s| s.ok_or_else(|| eyre::eyre!("Got `nil`"))) + .wrap_err("Failed to get `id`")?; + + let resources = ModConfigResources { + init: get_global(state, b"script\0") + .and_then(|s| s.map(PathBuf::from).ok_or_else(|| eyre::eyre!("Got `nil`"))) + .wrap_err("Failed to get `script`.")?, + data: get_global(state, b"data\0") + .wrap_err("Failed to get `data`.")? + .map(PathBuf::from), + localization: get_global(state, b"localization\0") + .wrap_err("Failed to get `localization`")? + .map(PathBuf::from), + }; + + lua::lua_close(state); + + (mod_id, resources) + }; + + Ok(ret) +} + +// Extracts the mod configuration from the mod archive. +// This may either be a proper `dtmt.cfg`, or the legacy `.mod` ID file. +// +// It also returns the directory where this file was found, used as root path. This +// allows flexibility in what the directory structure is exactly, since many people +// still end up creating tarbombs and Nexus does its own re-packaging. +#[tracing::instrument(skip(archive))] +fn extract_mod_config(archive: &mut ZipArchive) -> Result<(ModConfig, String)> { + if let Some(name) = find_archive_file(archive, "dtmt.cfg") { + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read mod config from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read mod config from archive")?; + + let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; + + let cfg = serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")?; + let root = name + .strip_suffix("dtmt.cfg") + .expect("String must end with that suffix") + .to_string(); + + Ok((cfg, root)) + } else if let Some(name) = find_archive_file(archive, ".mod") { + let (mod_id, resources) = { + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read `.mod` file from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read `.mod` file from archive")?; + + let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?; + parse_mod_id_file(&data).wrap_err("Invalid `.mod` file")? + }; + + let cfg = ModConfig { + bundled: false, + dir: PathBuf::new(), + id: mod_id.clone(), + name: mod_id, + summary: String::new(), + version: String::new(), + description: None, + author: None, + image: None, + categories: Vec::new(), + packages: Vec::new(), + resources, + depends: Vec::new(), + }; + let root = if let Some(index) = name.rfind('/') { + name[..index].to_string() + } else { + String::new() + }; + + Ok((cfg, root)) + } else { + eyre::bail!( + "Mod needs either a config file or `.mod` file. \ + Please get in touch with the author to provide a properly packaged mod." + ); + } +} + +#[tracing::instrument(skip(archive))] +fn extract_bundled_mod( + archive: &mut ZipArchive, + root: String, + dest: impl AsRef + std::fmt::Debug, +) -> Result>> { + let files: HashMap> = { + let name = archive + .file_names() + .find(|name| name.ends_with("files.sjson")) + .map(|s| s.to_string()) + .ok_or_else(|| eyre::eyre!("archive does not contain file index"))?; + + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read file index from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?; + serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")? + }; + + tracing::trace!(?files); + + let dest = dest.as_ref(); + tracing::trace!("Extracting mod archive to {}", dest.display()); + archive + .extract(dest) + .wrap_err_with(|| format!("Failed to extract archive to {}", dest.display()))?; + + let packages = files + .into_iter() + .map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect()))) + .collect(); + + tracing::trace!(?packages); + + Ok(packages) +} + +#[tracing::instrument(skip(archive))] +fn extract_legacy_mod( + archive: &mut ZipArchive, + root: String, + dest: impl Into + std::fmt::Debug, +) -> Result<()> { + let dest = dest.into(); + let file_count = archive.len(); + + for i in 0..file_count { + let mut f = archive + .by_index(i) + .wrap_err_with(|| format!("Failed to get file at index {}", i))?; + + let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else { + let err = eyre::eyre!("File name in archive is not a safe path value."); + return Err(err).with_suggestion(|| { + "Only use well-known applications to create the ZIP archive, \ + and don't create paths that point outside the archive directory." + }); + }; + + let Ok(suffix) = name.strip_prefix(&root) else { + tracing::warn!( + "Skipping file outside of the mod root directory: {}", + name.display() + ); + continue; + }; + let name = dest.join(suffix); + + if f.is_dir() { + // The majority of errors will actually be "X already exists". + // But rather than filter them invidually, we just ignore all of them. + // If there is a legitimate error of "couldn't create X", it will eventually fail when + // we try to put a file in there. + tracing::trace!("Creating directory '{}'", name.display()); + let _ = std::fs::create_dir_all(&name); + } else { + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err_with(|| format!("Failed to read file '{}'", name.display()))?; + + tracing::trace!("Writing file '{}'", name.display()); + let mut out = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&name) + .wrap_err_with(|| format!("Failed to open file '{}'", name.display()))?; + + out.write_all(&buf) + .wrap_err_with(|| format!("Failed to write to '{}'", name.display()))?; + } + } + + Ok(()) +} + +#[tracing::instrument(skip(state))] +pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result { + let data = fs::read(&info.path) + .await + .wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?; + let data = Cursor::new(data); + + let nexus = if let Some((_, id, _, _)) = info + .path + .file_name() + .and_then(|s| s.to_str()) + .and_then(NexusApi::parse_file_name) + { + if !state.nexus_api_key.is_empty() { + let api = NexusApi::new(state.nexus_api_key.to_string())?; + let mod_info = api + .mods_id(id) + .await + .wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?; + Some(NexusInfo::from(mod_info)) + } else { + None + } + } else { + None + }; + + let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?; + + if tracing::enabled!(tracing::Level::DEBUG) { + let names = archive.file_names().fold(String::new(), |mut s, name| { + s.push('\n'); + s.push_str(name); + s + }); + tracing::debug!("Archive contents:{}", names); + } + + let (mod_cfg, root) = + extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?; + tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id); + tracing::debug!(root, ?mod_cfg); + + let image = if let Some(path) = &mod_cfg.image { + let name = archive + .file_names() + .find(|name| name.ends_with(&path.display().to_string())) + .map(|s| s.to_string()) + .ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?; + + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read image file from archive")?; + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read file index from archive")?; + + // Druid somehow doesn't return an error compatible with eyre, here. + // So we have to wrap through `Display` manually. + let img = match ImageBuf::from_data(&buf) { + Ok(img) => img, + Err(err) => { + let err = Report::msg(err.to_string()).wrap_err("Invalid image data"); + return Err(err).with_suggestion(|| { + "Supported formats are: PNG, JPEG, Bitmap and WebP".to_string() + }); + } + }; + + Some(img) + } else { + None + }; + + tracing::trace!(?image); + + let mod_dir = state.data_dir.join(state.mod_dir.as_ref()); + let dest = mod_dir.join(&mod_cfg.id); + + tracing::trace!("Creating mods directory {}", dest.display()); + fs::create_dir_all(&dest) + .await + .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?; + + let packages = if mod_cfg.bundled { + extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")? + } else { + extract_legacy_mod(&mut archive, root, &dest).wrap_err("Failed to extract legacy mod")?; + + let data = serde_sjson::to_string(&mod_cfg).wrap_err("Failed to serialize mod config")?; + fs::write(dest.join("dtmt.cfg"), &data) + .await + .wrap_err("Failed to write mod config")?; + + Default::default() + }; + + if let Some(nexus) = &nexus { + let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?; + let path = dest.join("nexus.sjson"); + fs::write(&path, data.as_bytes()) + .await + .wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?; + } + + let info = ModInfo::new(mod_cfg, packages, image, nexus); + Ok(info) +} diff --git a/crates/dtmm/src/controller/mod.rs b/crates/dtmm/src/controller/mod.rs index eacc3ef..9c75e84 100644 --- a/crates/dtmm/src/controller/mod.rs +++ b/crates/dtmm/src/controller/mod.rs @@ -5,7 +5,9 @@ use serde::Deserialize; use tokio::fs; pub mod app; +pub mod deploy; pub mod game; +pub mod import; pub mod worker; #[tracing::instrument] diff --git a/crates/dtmm/src/controller/worker.rs b/crates/dtmm/src/controller/worker.rs index 238b3f3..518c0a8 100644 --- a/crates/dtmm/src/controller/worker.rs +++ b/crates/dtmm/src/controller/worker.rs @@ -13,7 +13,9 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::RwLock; use crate::controller::app::*; +use crate::controller::deploy::deploy_mods; use crate::controller::game::*; +use crate::controller::import::import_mod; use crate::state::AsyncAction; use crate::state::ACTION_FINISH_CHECK_UPDATE; use crate::state::ACTION_FINISH_LOAD_INITIAL; diff --git a/crates/dtmm/src/main.rs b/crates/dtmm/src/main.rs index 9ba838c..8069e6d 100644 --- a/crates/dtmm/src/main.rs +++ b/crates/dtmm/src/main.rs @@ -1,6 +1,7 @@ #![recursion_limit = "256"] #![feature(let_chains)] #![feature(arc_unwrap_or_clone)] +#![feature(iterator_try_collect)] #![windows_subsystem = "windows"] use std::path::PathBuf; diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index c43e973..8a461bd 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -109,7 +109,7 @@ pub(crate) struct ModInfo { #[data(ignore)] pub resources: ModResourceInfo, pub depends: Vector, - pub bundle: bool, + pub bundled: bool, #[data(ignore)] pub nexus: Option, } @@ -130,7 +130,7 @@ impl ModInfo { version: cfg.version, enabled: false, packages, - bundle: cfg.bundle, + bundled: cfg.bundled, image, categories: cfg.categories.into_iter().collect(), resources: ModResourceInfo { diff --git a/crates/dtmm/src/ui/window/main.rs b/crates/dtmm/src/ui/window/main.rs index 7aa2819..7de4418 100644 --- a/crates/dtmm/src/ui/window/main.rs +++ b/crates/dtmm/src/ui/window/main.rs @@ -145,7 +145,7 @@ fn build_mod_list() -> impl Widget { let tree = theme::icons::recolor_icon(tree, true, COLOR_YELLOW_LIGHT); - Svg::new(Arc::new(tree)).fix_height(druid::theme::TEXT_SIZE_NORMAL) + Svg::new(tree).fix_height(druid::theme::TEXT_SIZE_NORMAL) }; Either::new( diff --git a/crates/dtmt/src/cmd/migrate.rs b/crates/dtmt/src/cmd/migrate.rs index 8ecd648..2eacded 100644 --- a/crates/dtmt/src/cmd/migrate.rs +++ b/crates/dtmt/src/cmd/migrate.rs @@ -350,7 +350,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> localization: mod_file.localization, }, depends: vec![ModDependency::ID(String::from("DMF"))], - bundle: true, + bundled: true, }; tracing::debug!(?dtmt_cfg); diff --git a/lib/dtmt-shared/src/lib.rs b/lib/dtmt-shared/src/lib.rs index a162d3b..28e4694 100644 --- a/lib/dtmt-shared/src/lib.rs +++ b/lib/dtmt-shared/src/lib.rs @@ -63,7 +63,7 @@ pub struct ModConfig { #[serde(default)] pub depends: Vec, #[serde(default = "default_true", skip_serializing_if = "is_true")] - pub bundle: bool, + pub bundled: bool, } pub const STEAMAPP_ID: u32 = 1361210; -- 2.45.3 From dfaa39cd54651b077c21d4bb06c5ec1226eb4b75 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 10 Nov 2023 11:13:08 +0100 Subject: [PATCH 05/13] Move deployment directory for legacy mods This moves it back to its original place at `$game_dir/mods`. --- crates/dtmm/src/controller/deploy.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs index 6a804e7..ded840e 100644 --- a/crates/dtmm/src/controller/deploy.rs +++ b/crates/dtmm/src/controller/deploy.rs @@ -223,7 +223,7 @@ async fn copy_recursive( #[tracing::instrument(skip(state))] async fn copy_mod_folders(state: Arc) -> Result> { - let bundle_dir = Arc::new(state.game_dir.join("bundle")); + let game_dir = Arc::clone(&state.game_dir); let mut tasks = Vec::new(); @@ -237,11 +237,11 @@ async fn copy_mod_folders(state: Arc) -> Result> { let mod_id = mod_info.id.clone(); let mod_dir = Arc::clone(&state.mod_dir); - let bundle_dir = Arc::clone(&bundle_dir); + let game_dir = Arc::clone(&game_dir); let task = async move { let from = mod_dir.join(&mod_id); - let to = bundle_dir.join("mods").join(&mod_id); + let to = game_dir.join("mods").join(&mod_id); tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id); let _ = fs::create_dir_all(&to).await; -- 2.45.3 From eee1f500b8a8b6bae9fc6737048baaec7a62d62e Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Sun, 12 Nov 2023 23:20:10 +0100 Subject: [PATCH 06/13] Use template engine to build `mod_data.lua` The string-building version became too complex to maintain properly. --- Cargo.lock | 10 +++ crates/dtmm/Cargo.toml | 1 + crates/dtmm/assets/mod_data.lua.j2 | 27 ++++++ crates/dtmm/assets/mod_main.lua | 10 +-- crates/dtmm/src/controller/deploy.rs | 119 +++++++++++++-------------- 5 files changed, 99 insertions(+), 68 deletions(-) create mode 100644 crates/dtmm/assets/mod_data.lua.j2 diff --git a/Cargo.lock b/Cargo.lock index 011fa6b..8385e67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -906,6 +906,7 @@ dependencies = [ "futures", "lazy_static", "luajit2-sys", + "minijinja", "nexusmods", "oodle", "path-slash", @@ -2084,6 +2085,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minijinja" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb" +dependencies = [ + "serde", +] + [[package]] name = "minimal-lexical" version = "0.2.1" diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 1842a62..5f60220 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -35,3 +35,4 @@ ansi-parser = "0.9.0" string_template = "0.2.1" luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } async-recursion = "1.0.5" +minijinja = "1.0.10" diff --git a/crates/dtmm/assets/mod_data.lua.j2 b/crates/dtmm/assets/mod_data.lua.j2 new file mode 100644 index 0000000..9f87ad1 --- /dev/null +++ b/crates/dtmm/assets/mod_data.lua.j2 @@ -0,0 +1,27 @@ +return { +{% for mod in mods %} +{ + id = "{{ mod.id }}", + name = "{{ mod.name }}", + bundled = {{ mod.bundled }}, + packages = { + {% for pkg in mod.packages %} + "{{ pkg }}", + {% endfor %} + }, + run = function() + {% if mod.data is none %} + return dofile("{{ mod.init }}") + {% else %} + new_mod("{{ mod.id }}", { + mod_script = "{{ mod.init }}", + mod_data = "{{ mod.data }}", + {% if not mod.localization is none %} + mod_localization = "{{ mod.localization }}", + {% endif %} + }) + {% endif %} + end, +}, +{% endfor %} +} diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua index 6dbf3e2..a17f5be 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua @@ -61,14 +61,14 @@ local function patch_mod_loading_state() if state == "load_package" and package_manager:update() then log("StateBootLoadMods", "Packages loaded, loading mods") self._state = "load_mods" - local ModLoader = require("scripts/mods/dml/init") + local DML = require("scripts/mods/dml/init") local mod_data = require("scripts/mods/mod_data") - local mod_loader = ModLoader:new(mod_data, self._parent:gui()) + local mod_loader = DML.create_loader(mod_data, self._parent:gui()) - self._mod_loader = mod_loader + self._dml = DML Managers.mod = mod_loader - elseif state == "load_mods" and self._mod_loader:update(dt) then + elseif state == "load_mods" and self._dml.update(Managers.mod, dt) then log("StateBootLoadMods", "Mods loaded, exiting") return true, false end @@ -112,7 +112,7 @@ local require_store = {} -- This token is treated as a string template and filled by DTMM during deployment. -- This allows hiding unsafe I/O functions behind a setting. -- It's also a valid table definition, thereby degrading gracefully when not replaced. -local is_io_enabled = { { is_io_enabled } } -- luacheck: ignore 113 +local is_io_enabled = {{ is_io_enabled }} -- luacheck: ignore 113 local lua_libs = { debug = debug, os = { diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs index ded840e..53c4ef1 100644 --- a/crates/dtmm/src/controller/deploy.rs +++ b/crates/dtmm/src/controller/deploy.rs @@ -8,7 +8,7 @@ use color_eyre::eyre::Context; use color_eyre::{eyre, Help, Report, Result}; use futures::StreamExt; use futures::{stream, TryStreamExt}; -use path_slash::PathBufExt; +use minijinja::Environment; use sdk::filetype::lua; use sdk::filetype::package::Package; use sdk::murmur::Murmur64; @@ -201,10 +201,10 @@ async fn copy_recursive( async move { if is_dir { tracing::trace!("Creating directory '{}'", dest.display()); - fs::create_dir(&dest) - .await - .map(|_| ()) - .wrap_err_with(|| format!("Failed to create directory '{}'", dest.display())) + // Instead of trying to filter "already exists" errors out explicitly, + // we just ignore all. It'll fail eventually with the next copy operation. + let _ = fs::create_dir(&dest).await; + Ok(()) } else { tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display()); fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| { @@ -262,67 +262,60 @@ async fn copy_mod_folders(state: Arc) -> Result> { Ok(ids) } -fn build_mod_data_lua(state: Arc) -> String { - let mut lua = String::from("return {\n"); - - // DMF is handled explicitely by the loading procedures, as it actually drives most of that - // and should therefore not show up in the load order. - for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) { - lua.push_str(" {\n name = \""); - lua.push_str(&mod_info.name); - - lua.push_str("\",\n id = \""); - lua.push_str(&mod_info.id); - - lua.push_str("\",\n bundled = \""); - if mod_info.bundled { - lua.push_str("true"); - } else { - lua.push_str("false"); - } - - lua.push_str("\",\n run = function()\n"); - - let resources = &mod_info.resources; - if resources.data.is_some() || resources.localization.is_some() { - lua.push_str(" new_mod(\""); - lua.push_str(&mod_info.id); - lua.push_str("\", {\n mod_script = \""); - lua.push_str(&resources.init.to_slash_lossy()); - - if let Some(data) = resources.data.as_ref() { - lua.push_str("\",\n mod_data = \""); - lua.push_str(&data.to_slash_lossy()); - } - - if let Some(localization) = &resources.localization { - lua.push_str("\",\n mod_localization = \""); - lua.push_str(&localization.to_slash_lossy()); - } - - lua.push_str("\",\n })\n"); - } else { - lua.push_str(" return dofile(\""); - lua.push_str(&resources.init.to_slash_lossy()); - lua.push_str("\")\n"); - } - - lua.push_str(" end,\n packages = {\n"); - - for pkg_info in &mod_info.packages { - lua.push_str(" \""); - lua.push_str(&pkg_info.name); - lua.push_str("\",\n"); - } - - lua.push_str(" },\n },\n"); +fn build_mod_data_lua(state: Arc) -> Result { + #[derive(Serialize)] + struct TemplateDataMod { + id: String, + name: String, + bundled: bool, + init: String, + data: Option, + localization: Option, + packages: Vec, } - lua.push('}'); + let mut env = Environment::new(); + env.add_template("mod_data.lua", include_str!("../../assets/mod_data.lua.j2")) + .wrap_err("Failed to compile template for `mod_data.lua`")?; + let tmpl = env + .get_template("mod_data.lua") + .wrap_err("Failed to get template `mod_data.lua`")?; - tracing::debug!("mod_data_lua:\n{}", lua); + let data: Vec = state + .mods + .iter() + .filter_map(|m| { + if m.id == "dml" || !m.enabled { + return None; + } - lua + Some(TemplateDataMod { + id: m.id.clone(), + name: m.name.clone(), + bundled: m.bundled, + init: m.resources.init.to_string_lossy().to_string(), + data: m + .resources + .data + .as_ref() + .map(|p| p.to_string_lossy().to_string()), + localization: m + .resources + .localization + .as_ref() + .map(|p| p.to_string_lossy().to_string()), + packages: m.packages.iter().map(|p| p.name.clone()).collect(), + }) + }) + .collect(); + + let lua = tmpl + .render(minijinja::context!(mods => data)) + .wrap_err("Failed to render template `mod_data.lua`")?; + + tracing::debug!("mod_data.lua:\n{}", lua); + + Ok(lua) } #[tracing::instrument(skip_all)] @@ -340,7 +333,7 @@ async fn build_bundles(state: Arc) -> Result> { let span = tracing::debug_span!("Building mod data script"); let _enter = span.enter(); - let lua = build_mod_data_lua(state.clone()); + let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?; tracing::trace!("Compiling mod data script"); -- 2.45.3 From e162f6845739d99c364902b048be84f9b7f3a22a Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Tue, 14 Nov 2023 15:06:19 +0100 Subject: [PATCH 07/13] Fix missing `Mods.original_require` --- crates/dtmm/assets/mod_main.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua index a17f5be..2b329bf 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua @@ -138,7 +138,8 @@ Mods = { -- Fatshark's code scrubs them. -- The loader can then decide to pass them on to mods, or ignore them lua = setmetatable({}, { __index = lua_libs }), - require_store = require_store + require_store = require_store, + original_require = require, } local can_insert = function(filepath, new_result) -- 2.45.3 From b85d54ea1cdca21234112527898b9c6b98bee9ea Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Tue, 14 Nov 2023 15:07:07 +0100 Subject: [PATCH 08/13] Fix Nexusmods API key not being loaded from config --- crates/dtmm/src/controller/import.rs | 5 ++++- crates/dtmm/src/state/delegate.rs | 1 + crates/dtmm/src/util/config.rs | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs index 3e17d4c..05feba9 100644 --- a/crates/dtmm/src/controller/import.rs +++ b/crates/dtmm/src/controller/import.rs @@ -384,7 +384,10 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result for Delegate { state.config_path = Arc::new(config.path); state.data_dir = Arc::new(config.data_dir); state.game_dir = Arc::new(config.game_dir.unwrap_or_default()); + state.nexus_api_key = Arc::new(config.nexus_api_key.unwrap_or_default()); state.is_io_enabled = config.unsafe_io; } diff --git a/crates/dtmm/src/util/config.rs b/crates/dtmm/src/util/config.rs index 3a0d0b2..9affbb6 100644 --- a/crates/dtmm/src/util/config.rs +++ b/crates/dtmm/src/util/config.rs @@ -125,6 +125,9 @@ where .wrap_err_with(|| format!("Invalid config file {}", path.display()))?; cfg.path = path; + + tracing::debug!("Read config file '{}': {:?}", cfg.path.display(), cfg); + Ok(cfg) } Err(err) if err.kind() == ErrorKind::NotFound => { @@ -133,6 +136,11 @@ where .wrap_err_with(|| format!("Failed to read config file {}", path.display()))?; } + tracing::debug!( + "Config file not found at '{}', creating default.", + path.display() + ); + { let parent = default_path .parent() -- 2.45.3 From 2f746debf3503397c910c0ef825514aa014bc938 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Tue, 14 Nov 2023 15:38:51 +0100 Subject: [PATCH 09/13] Use mod name from Nexus if necessary Non-bundled mods come without `dtmt.cfg` and therefore no way to provide a user friendly name. Similar to the other fields, use the one from Nexus in that case. --- crates/dtmm/src/state/data.rs | 2 ++ crates/dtmm/src/ui/window/main.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index 8a461bd..64fdd28 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -73,6 +73,7 @@ impl From for ModDependency { #[derive(Clone, Data, Debug, Lens, serde::Serialize, serde::Deserialize)] pub(crate) struct NexusInfo { pub id: u64, + pub name: String, pub version: String, pub author: String, pub summary: Arc, @@ -83,6 +84,7 @@ impl From for NexusInfo { fn from(value: NexusMod) -> Self { Self { id: value.mod_id, + name: value.name, version: value.version, author: value.author, summary: Arc::new(value.summary), diff --git a/crates/dtmm/src/ui/window/main.rs b/crates/dtmm/src/ui/window/main.rs index 7de4418..baa8e22 100644 --- a/crates/dtmm/src/ui/window/main.rs +++ b/crates/dtmm/src/ui/window/main.rs @@ -135,8 +135,13 @@ fn build_mod_list() -> impl Widget { }) .lens(lens!((usize, Arc, bool), 1).then(ModInfo::enabled.in_arc())); - let name = - Label::raw().lens(lens!((usize, Arc, bool), 1).then(ModInfo::name.in_arc())); + let name = Label::dynamic(|info: &Arc, _| { + info.nexus + .as_ref() + .map(|n| n.name.clone()) + .unwrap_or_else(|| info.name.clone()) + }) + .lens(lens!((usize, Arc, bool), 1)); let version = { let icon = { -- 2.45.3 From 510cbcb8b44f61aa578c7d1321a1b5444e96b1c4 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Tue, 14 Nov 2023 16:19:07 +0100 Subject: [PATCH 10/13] Use version number from Nexus import Non-bundled mods come without a `dtmt.cfg`, and therefore without a version number. But we need a version number at import to compare to for the Nexus update check. --- crates/dtmm/src/controller/import.rs | 17 ++++++++++++----- lib/nexusmods/src/lib.rs | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs index 05feba9..c5b5948 100644 --- a/crates/dtmm/src/controller/import.rs +++ b/crates/dtmm/src/controller/import.rs @@ -372,7 +372,7 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result Result Result Result Result Result Date: Thu, 16 Nov 2023 00:06:16 +0100 Subject: [PATCH 11/13] Prevent excessive debug logs --- crates/dtmm/src/controller/worker.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/dtmm/src/controller/worker.rs b/crates/dtmm/src/controller/worker.rs index 518c0a8..152030f 100644 --- a/crates/dtmm/src/controller/worker.rs +++ b/crates/dtmm/src/controller/worker.rs @@ -38,7 +38,9 @@ async fn handle_action( action_queue: Arc>>, ) { while let Some(action) = action_queue.write().await.recv().await { - tracing::debug!(?action); + if cfg!(debug_assertions) && !matches!(action, AsyncAction::Log(_)) { + tracing::debug!(?action); + } let event_sink = event_sink.clone(); match action { -- 2.45.3 From 8928d22bf6b97ce396c666708b68c71a0e28b106 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 16 Nov 2023 14:29:06 +0100 Subject: [PATCH 12/13] Delay mod loading The initial implementation of DML ended up loading mods quite late, which did give it the benefit of all `Manager`s being available. This change therefore moves mod loading until after those are initialized. But contrary to old DML, we still create a separate game state to make sure the game doesn't advance until mods are loaded. This avoids race conditions like the one where LogMeIn needs to come early in the load order. --- crates/dtmm/assets/mod_main.lua | 186 ++++++++++++++++---------------- 1 file changed, 92 insertions(+), 94 deletions(-) diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua index 2b329bf..e4006f6 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua @@ -11,100 +11,6 @@ local log = function(category, format, ...) end end --- Patch `GameStateMachine.init` to add our own state for loading mods. --- In the future, Fatshark might provide us with a dedicated way to do this. -local function patch_mod_loading_state() - local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") - - -- A necessary override. - -- The original does not proxy `dt` to `_state_update`, but we need that. - StateBootSubStateBase.update = function(self, dt) - local done, error = self:_state_update(dt) - local params = self._params - - if error then - return StateError, { error } - elseif done then - local next_index = params.sub_state_index + 1 - params.sub_state_index = next_index - local next_state_data = params.states[next_index] - - if next_state_data then - return next_state_data[1], self._params - else - self._parent:sub_states_done() - end - end - end - - local StateBootLoadMods = class("StateBootLoadMods", "StateBootSubStateBase") - - StateBootLoadMods.on_enter = function(self, parent, params) - log("StateBootLoadMods", "Entered") - StateBootLoadMods.super.on_enter(self, parent, params) - - local state_params = self:_state_params() - local package_manager = state_params.package_manager - - self._state = "load_package" - self._package_manager = package_manager - self._package_handles = { - ["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadMods", nil), - ["packages/dml"] = package_manager:load("packages/dml", "StateBootLoadMods", nil), - } - end - - StateBootLoadMods._state_update = function(self, dt) - local state = self._state - local package_manager = self._package_manager - - if state == "load_package" and package_manager:update() then - log("StateBootLoadMods", "Packages loaded, loading mods") - self._state = "load_mods" - local DML = require("scripts/mods/dml/init") - - local mod_data = require("scripts/mods/mod_data") - local mod_loader = DML.create_loader(mod_data, self._parent:gui()) - - self._dml = DML - Managers.mod = mod_loader - elseif state == "load_mods" and self._dml.update(Managers.mod, dt) then - log("StateBootLoadMods", "Mods loaded, exiting") - return true, false - end - - return false, false - end - - local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") - - local patched = false - - local GameStateMachine_init = GameStateMachine.init - GameStateMachine.init = function(self, parent, start_state, params, ...) - if not patched then - log("mod_main", "Injecting mod loading state") - patched = true - - -- Hardcoded position after `StateRequireScripts`. - -- We do want to wait until then, so that most of the game's core - -- systems are at least loaded and can be hooked, even if they aren't - -- running, yet. - local pos = 4 - table.insert(params.states, pos, { - StateBootLoadMods, - { - package_manager = params.package_manager, - }, - }) - end - - GameStateMachine_init(self, parent, start_state, params, ...) - end - - log("mod_main", "Mod patching complete") -end - log("mod_main", "Initializing mods...") local require_store = {} @@ -199,6 +105,98 @@ end require("scripts/main") log("mod_main", "'scripts/main' loaded") +-- Inject our state into the game. The state needs to run after `StateGame._init_managers`, +-- since some parts of DMF, and presumably other mods, depend on some of those managers to exist. +local function patch_mod_loading_state() + local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base") + local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase") + local StateGameLoadMods = class("StateGameLoadMods") + + StateBootLoadDML.on_enter = function(self, parent, params) + log("StateBootLoadDML", "Entered") + StateBootLoadDML.super.on_enter(self, parent, params) + + local state_params = self:_state_params() + local package_manager = state_params.package_manager + + self._package_manager = package_manager + self._package_handles = { + ["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil), + ["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil), + } + end + + StateBootLoadDML._state_update = function(self, dt) + local package_manager = self._package_manager + + if package_manager:update() then + local DML = require("scripts/mods/dml/init") + local mod_data = require("scripts/mods/mod_data") + local mod_loader = DML.create_loader(mod_data) + Managers.mod = mod_loader + log("StateBootLoadDML", "DML loaded, exiting") + return true, false + end + + return false, false + end + + + function StateGameLoadMods:on_enter(_, params) + log("StateGameLoadMods", "Entered") + self._next_state = require("scripts/game_states/game/state_splash") + self._next_state_params = params + end + + function StateGameLoadMods:update(main_dt) + local state = self._loading_state + + -- We're relying on the fact that DML internally makes sure + -- that `Managers.mod:update()` is being called appropriately. + -- The implementation as of this writing is to hook `StateGame.update`. + if Managers.mod:all_mods_loaded() then + Log.info("StateGameLoadMods", "Mods loaded, exiting") + return self._next_state, self._next_state_params + end + end + + local GameStateMachine = require("scripts/foundation/utilities/game_state_machine") + local GameStateMachine_init = GameStateMachine.init + GameStateMachine.init = function(self, parent, start_state, params, creation_context, state_change_callbacks, name) + if name == "Main" then + log("mod_main", "Injecting StateBootLoadDML") + + -- Hardcoded position after `StateRequireScripts`. + -- We need to wait until then to even begin most of our stuff, + -- so that most of the game's core systems are at least loaded and can be hooked, + -- even if they aren't running, yet. + local pos = 4 + table.insert(params.states, pos, { + StateBootLoadDML, + { + package_manager = params.package_manager, + }, + }) + + GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) + elseif name == "Game" then + log("mod_main", "Injection StateGameLoadMods") + -- The second time around, we want to be the first, so we pass our own + -- 'start_state'. + -- We can't just have the state machine be initialized and then change its `_next_state`, as by the end of + -- `init`, a bunch of stuff will already be initialized. + GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name) + -- And since we're done now, we can revert the function to its original + GameStateMachine.init = GameStateMachine_init + + return + else + -- In all other cases, simply call the original + GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name) + end + end +end + -- Override `init` to run our injection function init() patch_mod_loading_state() -- 2.45.3 From c8b08cc2ccebed07e454c14bc6599b3085fa7a94 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 24 Nov 2023 00:50:56 +0100 Subject: [PATCH 13/13] dtmm: Use `dtmt.cfg` for non-bundled mods Closes #144. --- crates/dtmm/src/controller/import.rs | 106 +++++++++++++++------------ 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs index c5b5948..131d01f 100644 --- a/crates/dtmm/src/controller/import.rs +++ b/crates/dtmm/src/controller/import.rs @@ -31,7 +31,7 @@ fn find_archive_file( // from legacy mods. // 1. Create a global function `new_mod` that stores // the relevant bits in global variables. -// 2. Run the `.mod` file, which will merely return a table. +// 2. Run the `.mod` file, which will return a table. // 3. Run the `run` function from that table. // 4. Access the global variables from #1. #[tracing::instrument] @@ -203,6 +203,37 @@ end // still end up creating tarbombs and Nexus does its own re-packaging. #[tracing::instrument(skip(archive))] fn extract_mod_config(archive: &mut ZipArchive) -> Result<(ModConfig, String)> { + let legacy_mod_data = if let Some(name) = find_archive_file(archive, ".mod") { + let (mod_id, resources) = { + let mut f = archive + .by_name(&name) + .wrap_err("Failed to read `.mod` file from archive")?; + + let mut buf = Vec::with_capacity(f.size() as usize); + f.read_to_end(&mut buf) + .wrap_err("Failed to read `.mod` file from archive")?; + + let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?; + parse_mod_id_file(&data) + .wrap_err("Invalid `.mod` file") + .note( + "The `.mod` file's `run` function may not contain any additional logic \ + besides the default.", + ) + .suggestion("Contact the mod author to fix this.")? + }; + + let root = if let Some(index) = name.rfind('/') { + name[..index].to_string() + } else { + String::new() + }; + + Some((mod_id, resources, root)) + } else { + None + }; + if let Some(name) = find_archive_file(archive, "dtmt.cfg") { let mut f = archive .by_name(&name) @@ -214,52 +245,30 @@ fn extract_mod_config(archive: &mut ZipArchive) -> Result<(Mo let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?; - let cfg = serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")?; - let root = name - .strip_suffix("dtmt.cfg") - .expect("String must end with that suffix") - .to_string(); + let mut cfg: ModConfig = serde_sjson::from_str(&data) + .wrap_err("Failed to deserialize mod config") + .suggestion("Contact the mod author to fix this.")?; - Ok((cfg, root)) - } else if let Some(name) = find_archive_file(archive, ".mod") { - let (mod_id, resources) = { - let mut f = archive - .by_name(&name) - .wrap_err("Failed to read `.mod` file from archive")?; + if let Some((mod_id, resources, root)) = legacy_mod_data { + if cfg.id != mod_id { + let err = eyre::eyre!("Mod ID in `dtmt.cfg` does not match mod ID in `.mod` file"); + return Err(err).suggestion("Contact the mod author to fix this."); + } - let mut buf = Vec::with_capacity(f.size() as usize); - f.read_to_end(&mut buf) - .wrap_err("Failed to read `.mod` file from archive")?; + cfg.resources = resources; - let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?; - parse_mod_id_file(&data).wrap_err("Invalid `.mod` file")? - }; - - let cfg = ModConfig { - bundled: false, - dir: PathBuf::new(), - id: mod_id.clone(), - name: mod_id, - summary: String::new(), - version: String::new(), - description: None, - author: None, - image: None, - categories: Vec::new(), - packages: Vec::new(), - resources, - depends: Vec::new(), - }; - let root = if let Some(index) = name.rfind('/') { - name[..index].to_string() + Ok((cfg, root)) } else { - String::new() - }; + let root = name + .strip_suffix("dtmt.cfg") + .expect("String must end with that suffix") + .to_string(); - Ok((cfg, root)) + Ok((cfg, root)) + } } else { eyre::bail!( - "Mod needs either a config file or `.mod` file. \ + "Mod needs a config file or `.mod` file. \ Please get in touch with the author to provide a properly packaged mod." ); } @@ -322,11 +331,11 @@ fn extract_legacy_mod( .wrap_err_with(|| format!("Failed to get file at index {}", i))?; let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else { - let err = eyre::eyre!("File name in archive is not a safe path value."); - return Err(err).with_suggestion(|| { + let err = eyre::eyre!("File name in archive is not a safe path value.").suggestion( "Only use well-known applications to create the ZIP archive, \ - and don't create paths that point outside the archive directory." - }); + and don't create paths that point outside the archive directory.", + ); + return Err(err); }; let Ok(suffix) = name.strip_prefix(&root) else { @@ -430,10 +439,11 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result img, Err(err) => { - let err = Report::msg(err.to_string()).wrap_err("Invalid image data"); - return Err(err).with_suggestion(|| { - "Supported formats are: PNG, JPEG, Bitmap and WebP".to_string() - }); + let err = Report::msg(err.to_string()) + .wrap_err("Invalid image data") + .note("Supported formats are: PNG, JPEG, Bitmap and WebP") + .suggestion("Contact the mod author to fix this"); + return Err(err); } }; -- 2.45.3