bitsquid_dt/dtmt
bitsquid_dt/dtmt
2
Fork 0
Code Issues 34 Pull requests 11 Projects 1 Releases 3 Packages 1 Wiki Activity
Var Dump: 
dumpVar: only available in dev mode
Mailing List

Merge pull request 'Implement non-bundled mods' (#125) from feat/loose-files into master
Some checks are pending
build/linux Build for the target platform: linux
build/msvc Build for the target platform: msvc

Browse source 
Reviewed-on: #125
This commit is contained in:
Lucas Schwiderski 2023-11-24 11:52:47 +01:00
commit 12e01075d9
22 changed files with 1588 additions and 928 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

60
Cargo.lock generated
View file

@ -39,12 +39,10 @@ dependencies = [
[[package]]
name = "ansi-parser"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127"
version = "0.9.0"
dependencies = [
"heapless",
"nom 4.2.3",
"nom",
]
[[package]]
@ -381,7 +379,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom 7.1.3",
"nom",
]
[[package]]
@ -896,6 +894,7 @@ name = "dtmm"
version = "0.1.0"
dependencies = [
"ansi-parser",
"async-recursion",
"bitflags 1.3.2",
"clap",
"color-eyre",
@ -906,6 +905,8 @@ dependencies = [
"dtmt-shared",
"futures",
"lazy_static",
"luajit2-sys",
"minijinja",
"nexusmods",
"oodle",
"path-slash",
@ -1390,7 +1391,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check 0.9.4",
"version_check",
]
[[package]]
@ -1614,12 +1615,12 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
[[package]]
name = "heapless"
version = "0.5.6"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1"
checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422"
dependencies = [
"as-slice",
"generic-array 0.13.3",
"generic-array 0.14.7",
"hash32",
"stable_deref_trait",
]
@ -1747,7 +1748,7 @@ dependencies = [
"serde",
"sized-chunks",
"typenum",
"version_check 0.9.4",
"version_check",
]
[[package]]
@ -2084,6 +2085,15 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minijinja"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb"
dependencies = [
"serde",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -2175,16 +2185,6 @@ dependencies = [
"memoffset 0.6.5",
]
[[package]]
name = "nom"
version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
dependencies = [
"memchr",
"version_check 0.1.5",
]
[[package]]
name = "nom"
version = "7.1.3"
@ -2203,7 +2203,7 @@ checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3"
dependencies = [
"bytecount",
"memchr",
"nom 7.1.3",
"nom",
]
[[package]]
@ -2673,7 +2673,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"version_check 0.9.4",
"version_check",
]
[[package]]
@ -2684,7 +2684,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check 0.9.4",
"version_check",
]
[[package]]
@ -3102,7 +3102,7 @@ dependencies = [
name = "serde_sjson"
version = "1.0.0"
dependencies = [
"nom 7.1.3",
"nom",
"nom_locate",
"serde",
]
@ -3832,7 +3832,7 @@ version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check 0.9.4",
"version_check",
]
[[package]]
@ -3966,12 +3966,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29"
[[package]]
name = "version_check"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
[[package]]
name = "version_check"
version = "0.9.4"
@ -4384,7 +4378,3 @@ dependencies = [
"cc",
"pkg-config",
]
[[patch.unused]]
name = "ansi-parser"
version = "0.9.0"

5
Cargo.toml
View file

@ -7,10 +7,7 @@ members = [
"lib/oodle",
"lib/sdk",
"lib/serde_sjson",
]
exclude = [
"lib/color-eyre",
"lib/ansi-parser",
"lib/luajit2-sys",
]
[patch.crates-io]

5
crates/dtmm/Cargo.toml
View file

@ -31,5 +31,8 @@ lazy_static = "1.4.0"
colors-transform = "0.2.11"
usvg = "0.25.0"
druid-widget-nursery = "0.1"
ansi-parser = "0.8.0"
ansi-parser = "0.9.0"
string_template = "0.2.1"
luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" }
async-recursion = "1.0.5"
minijinja = "1.0.10"

27
crates/dtmm/assets/mod_data.lua.j2 Normal file
View file

@ -0,0 +1,27 @@
return {
{% for mod in mods %}
{
id = "{{ mod.id }}",
name = "{{ mod.name }}",
bundled = {{ mod.bundled }},
packages = {
{% for pkg in mod.packages %}
"{{ pkg }}",
{% endfor %}
},
run = function()
{% if mod.data is none %}
return dofile("{{ mod.init }}")
{% else %}
new_mod("{{ mod.id }}", {
mod_script = "{{ mod.init }}",
mod_data = "{{ mod.data }}",
{% if not mod.localization is none %}
mod_localization = "{{ mod.localization }}",
{% endif %}
})
{% endif %}
end,
},
{% endfor %}
}

191
crates/dtmm/assets/mod_main.lua
View file

@ -11,100 +11,6 @@ local log = function(category, format, ...)
end
end
-- Patch `GameStateMachine.init` to add our own state for loading mods.
-- In the future, Fatshark might provide us with a dedicated way to do this.
local function patch_mod_loading_state()
local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base")
-- A necessary override.
-- The original does not proxy `dt` to `_state_update`, but we need that.
StateBootSubStateBase.update = function(self, dt)
local done, error = self:_state_update(dt)
local params = self._params
if error then
return StateError, { error }
elseif done then
local next_index = params.sub_state_index + 1
params.sub_state_index = next_index
local next_state_data = params.states[next_index]
if next_state_data then
return next_state_data[1], self._params
else
self._parent:sub_states_done()
end
end
end
local StateBootLoadMods = class("StateBootLoadMods", "StateBootSubStateBase")
StateBootLoadMods.on_enter = function(self, parent, params)
log("StateBootLoadMods", "Entered")
StateBootLoadMods.super.on_enter(self, parent, params)
local state_params = self:_state_params()
local package_manager = state_params.package_manager
self._state = "load_package"
self._package_manager = package_manager
self._package_handles = {
["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadMods", nil),
["packages/dml"] = package_manager:load("packages/dml", "StateBootLoadMods", nil),
}
end
StateBootLoadMods._state_update = function(self, dt)
local state = self._state
local package_manager = self._package_manager
if state == "load_package" and package_manager:update() then
log("StateBootLoadMods", "Packages loaded, loading mods")
self._state = "load_mods"
local ModLoader = require("scripts/mods/dml/init")
local mod_data = require("scripts/mods/mod_data")
local mod_loader = ModLoader:new(mod_data, self._parent:gui())
self._mod_loader = mod_loader
Managers.mod = mod_loader
elseif state == "load_mods" and self._mod_loader:update(dt) then
log("StateBootLoadMods", "Mods loaded, exiting")
return true, false
end
return false, false
end
local GameStateMachine = require("scripts/foundation/utilities/game_state_machine")
local patched = false
local GameStateMachine_init = GameStateMachine.init
GameStateMachine.init = function(self, parent, start_state, params, ...)
if not patched then
log("mod_main", "Injecting mod loading state")
patched = true
-- Hardcoded position after `StateRequireScripts`.
-- We do want to wait until then, so that most of the game's core
-- systems are at least loaded and can be hooked, even if they aren't
-- running, yet.
local pos = 4
table.insert(params.states, pos, {
StateBootLoadMods,
{
package_manager = params.package_manager,
},
})
end
GameStateMachine_init(self, parent, start_state, params, ...)
end
log("mod_main", "Mod patching complete")
end
log("mod_main", "Initializing mods...")
local require_store = {}
@ -112,7 +18,7 @@ local require_store = {}
-- This token is treated as a string template and filled by DTMM during deployment.
-- This allows hiding unsafe I/O functions behind a setting.
-- It's also a valid table definition, thereby degrading gracefully when not replaced.
local is_io_enabled = { { is_io_enabled } } -- luacheck: ignore 113
local is_io_enabled = {{ is_io_enabled }} -- luacheck: ignore 113
local lua_libs = {
debug = debug,
os = {
@ -138,7 +44,8 @@ Mods = {
-- Fatshark's code scrubs them.
-- The loader can then decide to pass them on to mods, or ignore them
lua = setmetatable({}, { __index = lua_libs }),
require_store = require_store
require_store = require_store,
original_require = require,
}
local can_insert = function(filepath, new_result)
@ -198,6 +105,98 @@ end
require("scripts/main")
log("mod_main", "'scripts/main' loaded")
-- Inject our state into the game. The state needs to run after `StateGame._init_managers`,
-- since some parts of DMF, and presumably other mods, depend on some of those managers to exist.
local function patch_mod_loading_state()
local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base")
local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase")
local StateGameLoadMods = class("StateGameLoadMods")
StateBootLoadDML.on_enter = function(self, parent, params)
log("StateBootLoadDML", "Entered")
StateBootLoadDML.super.on_enter(self, parent, params)
local state_params = self:_state_params()
local package_manager = state_params.package_manager
self._package_manager = package_manager
self._package_handles = {
["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil),
["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil),
}
end
StateBootLoadDML._state_update = function(self, dt)
local package_manager = self._package_manager
if package_manager:update() then
local DML = require("scripts/mods/dml/init")
local mod_data = require("scripts/mods/mod_data")
local mod_loader = DML.create_loader(mod_data)
Managers.mod = mod_loader
log("StateBootLoadDML", "DML loaded, exiting")
return true, false
end
return false, false
end
function StateGameLoadMods:on_enter(_, params)
log("StateGameLoadMods", "Entered")
self._next_state = require("scripts/game_states/game/state_splash")
self._next_state_params = params
end
function StateGameLoadMods:update(main_dt)
local state = self._loading_state
-- We're relying on the fact that DML internally makes sure
-- that `Managers.mod:update()` is being called appropriately.
-- The implementation as of this writing is to hook `StateGame.update`.
if Managers.mod:all_mods_loaded() then
Log.info("StateGameLoadMods", "Mods loaded, exiting")
return self._next_state, self._next_state_params
end
end
local GameStateMachine = require("scripts/foundation/utilities/game_state_machine")
local GameStateMachine_init = GameStateMachine.init
GameStateMachine.init = function(self, parent, start_state, params, creation_context, state_change_callbacks, name)
if name == "Main" then
log("mod_main", "Injecting StateBootLoadDML")
-- Hardcoded position after `StateRequireScripts`.
-- We need to wait until then to even begin most of our stuff,
-- so that most of the game's core systems are at least loaded and can be hooked,
-- even if they aren't running, yet.
local pos = 4
table.insert(params.states, pos, {
StateBootLoadDML,
{
package_manager = params.package_manager,
},
})
GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name)
elseif name == "Game" then
log("mod_main", "Injection StateGameLoadMods")
-- The second time around, we want to be the first, so we pass our own
-- 'start_state'.
-- We can't just have the state machine be initialized and then change its `_next_state`, as by the end of
-- `init`, a bunch of stuff will already be initialized.
GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name)
-- And since we're done now, we can revert the function to its original
GameStateMachine.init = GameStateMachine_init
return
else
-- In all other cases, simply call the original
GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name)
end
end
end
-- Override `init` to run our injection
function init()
patch_mod_loading_state()

168
crates/dtmm/src/controller/app.rs
View file

@ -1,18 +1,17 @@
use std::collections::HashMap;
use std::io::{Cursor, ErrorKind, Read};
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use color_eyre::eyre::{self, Context};
use color_eyre::{Help, Report, Result};
use druid::im::Vector;
use druid::{FileInfo, ImageBuf};
use druid::ImageBuf;
use dtmt_shared::ModConfig;
use nexusmods::Api as NexusApi;
use tokio::fs::{self, DirEntry, File};
use tokio_stream::wrappers::ReadDirStream;
use tokio_stream::StreamExt;
use zip::ZipArchive;
use crate::state::{ActionState, InitialLoadResult, ModInfo, ModOrder, NexusInfo, PackageInfo};
use crate::util;
@ -20,161 +19,6 @@ use crate::util::config::{ConfigSerialize, LoadOrderEntry};
use super::read_sjson_file;
#[tracing::instrument(skip(state))]
pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<ModInfo> {
let data = fs::read(&info.path)
.await
.wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?;
let data = Cursor::new(data);
let nexus = if let Some((_, id, _, _)) = info
.path
.file_name()
.and_then(|s| s.to_str())
.and_then(NexusApi::parse_file_name)
{
if !state.nexus_api_key.is_empty() {
let api = NexusApi::new(state.nexus_api_key.to_string())?;
let mod_info = api
.mods_id(id)
.await
.wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?;
Some(NexusInfo::from(mod_info))
} else {
None
}
} else {
None
};
let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
if tracing::enabled!(tracing::Level::DEBUG) {
let names = archive.file_names().fold(String::new(), |mut s, name| {
s.push('\n');
s.push_str(name);
s
});
tracing::debug!("Archive contents:{}", names);
}
let dir_name = {
let f = archive.by_index(0).wrap_err("Archive is empty")?;
if !f.is_dir() {
let err = eyre::eyre!("archive does not have a top-level directory");
return Err(err).with_suggestion(|| "Use 'dtmt build' to create the mod archive.");
}
let name = f.name();
// The directory name is returned with a trailing slash, which we don't want
name[..(name.len().saturating_sub(1))].to_string()
};
tracing::info!("Importing mod {}", dir_name);
let names: Vec<_> = archive.file_names().map(|s| s.to_string()).collect();
let mod_cfg: ModConfig = {
let name = names
.iter()
.find(|name| name.ends_with("dtmt.cfg"))
.ok_or_else(|| eyre::eyre!("archive does not contain mod config"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read mod config from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read mod config from archive")?;
let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")?
};
tracing::debug!(?mod_cfg);
let files: HashMap<String, Vec<String>> = {
let name = names
.iter()
.find(|name| name.ends_with("files.sjson"))
.ok_or_else(|| eyre::eyre!("archive does not contain file index"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read file index from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")?
};
tracing::trace!(?files);
let image = if let Some(path) = &mod_cfg.image {
let name = names
.iter()
.find(|name| name.ends_with(&path.display().to_string()))
.ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read image file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
let img = match ImageBuf::from_data(&buf) {
Ok(img) => img,
Err(err) => {
let err = Report::msg(err.to_string()).wrap_err("Invalid image data");
return Err(err).with_suggestion(|| {
"Supported formats are: PNG, JPEG, Bitmap and WebP".to_string()
});
}
};
Some(img)
} else {
None
};
let mod_dir = state.mod_dir;
tracing::trace!("Creating mods directory {}", mod_dir.display());
fs::create_dir_all(Arc::as_ref(&mod_dir))
.await
.wrap_err_with(|| format!("Failed to create data directory {}", mod_dir.display()))?;
tracing::trace!("Extracting mod archive to {}", mod_dir.display());
archive
.extract(Arc::as_ref(&mod_dir))
.wrap_err_with(|| format!("Failed to extract archive to {}", mod_dir.display()))?;
if let Some(nexus) = &nexus {
let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?;
let path = mod_dir.join(&mod_cfg.id).join("nexus.sjson");
fs::write(&path, data.as_bytes())
.await
.wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?;
}
let packages = files
.into_iter()
.map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect())))
.collect();
let info = ModInfo::new(mod_cfg, packages, image, nexus);
Ok(info)
}
#[tracing::instrument(skip(state))]
pub(crate) async fn delete_mod(state: ActionState, info: &ModInfo) -> Result<()> {
let mod_dir = state.mod_dir.join(&info.id);
@ -229,9 +73,13 @@ async fn read_mod_dir_entry(res: Result<DirEntry>) -> Result<ModInfo> {
Err(err) => return Err(err),
};
let files: HashMap<String, Vec<String>> = read_sjson_file(&index_path)
let files: HashMap<String, Vec<String>> = if cfg.bundled {
read_sjson_file(&index_path)
.await
.wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?;
.wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?
} else {
Default::default()
};
let image = if let Some(path) = &cfg.image {
let path = entry.path().join(path);

868
crates/dtmm/src/controller/deploy.rs Normal file
View file

@ -0,0 +1,868 @@
use std::collections::HashMap;
use std::io::{Cursor, ErrorKind};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Help, Report, Result};
use futures::StreamExt;
use futures::{stream, TryStreamExt};
use minijinja::Environment;
use sdk::filetype::lua;
use sdk::filetype::package::Package;
use sdk::murmur::Murmur64;
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use serde::{Deserialize, Serialize};
use string_template::Template;
use time::OffsetDateTime;
use tokio::fs::{self, DirEntry};
use tokio::io::AsyncWriteExt;
use tracing::Instrument;
use super::read_sjson_file;
use crate::controller::app::check_mod_order;
use crate::state::{ActionState, PackageInfo};
pub const MOD_BUNDLE_NAME: &str = "packages/mods";
pub const BOOT_BUNDLE_NAME: &str = "packages/boot";
pub const DML_BUNDLE_NAME: &str = "packages/dml";
pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
pub const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini";
pub const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson";
#[derive(Debug, Serialize, Deserialize)]
pub struct DeploymentData {
pub bundles: Vec<String>,
pub mod_folders: Vec<String>,
#[serde(with = "time::serde::iso8601")]
pub timestamp: OffsetDateTime,
}
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
where
P: AsRef<Path> + std::fmt::Debug,
{
let path = path.as_ref();
let backup_path = {
let mut p = PathBuf::from(path);
let ext = if let Some(ext) = p.extension() {
ext.to_string_lossy().to_string() + ".bak"
} else {
String::from("bak")
};
p.set_extension(ext);
p
};
let file_name = path
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| String::from("file"));
let bin = match fs::read(&backup_path).await {
Ok(bin) => bin,
Err(err) if err.kind() == ErrorKind::NotFound => {
// TODO: This doesn't need to be awaited here, yet.
// I only need to make sure it has finished before writing the changed bundle.
tracing::debug!(
"Backup does not exist. Backing up original {} to '{}'",
file_name,
backup_path.display()
);
fs::copy(path, &backup_path).await.wrap_err_with(|| {
format!(
"Failed to back up {} '{}' to '{}'",
file_name,
path.display(),
backup_path.display()
)
})?;
tracing::debug!("Reading {} from original '{}'", file_name, path.display());
fs::read(path).await.wrap_err_with(|| {
format!("Failed to read {} file: {}", file_name, path.display())
})?
}
Err(err) => {
return Err(err).wrap_err_with(|| {
format!(
"Failed to read {} from backup '{}'",
file_name,
backup_path.display()
)
});
}
};
Ok(bin)
}
#[tracing::instrument(skip_all)]
async fn patch_game_settings(state: Arc<ActionState>) -> Result<()> {
let settings_path = state.game_dir.join("bundle").join(SETTINGS_FILE_PATH);
let settings = read_file_with_backup(&settings_path)
.await
.wrap_err("Failed to read settings.ini")?;
let settings = String::from_utf8(settings).wrap_err("Settings.ini is not valid UTF-8")?;
let mut f = fs::File::create(&settings_path)
.await
.wrap_err_with(|| format!("Failed to open {}", settings_path.display()))?;
let Some(i) = settings.find("boot_script =") else {
eyre::bail!("couldn't find 'boot_script' field");
};
f.write_all(settings[0..i].as_bytes()).await?;
f.write_all(b"boot_script = \"scripts/mod_main\"").await?;
let Some(j) = settings[i..].find('\n') else {
eyre::bail!("couldn't find end of 'boot_script' field");
};
f.write_all(settings[(i + j)..].as_bytes()).await?;
Ok(())
}
#[tracing::instrument(skip_all, fields(package = info.name))]
fn make_package(info: &PackageInfo) -> Result<Package> {
let mut pkg = Package::new(info.name.clone(), PathBuf::new());
for f in &info.files {
let mut it = f.rsplit('.');
let file_type = it
.next()
.ok_or_else(|| eyre::eyre!("missing file extension"))
.and_then(BundleFileType::from_str)
.wrap_err("Invalid file name in package info")?;
let name: String = it.collect();
pkg.add_file(file_type, name);
}
Ok(pkg)
}
#[tracing::instrument]
async fn copy_recursive(
from: impl Into<PathBuf> + std::fmt::Debug,
to: impl AsRef<Path> + std::fmt::Debug,
) -> Result<()> {
let to = to.as_ref();
#[tracing::instrument]
async fn handle_dir(from: PathBuf) -> Result<Vec<(bool, DirEntry)>> {
let mut dir = fs::read_dir(&from)
.await
.wrap_err("Failed to read directory")?;
let mut entries = Vec::new();
while let Some(entry) = dir.next_entry().await? {
let meta = entry.metadata().await.wrap_err_with(|| {
format!("Failed to get metadata for '{}'", entry.path().display())
})?;
entries.push((meta.is_dir(), entry));
}
Ok(entries)
}
let base = from.into();
stream::unfold(vec![base.clone()], |mut state| async {
let from = state.pop()?;
let inner = match handle_dir(from).await {
Ok(entries) => {
for (is_dir, entry) in &entries {
if *is_dir {
state.push(entry.path());
}
}
stream::iter(entries).map(Ok).left_stream()
}
Err(e) => stream::once(async { Err(e) }).right_stream(),
};
Some((inner, state))
})
.flatten()
.try_for_each(|(is_dir, entry)| {
let path = entry.path();
let dest = path
.strip_prefix(&base)
.map(|suffix| to.join(suffix))
.expect("all entries are relative to the directory we are walking");
async move {
if is_dir {
tracing::trace!("Creating directory '{}'", dest.display());
// Instead of trying to filter "already exists" errors out explicitly,
// we just ignore all. It'll fail eventually with the next copy operation.
let _ = fs::create_dir(&dest).await;
Ok(())
} else {
tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display());
fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| {
format!(
"Failed to copy file '{}' -> '{}'",
path.display(),
dest.display()
)
})
}
}
})
.await
.map(|_| ())
}
#[tracing::instrument(skip(state))]
async fn copy_mod_folders(state: Arc<ActionState>) -> Result<Vec<String>> {
let game_dir = Arc::clone(&state.game_dir);
let mut tasks = Vec::new();
for mod_info in state
.mods
.iter()
.filter(|m| m.id != "dml" && m.enabled && !m.bundled)
{
let span = tracing::trace_span!("copying legacy mod", name = mod_info.name);
let _enter = span.enter();
let mod_id = mod_info.id.clone();
let mod_dir = Arc::clone(&state.mod_dir);
let game_dir = Arc::clone(&game_dir);
let task = async move {
let from = mod_dir.join(&mod_id);
let to = game_dir.join("mods").join(&mod_id);
tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id);
let _ = fs::create_dir_all(&to).await;
copy_recursive(&from, &to).await.wrap_err_with(|| {
format!(
"Failed to copy legacy mod from '{}' to '{}'",
from.display(),
to.display()
)
})?;
Ok::<_, Report>(mod_id)
};
tasks.push(task);
}
let ids = futures::future::try_join_all(tasks).await?;
Ok(ids)
}
fn build_mod_data_lua(state: Arc<ActionState>) -> Result<String> {
#[derive(Serialize)]
struct TemplateDataMod {
id: String,
name: String,
bundled: bool,
init: String,
data: Option<String>,
localization: Option<String>,
packages: Vec<String>,
}
let mut env = Environment::new();
env.add_template("mod_data.lua", include_str!("../../assets/mod_data.lua.j2"))
.wrap_err("Failed to compile template for `mod_data.lua`")?;
let tmpl = env
.get_template("mod_data.lua")
.wrap_err("Failed to get template `mod_data.lua`")?;
let data: Vec<TemplateDataMod> = state
.mods
.iter()
.filter_map(|m| {
if m.id == "dml" || !m.enabled {
return None;
}
Some(TemplateDataMod {
id: m.id.clone(),
name: m.name.clone(),
bundled: m.bundled,
init: m.resources.init.to_string_lossy().to_string(),
data: m
.resources
.data
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
localization: m
.resources
.localization
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
packages: m.packages.iter().map(|p| p.name.clone()).collect(),
})
})
.collect();
let lua = tmpl
.render(minijinja::context!(mods => data))
.wrap_err("Failed to render template `mod_data.lua`")?;
tracing::debug!("mod_data.lua:\n{}", lua);
Ok(lua)
}
#[tracing::instrument(skip_all)]
async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string());
let mut tasks = Vec::new();
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let mut bundles = Vec::new();
{
tracing::trace!("Building mod data script");
let span = tracing::debug_span!("Building mod data script");
let _enter = span.enter();
let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?;
tracing::trace!("Compiling mod data script");
let file =
lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?;
tracing::trace!("Compile mod data script");
mod_bundle.add_file(file);
}
tracing::trace!("Preparing tasks to deploy bundle files");
for mod_info in state
.mods
.iter()
.filter(|m| m.id != "dml" && m.enabled && m.bundled)
{
let span = tracing::trace_span!("building mod packages", name = mod_info.name);
let _enter = span.enter();
let mod_dir = state.mod_dir.join(&mod_info.id);
for pkg_info in &mod_info.packages {
let span = tracing::trace_span!("building package", name = pkg_info.name);
let _enter = span.enter();
tracing::trace!(
"Building package {} for mod {}",
pkg_info.name,
mod_info.name
);
let pkg = make_package(pkg_info).wrap_err("Failed to make package")?;
let mut variant = BundleFileVariant::new();
let bin = pkg
.to_binary()
.wrap_err("Failed to serialize package to binary")?;
variant.set_data(bin);
let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package);
file.add_variant(variant);
tracing::trace!(
"Compiled package {} for mod {}",
pkg_info.name,
mod_info.name
);
mod_bundle.add_file(file);
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = mod_dir.join(&bundle_name);
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
// Explicitely drop the guard, so that we can move the span
// into the async operation
drop(_enter);
let ctx = state.ctx.clone();
let task = async move {
let bundle = {
let bin = fs::read(&src).await.wrap_err_with(|| {
format!("Failed to read bundle file '{}'", src.display())
})?;
let name = Bundle::get_name_from_path(&ctx, &src);
Bundle::from_binary(&ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?
};
tracing::debug!(
src = %src.display(),
dest = %dest.display(),
"Copying bundle '{}' for mod '{}'",
pkg_name,
mod_name,
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
Ok::<Bundle, color_eyre::Report>(bundle)
}
.instrument(span);
tasks.push(task);
}
}
tracing::debug!("Copying {} mod bundles", tasks.len());
let mut tasks = stream::iter(tasks).buffer_unordered(10);
while let Some(res) = tasks.next().await {
let bundle = res?;
bundles.push(bundle);
}
{
let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64()));
tracing::trace!("Writing mod bundle to '{}'", path.display());
fs::write(&path, mod_bundle.to_binary()?)
.await
.wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?;
}
bundles.push(mod_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all)]
async fn patch_boot_bundle(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())));
let mut bundles = Vec::with_capacity(2);
let mut boot_bundle = async {
let bin = read_file_with_backup(&bundle_path)
.await
.wrap_err("Failed to read boot bundle")?;
Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin)
.wrap_err("Failed to parse boot bundle")
}
.instrument(tracing::trace_span!("read boot bundle"))
.await
.wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?;
{
tracing::trace!("Adding mod package file to boot bundle");
let span = tracing::trace_span!("create mod package file");
let _enter = span.enter();
let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new());
for mod_info in &state.mods {
for pkg_info in &mod_info.packages {
pkg.add_file(BundleFileType::Package, &pkg_info.name);
}
}
pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT);
let mut variant = BundleFileVariant::new();
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
tracing::trace!("Handling DML packages and bundle");
let span = tracing::trace_span!("handle DML");
let _enter = span.enter();
let mut variant = BundleFileVariant::new();
let mod_info = state
.mods
.iter()
.find(|m| m.id == "dml")
.ok_or_else(|| eyre::eyre!("DML not found in mod list"))?;
let pkg_info = mod_info
.packages
.get(0)
.ok_or_else(|| eyre::eyre!("invalid mod package for DML"))
.with_suggestion(|| "Re-download and import the newest version.".to_string())?;
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = state.mod_dir.join(&mod_info.id).join(&bundle_name);
{
let bin = fs::read(&src)
.await
.wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?;
let name = Bundle::get_name_from_path(&state.ctx, &src);
let dml_bundle = Bundle::from_binary(&state.ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?;
bundles.push(dml_bundle);
};
{
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
tracing::debug!(
"Copying bundle {} for mod {}: {} -> {}",
pkg_name,
mod_name,
src.display(),
dest.display()
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
}
let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?;
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
let span = tracing::debug_span!("Importing mod main script");
let _enter = span.enter();
let is_io_enabled = format!("{}", state.is_io_enabled);
let mut data = HashMap::new();
data.insert("is_io_enabled", is_io_enabled.as_str());
let tmpl = include_str!("../../assets/mod_main.lua");
let lua = Template::new(tmpl).render(&data);
tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
let file =
lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
boot_bundle.add_file(file);
}
async {
let bin = boot_bundle
.to_binary()
.wrap_err("Failed to serialize boot bundle")?;
fs::write(&bundle_path, bin)
.await
.wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display()))
}
.instrument(tracing::trace_span!("write boot bundle"))
.await?;
bundles.push(boot_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn patch_bundle_database<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let mut db = {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("Failed to read bundle database")?;
let mut r = Cursor::new(bin);
let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?;
tracing::trace!("Finished parsing bundle database");
db
};
for bundle in bundles.as_ref() {
tracing::trace!("Adding '{}' to bundle database", bundle.name().display());
db.add_bundle(bundle);
}
{
let bin = db
.to_binary()
.wrap_err("Failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})?;
}
Ok(())
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn write_deployment_data<B>(
state: Arc<ActionState>,
bundles: B,
mod_folders: Vec<String>,
) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let info = DeploymentData {
timestamp: OffsetDateTime::now_utc(),
bundles: bundles
.as_ref()
.iter()
.map(|bundle| format!("{:x}", bundle.name().to_murmur64()))
.collect(),
// TODO:
mod_folders,
};
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?;
fs::write(&path, &data)
.await
.wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.game_dir.display(),
mods = state.mods.len()
))]
pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> {
let state = Arc::new(state);
let bundle_dir = state.game_dir.join("bundle");
let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()));
if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999")))
.await
.is_ok()
{
let err = eyre::eyre!("Found dtkit-patch-based mod installation.");
return Err(err)
.with_suggestion(|| {
"If you're a mod author and saved projects directly in 'mods/', \
use DTMT to migrate them to the new project structure."
.to_string()
})
.with_suggestion(|| {
"Click 'Reset Game' to remove the previous mod installation.".to_string()
});
}
let (_, game_info, deployment_info) = tokio::try_join!(
async {
fs::metadata(&bundle_dir)
.await
.wrap_err("Failed to open game bundle directory")
.with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.")
},
async {
tokio::task::spawn_blocking(dtmt_shared::collect_game_info)
.await
.map_err(Report::new)
},
async {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
match read_sjson_file::<_, DeploymentData>(path).await {
Ok(data) => Ok(Some(data)),
Err(err) => {
if let Some(err) = err.downcast_ref::<std::io::Error>()
&& err.kind() == ErrorKind::NotFound
{
Ok(None)
} else {
Err(err).wrap_err("Failed to read deployment data")
}
}
}
}
)
.wrap_err("Failed to gather deployment information")?;
tracing::debug!(?game_info, ?deployment_info);
if let Some(game_info) = game_info {
if deployment_info
.as_ref()
.map(|i| game_info.last_updated > i.timestamp)
.unwrap_or(false)
{
tracing::warn!(
"Game was updated since last mod deployment. \
Attempting to reconcile game files."
);
tokio::try_join!(
async {
let path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let backup_path = path.with_extension("data.bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for bundle database.")
},
async {
let path = bundle_dir.join(boot_bundle_path);
let backup_path = path.with_extension("bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for boot bundle")
}
)
.with_suggestion(|| {
"Reset the game using 'Reset Game', then verify game files.".to_string()
})?;
tracing::info!(
"Successfully re-created game file backups. \
Continuing mod deployment."
);
}
}
check_mod_order(&state)?;
tracing::info!(
"Deploying {} mods to '{}'.",
state.mods.iter().filter(|i| i.enabled).count(),
bundle_dir.display()
);
tracing::info!("Copy legacy mod folders");
let mod_folders = copy_mod_folders(state.clone())
.await
.wrap_err("Failed to copy mod folders")?;
tracing::info!("Build mod bundles");
let mut bundles = build_bundles(state.clone())
.await
.wrap_err("Failed to build mod bundles")?;
tracing::info!("Patch boot bundle");
let mut boot_bundles = patch_boot_bundle(state.clone())
.await
.wrap_err("Failed to patch boot bundle")?;
bundles.append(&mut boot_bundles);
if let Some(info) = &deployment_info {
let bundle_dir = Arc::new(bundle_dir);
// Remove bundles from the previous deployment that don't match the current one.
// I.e. mods that used to be installed/enabled but aren't anymore.
{
let tasks = info.bundles.iter().cloned().filter_map(|file_name| {
let is_being_deployed = bundles.iter().any(|b2| {
let name = format!("{:016x}", b2.name());
file_name == name
});
if !is_being_deployed {
let bundle_dir = bundle_dir.clone();
let task = async move {
let path = bundle_dir.join(&file_name);
tracing::debug!("Removing unused bundle '{}'", file_name);
if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| {
format!("Failed to remove unused bundle '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
// Do the same thing for mod folders
{
let tasks = info.mod_folders.iter().filter_map(|mod_id| {
let is_being_deployed = mod_folders.iter().any(|id| id == mod_id);
if !is_being_deployed {
let path = bundle_dir.join("mods").join(mod_id);
tracing::debug!("Removing unused mod folder '{}'", path.display());
let task = async move {
if let Err(err) = fs::remove_dir_all(&path).await.wrap_err_with(|| {
format!("Failed to remove unused legacy mod '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
}
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("Failed to patch game settings")?;
tracing::info!("Patching bundle database");
patch_bundle_database(state.clone(), &bundles)
.await
.wrap_err("Failed to patch bundle database")?;
tracing::info!("Writing deployment data");
write_deployment_data(state.clone(), &bundles, mod_folders)
.await
.wrap_err("Failed to write deployment data")?;
tracing::info!("Finished deploying mods");
Ok(())
}

622
crates/dtmm/src/controller/game.rs
View file

@ -1,46 +1,19 @@
use std::collections::HashMap;
use std::io::{self, Cursor, ErrorKind};
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Help, Report, Result};
use futures::stream;
use futures::StreamExt;
use path_slash::PathBufExt;
use sdk::filetype::lua;
use sdk::filetype::package::Package;
use color_eyre::{eyre, Result};
use sdk::murmur::Murmur64;
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use serde::{Deserialize, Serialize};
use string_template::Template;
use time::OffsetDateTime;
use tokio::fs;
use tokio::fs::{self};
use tokio::io::AsyncWriteExt;
use tracing::Instrument;
use super::read_sjson_file;
use crate::controller::app::check_mod_order;
use crate::state::{ActionState, PackageInfo};
use crate::controller::deploy::{
DeploymentData, BOOT_BUNDLE_NAME, BUNDLE_DATABASE_NAME, DEPLOYMENT_DATA_PATH,
};
use crate::state::ActionState;
const MOD_BUNDLE_NAME: &str = "packages/mods";
const BOOT_BUNDLE_NAME: &str = "packages/boot";
const DML_BUNDLE_NAME: &str = "packages/dml";
const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini";
const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson";
#[derive(Debug, Serialize, Deserialize)]
struct DeploymentData {
bundles: Vec<String>,
#[serde(with = "time::serde::iso8601")]
timestamp: OffsetDateTime,
}
use super::deploy::SETTINGS_FILE_PATH;
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
@ -130,585 +103,6 @@ async fn patch_game_settings(state: Arc<ActionState>) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip_all, fields(package = info.name))]
fn make_package(info: &PackageInfo) -> Result<Package> {
let mut pkg = Package::new(info.name.clone(), PathBuf::new());
for f in &info.files {
let mut it = f.rsplit('.');
let file_type = it
.next()
.ok_or_else(|| eyre::eyre!("missing file extension"))
.and_then(BundleFileType::from_str)
.wrap_err("Invalid file name in package info")?;
let name: String = it.collect();
pkg.add_file(file_type, name);
}
Ok(pkg)
}
fn build_mod_data_lua(state: Arc<ActionState>) -> String {
let mut lua = String::from("return {\n");
// DMF is handled explicitely by the loading procedures, as it actually drives most of that
// and should therefore not show up in the load order.
for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) {
lua.push_str(" {\n name = \"");
lua.push_str(&mod_info.name);
lua.push_str("\",\n id = \"");
lua.push_str(&mod_info.id);
lua.push_str("\",\n run = function()\n");
let resources = &mod_info.resources;
if resources.data.is_some() || resources.localization.is_some() {
lua.push_str(" new_mod(\"");
lua.push_str(&mod_info.id);
lua.push_str("\", {\n mod_script = \"");
lua.push_str(&resources.init.to_slash_lossy());
if let Some(data) = resources.data.as_ref() {
lua.push_str("\",\n mod_data = \"");
lua.push_str(&data.to_slash_lossy());
}
if let Some(localization) = &resources.localization {
lua.push_str("\",\n mod_localization = \"");
lua.push_str(&localization.to_slash_lossy());
}
lua.push_str("\",\n })\n");
} else {
lua.push_str(" return dofile(\"");
lua.push_str(&resources.init.to_slash_lossy());
lua.push_str("\")\n");
}
lua.push_str(" end,\n packages = {\n");
for pkg_info in &mod_info.packages {
lua.push_str(" \"");
lua.push_str(&pkg_info.name);
lua.push_str("\",\n");
}
lua.push_str(" },\n },\n");
}
lua.push('}');
tracing::debug!("mod_data_lua:\n{}", lua);
lua
}
#[tracing::instrument(skip_all)]
async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string());
let mut tasks = Vec::new();
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let mut bundles = Vec::new();
{
tracing::trace!("Building mod data script");
let span = tracing::debug_span!("Building mod data script");
let _enter = span.enter();
let lua = build_mod_data_lua(state.clone());
tracing::trace!("Compiling mod data script");
let file =
lua::compile(MOD_DATA_SCRIPT, &lua).wrap_err("Failed to compile mod data Lua file")?;
tracing::trace!("Compile mod data script");
mod_bundle.add_file(file);
}
tracing::trace!("Preparing tasks to deploy bundle files");
for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) {
let span = tracing::trace_span!("building mod packages", name = mod_info.name);
let _enter = span.enter();
let mod_dir = state.mod_dir.join(&mod_info.id);
for pkg_info in &mod_info.packages {
let span = tracing::trace_span!("building package", name = pkg_info.name);
let _enter = span.enter();
tracing::trace!(
"Building package {} for mod {}",
pkg_info.name,
mod_info.name
);
let pkg = make_package(pkg_info).wrap_err("Failed to make package")?;
let mut variant = BundleFileVariant::new();
let bin = pkg
.to_binary()
.wrap_err("Failed to serialize package to binary")?;
variant.set_data(bin);
let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package);
file.add_variant(variant);
tracing::trace!(
"Compiled package {} for mod {}",
pkg_info.name,
mod_info.name
);
mod_bundle.add_file(file);
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = mod_dir.join(&bundle_name);
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
// Explicitely drop the guard, so that we can move the span
// into the async operation
drop(_enter);
let ctx = state.ctx.clone();
let task = async move {
let bundle = {
let bin = fs::read(&src).await.wrap_err_with(|| {
format!("Failed to read bundle file '{}'", src.display())
})?;
let name = Bundle::get_name_from_path(&ctx, &src);
Bundle::from_binary(&ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?
};
tracing::debug!(
src = %src.display(),
dest = %dest.display(),
"Copying bundle '{}' for mod '{}'",
pkg_name,
mod_name,
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
Ok::<Bundle, color_eyre::Report>(bundle)
}
.instrument(span);
tasks.push(task);
}
}
tracing::debug!("Copying {} mod bundles", tasks.len());
let mut tasks = stream::iter(tasks).buffer_unordered(10);
while let Some(res) = tasks.next().await {
let bundle = res?;
bundles.push(bundle);
}
{
let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64()));
tracing::trace!("Writing mod bundle to '{}'", path.display());
fs::write(&path, mod_bundle.to_binary()?)
.await
.wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?;
}
bundles.push(mod_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all)]
async fn patch_boot_bundle(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())));
let mut bundles = Vec::with_capacity(2);
let mut boot_bundle = async {
let bin = read_file_with_backup(&bundle_path)
.await
.wrap_err("Failed to read boot bundle")?;
Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin)
.wrap_err("Failed to parse boot bundle")
}
.instrument(tracing::trace_span!("read boot bundle"))
.await
.wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?;
{
tracing::trace!("Adding mod package file to boot bundle");
let span = tracing::trace_span!("create mod package file");
let _enter = span.enter();
let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new());
for mod_info in &state.mods {
for pkg_info in &mod_info.packages {
pkg.add_file(BundleFileType::Package, &pkg_info.name);
}
}
pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT);
let mut variant = BundleFileVariant::new();
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
tracing::trace!("Handling DML packages and bundle");
let span = tracing::trace_span!("handle DML");
let _enter = span.enter();
let mut variant = BundleFileVariant::new();
let mod_info = state
.mods
.iter()
.find(|m| m.id == "dml")
.ok_or_else(|| eyre::eyre!("DML not found in mod list"))?;
let pkg_info = mod_info
.packages
.get(0)
.ok_or_else(|| eyre::eyre!("invalid mod package for DML"))
.with_suggestion(|| "Re-download and import the newest version.".to_string())?;
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = state.mod_dir.join(&mod_info.id).join(&bundle_name);
{
let bin = fs::read(&src)
.await
.wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?;
let name = Bundle::get_name_from_path(&state.ctx, &src);
let dml_bundle = Bundle::from_binary(&state.ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?;
bundles.push(dml_bundle);
};
{
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
tracing::debug!(
"Copying bundle {} for mod {}: {} -> {}",
pkg_name,
mod_name,
src.display(),
dest.display()
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
}
let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?;
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
let span = tracing::debug_span!("Importing mod main script");
let _enter = span.enter();
let is_io_enabled = format!("{}", state.is_io_enabled);
let mut data = HashMap::new();
data.insert("is_io_enabled", is_io_enabled.as_str());
let tmpl = include_str!("../../assets/mod_main.lua");
let lua = Template::new(tmpl).render(&data);
tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
let file =
lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
boot_bundle.add_file(file);
}
async {
let bin = boot_bundle
.to_binary()
.wrap_err("Failed to serialize boot bundle")?;
fs::write(&bundle_path, bin)
.await
.wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display()))
}
.instrument(tracing::trace_span!("write boot bundle"))
.await?;
bundles.push(boot_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn patch_bundle_database<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let mut db = {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("Failed to read bundle database")?;
let mut r = Cursor::new(bin);
let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?;
tracing::trace!("Finished parsing bundle database");
db
};
for bundle in bundles.as_ref() {
tracing::trace!("Adding '{}' to bundle database", bundle.name().display());
db.add_bundle(bundle);
}
{
let bin = db
.to_binary()
.wrap_err("Failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})?;
}
Ok(())
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn write_deployment_data<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let info = DeploymentData {
timestamp: OffsetDateTime::now_utc(),
bundles: bundles
.as_ref()
.iter()
.map(|bundle| format!("{:x}", bundle.name().to_murmur64()))
.collect(),
};
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?;
fs::write(&path, &data)
.await
.wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.game_dir.display(),
mods = state.mods.len()
))]
pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> {
let state = Arc::new(state);
let bundle_dir = state.game_dir.join("bundle");
let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()));
if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999")))
.await
.is_ok()
{
let err = eyre::eyre!("Found dtkit-patch-based mod installation.");
return Err(err)
.with_suggestion(|| {
"If you're a mod author and saved projects directly in 'mods/', \
use DTMT to migrate them to the new project structure."
.to_string()
})
.with_suggestion(|| {
"Click 'Reset Game' to remove the previous mod installation.".to_string()
});
}
let (_, game_info, deployment_info) = tokio::try_join!(
async {
fs::metadata(&bundle_dir)
.await
.wrap_err("Failed to open game bundle directory")
.with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.")
},
async {
tokio::task::spawn_blocking(dtmt_shared::collect_game_info)
.await
.map_err(Report::new)
},
async {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
match read_sjson_file::<_, DeploymentData>(path)
.await
{
Ok(data) => Ok(Some(data)),
Err(err) => {
if let Some(err) = err.downcast_ref::<std::io::Error>() && err.kind() == ErrorKind::NotFound {
Ok(None)
} else {
Err(err).wrap_err("Failed to read deployment data")
}
}
}
}
)
.wrap_err("Failed to gather deployment information")?;
tracing::debug!(?game_info, ?deployment_info);
if let Some(game_info) = game_info {
if deployment_info
.as_ref()
.map(|i| game_info.last_updated > i.timestamp)
.unwrap_or(false)
{
tracing::warn!(
"Game was updated since last mod deployment. \
Attempting to reconcile game files."
);
tokio::try_join!(
async {
let path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let backup_path = path.with_extension("data.bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for bundle database.")
},
async {
let path = bundle_dir.join(boot_bundle_path);
let backup_path = path.with_extension("bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for boot bundle")
}
)
.with_suggestion(|| {
"Reset the game using 'Reset Game', then verify game files.".to_string()
})?;
tracing::info!(
"Successfully re-created game file backups. \
Continuing mod deployment."
);
}
}
check_mod_order(&state)?;
tracing::info!(
"Deploying {} mods to '{}'.",
state.mods.iter().filter(|i| i.enabled).count(),
bundle_dir.display()
);
tracing::info!("Build mod bundles");
let mut bundles = build_bundles(state.clone())
.await
.wrap_err("Failed to build mod bundles")?;
tracing::info!("Patch boot bundle");
let mut more_bundles = patch_boot_bundle(state.clone())
.await
.wrap_err("Failed to patch boot bundle")?;
bundles.append(&mut more_bundles);
if let Some(info) = &deployment_info {
let bundle_dir = Arc::new(bundle_dir);
let tasks = info.bundles.iter().cloned().filter_map(|file_name| {
let contains = bundles.iter().any(|b2| {
let name = format!("{:016x}", b2.name());
file_name == name
});
if !contains {
let bundle_dir = bundle_dir.clone();
let task = async move {
let path = bundle_dir.join(&file_name);
tracing::debug!("Removing unused bundle '{}'", file_name);
if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| {
format!("Failed to remove unused bundle '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("Failed to patch game settings")?;
tracing::info!("Patching bundle database");
patch_bundle_database(state.clone(), &bundles)
.await
.wrap_err("Failed to patch bundle database")?;
tracing::info!("Writing deployment data");
write_deployment_data(state.clone(), &bundles)
.await
.wrap_err("Failed to write deployment data")?;
tracing::info!("Finished deploying mods");
Ok(())
}
#[tracing::instrument(skip_all)]
async fn reset_dtkit_patch(state: ActionState) -> Result<()> {
let bundle_dir = state.game_dir.join("bundle");

495
crates/dtmm/src/controller/import.rs Normal file
View file

@ -0,0 +1,495 @@
use std::collections::HashMap;
use std::ffi::CStr;
use std::io::{Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use color_eyre::eyre::{self, Context};
use color_eyre::{Help, Report, Result};
use druid::im::Vector;
use druid::{FileInfo, ImageBuf};
use dtmt_shared::{ModConfig, ModConfigResources};
use luajit2_sys as lua;
use nexusmods::Api as NexusApi;
use tokio::fs;
use zip::ZipArchive;
use crate::state::{ActionState, ModInfo, NexusInfo, PackageInfo};
fn find_archive_file<R: Read + Seek>(
archive: &ZipArchive<R>,
name: impl AsRef<str>,
) -> Option<String> {
let path = archive
.file_names()
.find(|path| path.ends_with(name.as_ref()))
.map(|s| s.to_string());
path
}
// Runs the content of a `.mod` file to extract what data we can get
// from legacy mods.
// 1. Create a global function `new_mod` that stores
// the relevant bits in global variables.
// 2. Run the `.mod` file, which will return a table.
// 3. Run the `run` function from that table.
// 4. Access the global variables from #1.
#[tracing::instrument]
fn parse_mod_id_file(data: &str) -> Result<(String, ModConfigResources)> {
tracing::debug!("Parsing mod file:\n{}", data);
let ret = unsafe {
let state = lua::luaL_newstate();
lua::luaL_openlibs(state);
let run = b"
function fassert() end
function new_mod(id, resources)
_G.id = id
_G.script = resources.mod_script
_G.data = resources.mod_data
_G.localization = resources.mod_localization
end
\0";
match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRSYNTAX => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Invalid syntax: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to create `new_mod`")
}
_ => unreachable!(),
}
match lua::lua_pcall(state, 0, 0, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run buffer: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run buffer")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let name = b".mod\0";
match lua::luaL_loadbuffer(
state,
data.as_ptr() as _,
data.len() as _,
name.as_ptr() as _,
) as u32
{
lua::LUA_OK => {}
lua::LUA_ERRSYNTAX => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Invalid syntax: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to load `.mod` file buffer")
}
_ => unreachable!(),
}
match lua::lua_pcall(state, 0, 1, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run `.mod` file: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run `.mod` file")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let key = b"run\0";
lua::lua_pushstring(state, key.as_ptr() as _);
lua::lua_gettable(state, -2);
match lua::lua_pcall(state, 0, 0, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run `.mod.run`: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run `.mod.run`")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let get_global = |state, key: &[u8]| {
lua::lua_getglobal(state, key.as_ptr() as _);
if lua::lua_isnil(state, -1) != 0 {
return Ok(None);
}
let s = lua::lua_tostring(state, -1);
if s.is_null() {
eyre::bail!("Expected string, got NULL");
}
let ret = CStr::from_ptr(s).to_string_lossy().to_string();
lua::lua_pop(state, 1);
Ok(Some(ret))
};
let mod_id = get_global(state, b"id\0")
.and_then(|s| s.ok_or_else(|| eyre::eyre!("Got `nil`")))
.wrap_err("Failed to get `id`")?;
let resources = ModConfigResources {
init: get_global(state, b"script\0")
.and_then(|s| s.map(PathBuf::from).ok_or_else(|| eyre::eyre!("Got `nil`")))
.wrap_err("Failed to get `script`.")?,
data: get_global(state, b"data\0")
.wrap_err("Failed to get `data`.")?
.map(PathBuf::from),
localization: get_global(state, b"localization\0")
.wrap_err("Failed to get `localization`")?
.map(PathBuf::from),
};
lua::lua_close(state);
(mod_id, resources)
};
Ok(ret)
}
// Extracts the mod configuration from the mod archive.
// This may either be a proper `dtmt.cfg`, or the legacy `<mod_name>.mod` ID file.
//
// It also returns the directory where this file was found, used as root path. This
// allows flexibility in what the directory structure is exactly, since many people
// still end up creating tarbombs and Nexus does its own re-packaging.
#[tracing::instrument(skip(archive))]
fn extract_mod_config<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Result<(ModConfig, String)> {
let legacy_mod_data = if let Some(name) = find_archive_file(archive, ".mod") {
let (mod_id, resources) = {
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read `.mod` file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read `.mod` file from archive")?;
let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?;
parse_mod_id_file(&data)
.wrap_err("Invalid `.mod` file")
.note(
"The `.mod` file's `run` function may not contain any additional logic \
besides the default.",
)
.suggestion("Contact the mod author to fix this.")?
};
let root = if let Some(index) = name.rfind('/') {
name[..index].to_string()
} else {
String::new()
};
Some((mod_id, resources, root))
} else {
None
};
if let Some(name) = find_archive_file(archive, "dtmt.cfg") {
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read mod config from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read mod config from archive")?;
let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?;
let mut cfg: ModConfig = serde_sjson::from_str(&data)
.wrap_err("Failed to deserialize mod config")
.suggestion("Contact the mod author to fix this.")?;
if let Some((mod_id, resources, root)) = legacy_mod_data {
if cfg.id != mod_id {
let err = eyre::eyre!("Mod ID in `dtmt.cfg` does not match mod ID in `.mod` file");
return Err(err).suggestion("Contact the mod author to fix this.");
}
cfg.resources = resources;
Ok((cfg, root))
} else {
let root = name
.strip_suffix("dtmt.cfg")
.expect("String must end with that suffix")
.to_string();
Ok((cfg, root))
}
} else {
eyre::bail!(
"Mod needs a config file or `.mod` file. \
Please get in touch with the author to provide a properly packaged mod."
);
}
}
#[tracing::instrument(skip(archive))]
fn extract_bundled_mod<R: Read + Seek>(
archive: &mut ZipArchive<R>,
root: String,
dest: impl AsRef<Path> + std::fmt::Debug,
) -> Result<Vector<Arc<PackageInfo>>> {
let files: HashMap<String, Vec<String>> = {
let name = archive
.file_names()
.find(|name| name.ends_with("files.sjson"))
.map(|s| s.to_string())
.ok_or_else(|| eyre::eyre!("archive does not contain file index"))?;
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read file index from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")?
};
tracing::trace!(?files);
let dest = dest.as_ref();
tracing::trace!("Extracting mod archive to {}", dest.display());
archive
.extract(dest)
.wrap_err_with(|| format!("Failed to extract archive to {}", dest.display()))?;
let packages = files
.into_iter()
.map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect())))
.collect();
tracing::trace!(?packages);
Ok(packages)
}
#[tracing::instrument(skip(archive))]
fn extract_legacy_mod<R: Read + Seek>(
archive: &mut ZipArchive<R>,
root: String,
dest: impl Into<PathBuf> + std::fmt::Debug,
) -> Result<()> {
let dest = dest.into();
let file_count = archive.len();
for i in 0..file_count {
let mut f = archive
.by_index(i)
.wrap_err_with(|| format!("Failed to get file at index {}", i))?;
let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else {
let err = eyre::eyre!("File name in archive is not a safe path value.").suggestion(
"Only use well-known applications to create the ZIP archive, \
and don't create paths that point outside the archive directory.",
);
return Err(err);
};
let Ok(suffix) = name.strip_prefix(&root) else {
tracing::warn!(
"Skipping file outside of the mod root directory: {}",
name.display()
);
continue;
};
let name = dest.join(suffix);
if f.is_dir() {
// The majority of errors will actually be "X already exists".
// But rather than filter them invidually, we just ignore all of them.
// If there is a legitimate error of "couldn't create X", it will eventually fail when
// we try to put a file in there.
tracing::trace!("Creating directory '{}'", name.display());
let _ = std::fs::create_dir_all(&name);
} else {
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err_with(|| format!("Failed to read file '{}'", name.display()))?;
tracing::trace!("Writing file '{}'", name.display());
let mut out = std::fs::OpenOptions::new()
.write(true)
.create(true)
.open(&name)
.wrap_err_with(|| format!("Failed to open file '{}'", name.display()))?;
out.write_all(&buf)
.wrap_err_with(|| format!("Failed to write to '{}'", name.display()))?;
}
}
Ok(())
}
#[tracing::instrument(skip(state))]
pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<ModInfo> {
let data = fs::read(&info.path)
.await
.wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?;
let data = Cursor::new(data);
let nexus = if let Some((_, id, version, _)) = info
.path
.file_name()
.and_then(|s| s.to_str())
.and_then(NexusApi::parse_file_name)
{
if !state.nexus_api_key.is_empty() {
let api = NexusApi::new(state.nexus_api_key.to_string())?;
let mod_info = api
.mods_id(id)
.await
.wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?;
let info = NexusInfo::from(mod_info);
tracing::debug!("{:?}", info);
Some((info, version))
} else {
None
}
} else {
None
};
let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
if tracing::enabled!(tracing::Level::DEBUG) {
let names = archive.file_names().fold(String::new(), |mut s, name| {
s.push('\n');
s.push_str(name);
s
});
tracing::debug!("Archive contents:{}", names);
}
let (mut mod_cfg, root) =
extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?;
tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id);
tracing::debug!(root, ?mod_cfg);
let image = if let Some(path) = &mod_cfg.image {
let name = archive
.file_names()
.find(|name| name.ends_with(&path.display().to_string()))
.map(|s| s.to_string())
.ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?;
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read image file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
let img = match ImageBuf::from_data(&buf) {
Ok(img) => img,
Err(err) => {
let err = Report::msg(err.to_string())
.wrap_err("Invalid image data")
.note("Supported formats are: PNG, JPEG, Bitmap and WebP")
.suggestion("Contact the mod author to fix this");
return Err(err);
}
};
Some(img)
} else {
None
};
tracing::trace!(?image);
let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
let dest = mod_dir.join(&mod_cfg.id);
tracing::trace!("Creating mods directory {}", dest.display());
fs::create_dir_all(&dest)
.await
.wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
let packages = if mod_cfg.bundled {
extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")?
} else {
extract_legacy_mod(&mut archive, root, &dest).wrap_err("Failed to extract legacy mod")?;
if let Some((_, version)) = &nexus {
// We use the version number stored in the `ModInfo` to compare against the `NexusInfo`
// for version checks. So for this one, we can't actually rely on merely shadowing,
// like with the other fields.
mod_cfg.version = version.clone();
}
let data = serde_sjson::to_string(&mod_cfg).wrap_err("Failed to serialize mod config")?;
fs::write(dest.join("dtmt.cfg"), &data)
.await
.wrap_err("Failed to write mod config")?;
Default::default()
};
if let Some((nexus, _)) = &nexus {
let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?;
let path = dest.join("nexus.sjson");
fs::write(&path, data.as_bytes())
.await
.wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?;
}
let info = ModInfo::new(mod_cfg, packages, image, nexus.map(|(info, _)| info));
Ok(info)
}

2
crates/dtmm/src/controller/mod.rs
View file

@ -5,7 +5,9 @@ use serde::Deserialize;
use tokio::fs;
pub mod app;
pub mod deploy;
pub mod game;
pub mod import;
pub mod worker;
#[tracing::instrument]

4
crates/dtmm/src/controller/worker.rs
View file

@ -13,7 +13,9 @@ use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::RwLock;
use crate::controller::app::*;
use crate::controller::deploy::deploy_mods;
use crate::controller::game::*;
use crate::controller::import::import_mod;
use crate::state::AsyncAction;
use crate::state::ACTION_FINISH_CHECK_UPDATE;
use crate::state::ACTION_FINISH_LOAD_INITIAL;
@ -36,7 +38,9 @@ async fn handle_action(
action_queue: Arc<RwLock<UnboundedReceiver<AsyncAction>>>,
) {
while let Some(action) = action_queue.write().await.recv().await {
if cfg!(debug_assertions) && !matches!(action, AsyncAction::Log(_)) {
tracing::debug!(?action);
}
let event_sink = event_sink.clone();
match action {

1
crates/dtmm/src/main.rs
View file

@ -1,6 +1,7 @@
#![recursion_limit = "256"]
#![feature(let_chains)]
#![feature(arc_unwrap_or_clone)]
#![feature(iterator_try_collect)]
#![windows_subsystem = "windows"]
use std::path::PathBuf;

4
crates/dtmm/src/state/data.rs
View file

@ -73,6 +73,7 @@ impl From<dtmt_shared::ModDependency> for ModDependency {
#[derive(Clone, Data, Debug, Lens, serde::Serialize, serde::Deserialize)]
pub(crate) struct NexusInfo {
pub id: u64,
pub name: String,
pub version: String,
pub author: String,
pub summary: Arc<String>,
@ -83,6 +84,7 @@ impl From<NexusMod> for NexusInfo {
fn from(value: NexusMod) -> Self {
Self {
id: value.mod_id,
name: value.name,
version: value.version,
author: value.author,
summary: Arc::new(value.summary),
@ -109,6 +111,7 @@ pub(crate) struct ModInfo {
#[data(ignore)]
pub resources: ModResourceInfo,
pub depends: Vector<ModDependency>,
pub bundled: bool,
#[data(ignore)]
pub nexus: Option<NexusInfo>,
}
@ -129,6 +132,7 @@ impl ModInfo {
version: cfg.version,
enabled: false,
packages,
bundled: cfg.bundled,
image,
categories: cfg.categories.into_iter().collect(),
resources: ModResourceInfo {

1
crates/dtmm/src/state/delegate.rs
View file

@ -411,6 +411,7 @@ impl AppDelegate<State> for Delegate {
state.config_path = Arc::new(config.path);
state.data_dir = Arc::new(config.data_dir);
state.game_dir = Arc::new(config.game_dir.unwrap_or_default());
state.nexus_api_key = Arc::new(config.nexus_api_key.unwrap_or_default());
state.is_io_enabled = config.unsafe_io;
}

11
crates/dtmm/src/ui/window/main.rs
View file

@ -135,8 +135,13 @@ fn build_mod_list() -> impl Widget<State> {
})
.lens(lens!((usize, Arc<ModInfo>, bool), 1).then(ModInfo::enabled.in_arc()));
let name =
Label::raw().lens(lens!((usize, Arc<ModInfo>, bool), 1).then(ModInfo::name.in_arc()));
let name = Label::dynamic(|info: &Arc<ModInfo>, _| {
info.nexus
.as_ref()
.map(|n| n.name.clone())
.unwrap_or_else(|| info.name.clone())
})
.lens(lens!((usize, Arc<ModInfo>, bool), 1));
let version = {
let icon = {
@ -145,7 +150,7 @@ fn build_mod_list() -> impl Widget<State> {
let tree = theme::icons::recolor_icon(tree, true, COLOR_YELLOW_LIGHT);
Svg::new(Arc::new(tree)).fix_height(druid::theme::TEXT_SIZE_NORMAL)
Svg::new(tree).fix_height(druid::theme::TEXT_SIZE_NORMAL)
};
Either::new(

8
crates/dtmm/src/util/config.rs
View file

@ -125,6 +125,9 @@ where
.wrap_err_with(|| format!("Invalid config file {}", path.display()))?;
cfg.path = path;
tracing::debug!("Read config file '{}': {:?}", cfg.path.display(), cfg);
Ok(cfg)
}
Err(err) if err.kind() == ErrorKind::NotFound => {
@ -133,6 +136,11 @@ where
.wrap_err_with(|| format!("Failed to read config file {}", path.display()))?;
}
tracing::debug!(
"Config file not found at '{}', creating default.",
path.display()
);
{
let parent = default_path
.parent()

5
crates/dtmt/src/cmd/dictionary.rs
View file

@ -145,7 +145,10 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<(
.get_one::<HashGroup>("group")
.expect("required argument not found");
let r: BufReader<Box<dyn tokio::io::AsyncRead + std::marker::Unpin>> = if let Some(name) = path.file_name() && name == "-" {
let r: BufReader<Box<dyn tokio::io::AsyncRead + std::marker::Unpin>> = if let Some(name) =
path.file_name()
&& name == "-"
{
let f = tokio::io::stdin();
BufReader::new(Box::new(f))
} else {

1
crates/dtmt/src/cmd/migrate.rs
View file

@ -350,6 +350,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
localization: mod_file.localization,
},
depends: vec![ModDependency::ID(String::from("DMF"))],
bundled: true,
};
tracing::debug!(?dtmt_cfg);

13
lib/dtmt-shared/src/lib.rs
View file

@ -30,6 +30,17 @@ pub enum ModDependency {
Config { id: String, order: ModOrder },
}
// A bit dumb, but serde doesn't support literal values with the
// `default` attribute, only paths.
fn default_true() -> bool {
true
}
// Similarly dumb, as the `skip_serializing_if` attribute needs a function
fn is_true(val: &bool) -> bool {
*val
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct ModConfig {
#[serde(skip)]
@ -51,6 +62,8 @@ pub struct ModConfig {
pub resources: ModConfigResources,
#[serde(default)]
pub depends: Vec<ModDependency>,
#[serde(default = "default_true", skip_serializing_if = "is_true")]
pub bundled: bool,
}
pub const STEAMAPP_ID: u32 = 1361210;

4
lib/nexusmods/src/lib.rs
View file

@ -112,7 +112,7 @@ impl Api {
RE.captures(name.as_ref()).and_then(|cap| {
let name = cap.name("name").map(|s| s.as_str().to_string())?;
let mod_id = cap.name("mod_id").and_then(|s| s.as_str().parse().ok())?;
let version = cap.name("version").map(|s| s.as_str().to_string())?;
let version = cap.name("version").map(|s| s.as_str().replace('-', "."))?;
let updated = cap
.name("updated")
.and_then(|s| s.as_str().parse().ok())
@ -154,7 +154,7 @@ impl Api {
self.mods_download_link(nxm.mod_id, nxm.file_id, nxm.key, nxm.expires)
)?;
let Some(download_url) = download_info.get(0).map(|i| i.uri.clone()) else {
let Some(download_url) = download_info.first().map(|i| i.uri.clone()) else {
return Err(Error::InvalidNXM("no download link", url));
};

7
lib/sdk/src/bundle/mod.rs
View file

@ -227,12 +227,9 @@ impl Bundle {
let _enter = span.enter();
tracing::trace!(num_files = self.files.len());
self.files
.iter()
.fold(Ok::<Vec<u8>, Report>(Vec::new()), |data, file| {
let mut data = data?;
self.files.iter().try_fold(Vec::new(), |mut data, file| {
data.append(&mut file.to_binary()?);
Ok(data)
Ok::<_, Report>(data)
})?
};

4
lib/sdk/src/filetype/lua.rs
View file

@ -53,8 +53,8 @@ where
let mut buf = vec![0u8; length];
r.read_exact(&mut buf)?;
let mut s = String::from_utf8(buf)
.wrap_err_with(|| format!("Invalid byte sequence for LuaJIT bytecode name"))?;
let mut s =
String::from_utf8(buf).wrap_err("Invalid byte sequence for LuaJIT bytecode name")?;
// Remove the leading `@`
s.remove(0);
s