bitsquid_dt/dtmt
bitsquid_dt/dtmt
2
Fork 0
Code Issues 34 Pull requests 2 Projects 1 Releases 3 Packages 1 Wiki Activity
Var Dump: 
dumpVar: only available in dev mode
Mailing List

Compare commits

base: bitsquid_dt:12e01075d910563d57524c6d1b1026a6207ab635
Branches Tags
bitsquid_dt:master
bitsquid_dt:feat/misc
bitsquid_dt:feat/textures
bitsquid_dt:experiment/word-generation
bitsquid_dt:feat/updates
bitsquid_dt:wip/loose-files
bitsquid_dt:feat/code-sign
bitsquid_dt:feat/widget-extra
bitsquid_dt:dtmm-v1.0.0-rc3
bitsquid_dt:dtmm-v1.0.0-rc2
bitsquid_dt:dtmm-v1.0.0-rc1
bitsquid_dt:dtmm-v1.0.0-rc0
bitsquid_dt:dtmt-v0.3.0
bitsquid_dt:dtmm-v0.1.0
bitsquid_dt:v0.2.0
bitsquid_dt:v0.1.1
..
compare: bitsquid_dt:f63dbd95a7ad36e7bb865d575b31b46f77b2caaf
Branches Tags
bitsquid_dt:feat/misc
bitsquid_dt:feat/textures
bitsquid_dt:master
bitsquid_dt:experiment/word-generation
bitsquid_dt:feat/updates
bitsquid_dt:wip/loose-files
bitsquid_dt:feat/code-sign
bitsquid_dt:feat/widget-extra
bitsquid_dt:dtmm-v1.0.0-rc3
bitsquid_dt:dtmm-v1.0.0-rc2
bitsquid_dt:dtmm-v1.0.0-rc1
bitsquid_dt:dtmm-v1.0.0-rc0
bitsquid_dt:dtmt-v0.3.0
bitsquid_dt:dtmm-v0.1.0
bitsquid_dt:v0.2.0
bitsquid_dt:v0.1.1

No commits in common. "12e01075d910563d57524c6d1b1026a6207ab635" and "f63dbd95a7ad36e7bb865d575b31b46f77b2caaf" have entirely different histories.
12e01075d9 ... f63dbd95a7

22 changed files with 929 additions and 1589 deletions
Download patch file Download diff file Expand all files Collapse all files

60
Cargo.lock generated
View file

@ -39,10 +39,12 @@ dependencies = [
[[package]]
name = "ansi-parser"
version = "0.9.0"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcb2392079bf27198570d6af79ecbd9ec7d8f16d3ec6b60933922fdb66287127"
dependencies = [
"heapless",
"nom",
"nom 4.2.3",
]
[[package]]
@ -379,7 +381,7 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
"nom 7.1.3",
]
[[package]]
@ -894,7 +896,6 @@ name = "dtmm"
version = "0.1.0"
dependencies = [
"ansi-parser",
"async-recursion",
"bitflags 1.3.2",
"clap",
"color-eyre",
@ -905,8 +906,6 @@ dependencies = [
"dtmt-shared",
"futures",
"lazy_static",
"luajit2-sys",
"minijinja",
"nexusmods",
"oodle",
"path-slash",
@ -1391,7 +1390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
"version_check 0.9.4",
]
[[package]]
@ -1615,12 +1614,12 @@ checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156"
[[package]]
name = "heapless"
version = "0.6.1"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422"
checksum = "74911a68a1658cfcfb61bc0ccfbd536e3b6e906f8c2f7883ee50157e3e2184f1"
dependencies = [
"as-slice",
"generic-array 0.14.7",
"generic-array 0.13.3",
"hash32",
"stable_deref_trait",
]
@ -1748,7 +1747,7 @@ dependencies = [
"serde",
"sized-chunks",
"typenum",
"version_check",
"version_check 0.9.4",
]
[[package]]
@ -2085,15 +2084,6 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minijinja"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "208758577ef2c86cf5dd3e85730d161413ec3284e2d73b2ef65d9a24d9971bcb"
dependencies = [
"serde",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -2185,6 +2175,16 @@ dependencies = [
"memoffset 0.6.5",
]
[[package]]
name = "nom"
version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
dependencies = [
"memchr",
"version_check 0.1.5",
]
[[package]]
name = "nom"
version = "7.1.3"
@ -2203,7 +2203,7 @@ checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3"
dependencies = [
"bytecount",
"memchr",
"nom",
"nom 7.1.3",
]
[[package]]
@ -2673,7 +2673,7 @@ dependencies = [
"proc-macro2",
"quote",
"syn 1.0.109",
"version_check",
"version_check 0.9.4",
]
[[package]]
@ -2684,7 +2684,7 @@ checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
"version_check 0.9.4",
]
[[package]]
@ -3102,7 +3102,7 @@ dependencies = [
name = "serde_sjson"
version = "1.0.0"
dependencies = [
"nom",
"nom 7.1.3",
"nom_locate",
"serde",
]
@ -3832,7 +3832,7 @@ version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
dependencies = [
"version_check",
"version_check 0.9.4",
]
[[package]]
@ -3966,6 +3966,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "579a42fc0b8e0c63b76519a339be31bed574929511fa53c1a3acae26eb258f29"
[[package]]
name = "version_check"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
[[package]]
name = "version_check"
version = "0.9.4"
@ -4378,3 +4384,7 @@ dependencies = [
"cc",
"pkg-config",
]
[[patch.unused]]
name = "ansi-parser"
version = "0.9.0"

5
Cargo.toml
View file

@ -7,7 +7,10 @@ members = [
"lib/oodle",
"lib/sdk",
"lib/serde_sjson",
"lib/luajit2-sys",
]
exclude = [
"lib/color-eyre",
"lib/ansi-parser",
]
[patch.crates-io]

5
crates/dtmm/Cargo.toml
View file

@ -31,8 +31,5 @@ lazy_static = "1.4.0"
colors-transform = "0.2.11"
usvg = "0.25.0"
druid-widget-nursery = "0.1"
ansi-parser = "0.9.0"
ansi-parser = "0.8.0"
string_template = "0.2.1"
luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" }
async-recursion = "1.0.5"
minijinja = "1.0.10"

27
crates/dtmm/assets/mod_data.lua.j2
View file

@ -1,27 +0,0 @@
return {
{% for mod in mods %}
{
id = "{{ mod.id }}",
name = "{{ mod.name }}",
bundled = {{ mod.bundled }},
packages = {
{% for pkg in mod.packages %}
"{{ pkg }}",
{% endfor %}
},
run = function()
{% if mod.data is none %}
return dofile("{{ mod.init }}")
{% else %}
new_mod("{{ mod.id }}", {
mod_script = "{{ mod.init }}",
mod_data = "{{ mod.data }}",
{% if not mod.localization is none %}
mod_localization = "{{ mod.localization }}",
{% endif %}
})
{% endif %}
end,
},
{% endfor %}
}

189
crates/dtmm/assets/mod_main.lua
View file

@ -11,6 +11,100 @@ local log = function(category, format, ...)
end
end
-- Patch `GameStateMachine.init` to add our own state for loading mods.
-- In the future, Fatshark might provide us with a dedicated way to do this.
local function patch_mod_loading_state()
local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base")
-- A necessary override.
-- The original does not proxy `dt` to `_state_update`, but we need that.
StateBootSubStateBase.update = function(self, dt)
local done, error = self:_state_update(dt)
local params = self._params
if error then
return StateError, { error }
elseif done then
local next_index = params.sub_state_index + 1
params.sub_state_index = next_index
local next_state_data = params.states[next_index]
if next_state_data then
return next_state_data[1], self._params
else
self._parent:sub_states_done()
end
end
end
local StateBootLoadMods = class("StateBootLoadMods", "StateBootSubStateBase")
StateBootLoadMods.on_enter = function(self, parent, params)
log("StateBootLoadMods", "Entered")
StateBootLoadMods.super.on_enter(self, parent, params)
local state_params = self:_state_params()
local package_manager = state_params.package_manager
self._state = "load_package"
self._package_manager = package_manager
self._package_handles = {
["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadMods", nil),
["packages/dml"] = package_manager:load("packages/dml", "StateBootLoadMods", nil),
}
end
StateBootLoadMods._state_update = function(self, dt)
local state = self._state
local package_manager = self._package_manager
if state == "load_package" and package_manager:update() then
log("StateBootLoadMods", "Packages loaded, loading mods")
self._state = "load_mods"
local ModLoader = require("scripts/mods/dml/init")
local mod_data = require("scripts/mods/mod_data")
local mod_loader = ModLoader:new(mod_data, self._parent:gui())
self._mod_loader = mod_loader
Managers.mod = mod_loader
elseif state == "load_mods" and self._mod_loader:update(dt) then
log("StateBootLoadMods", "Mods loaded, exiting")
return true, false
end
return false, false
end
local GameStateMachine = require("scripts/foundation/utilities/game_state_machine")
local patched = false
local GameStateMachine_init = GameStateMachine.init
GameStateMachine.init = function(self, parent, start_state, params, ...)
if not patched then
log("mod_main", "Injecting mod loading state")
patched = true
-- Hardcoded position after `StateRequireScripts`.
-- We do want to wait until then, so that most of the game's core
-- systems are at least loaded and can be hooked, even if they aren't
-- running, yet.
local pos = 4
table.insert(params.states, pos, {
StateBootLoadMods,
{
package_manager = params.package_manager,
},
})
end
GameStateMachine_init(self, parent, start_state, params, ...)
end
log("mod_main", "Mod patching complete")
end
log("mod_main", "Initializing mods...")
local require_store = {}
@ -44,8 +138,7 @@ Mods = {
-- Fatshark's code scrubs them.
-- The loader can then decide to pass them on to mods, or ignore them
lua = setmetatable({}, { __index = lua_libs }),
require_store = require_store,
original_require = require,
require_store = require_store
}
local can_insert = function(filepath, new_result)
@ -105,98 +198,6 @@ end
require("scripts/main")
log("mod_main", "'scripts/main' loaded")
-- Inject our state into the game. The state needs to run after `StateGame._init_managers`,
-- since some parts of DMF, and presumably other mods, depend on some of those managers to exist.
local function patch_mod_loading_state()
local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base")
local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase")
local StateGameLoadMods = class("StateGameLoadMods")
StateBootLoadDML.on_enter = function(self, parent, params)
log("StateBootLoadDML", "Entered")
StateBootLoadDML.super.on_enter(self, parent, params)
local state_params = self:_state_params()
local package_manager = state_params.package_manager
self._package_manager = package_manager
self._package_handles = {
["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil),
["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil),
}
end
StateBootLoadDML._state_update = function(self, dt)
local package_manager = self._package_manager
if package_manager:update() then
local DML = require("scripts/mods/dml/init")
local mod_data = require("scripts/mods/mod_data")
local mod_loader = DML.create_loader(mod_data)
Managers.mod = mod_loader
log("StateBootLoadDML", "DML loaded, exiting")
return true, false
end
return false, false
end
function StateGameLoadMods:on_enter(_, params)
log("StateGameLoadMods", "Entered")
self._next_state = require("scripts/game_states/game/state_splash")
self._next_state_params = params
end
function StateGameLoadMods:update(main_dt)
local state = self._loading_state
-- We're relying on the fact that DML internally makes sure
-- that `Managers.mod:update()` is being called appropriately.
-- The implementation as of this writing is to hook `StateGame.update`.
if Managers.mod:all_mods_loaded() then
Log.info("StateGameLoadMods", "Mods loaded, exiting")
return self._next_state, self._next_state_params
end
end
local GameStateMachine = require("scripts/foundation/utilities/game_state_machine")
local GameStateMachine_init = GameStateMachine.init
GameStateMachine.init = function(self, parent, start_state, params, creation_context, state_change_callbacks, name)
if name == "Main" then
log("mod_main", "Injecting StateBootLoadDML")
-- Hardcoded position after `StateRequireScripts`.
-- We need to wait until then to even begin most of our stuff,
-- so that most of the game's core systems are at least loaded and can be hooked,
-- even if they aren't running, yet.
local pos = 4
table.insert(params.states, pos, {
StateBootLoadDML,
{
package_manager = params.package_manager,
},
})
GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name)
elseif name == "Game" then
log("mod_main", "Injection StateGameLoadMods")
-- The second time around, we want to be the first, so we pass our own
-- 'start_state'.
-- We can't just have the state machine be initialized and then change its `_next_state`, as by the end of
-- `init`, a bunch of stuff will already be initialized.
GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name)
-- And since we're done now, we can revert the function to its original
GameStateMachine.init = GameStateMachine_init
return
else
-- In all other cases, simply call the original
GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name)
end
end
end
-- Override `init` to run our injection
function init()
patch_mod_loading_state()

168
crates/dtmm/src/controller/app.rs
View file

@ -1,17 +1,18 @@
use std::collections::HashMap;
use std::io::ErrorKind;
use std::io::{Cursor, ErrorKind, Read};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use color_eyre::eyre::{self, Context};
use color_eyre::{Help, Report, Result};
use druid::im::Vector;
use druid::ImageBuf;
use druid::{FileInfo, ImageBuf};
use dtmt_shared::ModConfig;
use nexusmods::Api as NexusApi;
use tokio::fs::{self, DirEntry, File};
use tokio_stream::wrappers::ReadDirStream;
use tokio_stream::StreamExt;
use zip::ZipArchive;
use crate::state::{ActionState, InitialLoadResult, ModInfo, ModOrder, NexusInfo, PackageInfo};
use crate::util;
@ -19,6 +20,161 @@ use crate::util::config::{ConfigSerialize, LoadOrderEntry};
use super::read_sjson_file;
#[tracing::instrument(skip(state))]
pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<ModInfo> {
let data = fs::read(&info.path)
.await
.wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?;
let data = Cursor::new(data);
let nexus = if let Some((_, id, _, _)) = info
.path
.file_name()
.and_then(|s| s.to_str())
.and_then(NexusApi::parse_file_name)
{
if !state.nexus_api_key.is_empty() {
let api = NexusApi::new(state.nexus_api_key.to_string())?;
let mod_info = api
.mods_id(id)
.await
.wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?;
Some(NexusInfo::from(mod_info))
} else {
None
}
} else {
None
};
let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
if tracing::enabled!(tracing::Level::DEBUG) {
let names = archive.file_names().fold(String::new(), |mut s, name| {
s.push('\n');
s.push_str(name);
s
});
tracing::debug!("Archive contents:{}", names);
}
let dir_name = {
let f = archive.by_index(0).wrap_err("Archive is empty")?;
if !f.is_dir() {
let err = eyre::eyre!("archive does not have a top-level directory");
return Err(err).with_suggestion(|| "Use 'dtmt build' to create the mod archive.");
}
let name = f.name();
// The directory name is returned with a trailing slash, which we don't want
name[..(name.len().saturating_sub(1))].to_string()
};
tracing::info!("Importing mod {}", dir_name);
let names: Vec<_> = archive.file_names().map(|s| s.to_string()).collect();
let mod_cfg: ModConfig = {
let name = names
.iter()
.find(|name| name.ends_with("dtmt.cfg"))
.ok_or_else(|| eyre::eyre!("archive does not contain mod config"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read mod config from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read mod config from archive")?;
let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize mod config")?
};
tracing::debug!(?mod_cfg);
let files: HashMap<String, Vec<String>> = {
let name = names
.iter()
.find(|name| name.ends_with("files.sjson"))
.ok_or_else(|| eyre::eyre!("archive does not contain file index"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read file index from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")?
};
tracing::trace!(?files);
let image = if let Some(path) = &mod_cfg.image {
let name = names
.iter()
.find(|name| name.ends_with(&path.display().to_string()))
.ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?;
let mut f = archive
.by_name(name)
.wrap_err("Failed to read image file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
let img = match ImageBuf::from_data(&buf) {
Ok(img) => img,
Err(err) => {
let err = Report::msg(err.to_string()).wrap_err("Invalid image data");
return Err(err).with_suggestion(|| {
"Supported formats are: PNG, JPEG, Bitmap and WebP".to_string()
});
}
};
Some(img)
} else {
None
};
let mod_dir = state.mod_dir;
tracing::trace!("Creating mods directory {}", mod_dir.display());
fs::create_dir_all(Arc::as_ref(&mod_dir))
.await
.wrap_err_with(|| format!("Failed to create data directory {}", mod_dir.display()))?;
tracing::trace!("Extracting mod archive to {}", mod_dir.display());
archive
.extract(Arc::as_ref(&mod_dir))
.wrap_err_with(|| format!("Failed to extract archive to {}", mod_dir.display()))?;
if let Some(nexus) = &nexus {
let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?;
let path = mod_dir.join(&mod_cfg.id).join("nexus.sjson");
fs::write(&path, data.as_bytes())
.await
.wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?;
}
let packages = files
.into_iter()
.map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect())))
.collect();
let info = ModInfo::new(mod_cfg, packages, image, nexus);
Ok(info)
}
#[tracing::instrument(skip(state))]
pub(crate) async fn delete_mod(state: ActionState, info: &ModInfo) -> Result<()> {
let mod_dir = state.mod_dir.join(&info.id);
@ -73,13 +229,9 @@ async fn read_mod_dir_entry(res: Result<DirEntry>) -> Result<ModInfo> {
Err(err) => return Err(err),
};
let files: HashMap<String, Vec<String>> = if cfg.bundled {
read_sjson_file(&index_path)
let files: HashMap<String, Vec<String>> = read_sjson_file(&index_path)
.await
.wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?
} else {
Default::default()
};
.wrap_err_with(|| format!("Failed to read file index '{}'", index_path.display()))?;
let image = if let Some(path) = &cfg.image {
let path = entry.path().join(path);

868
crates/dtmm/src/controller/deploy.rs
View file

@ -1,868 +0,0 @@
use std::collections::HashMap;
use std::io::{Cursor, ErrorKind};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Help, Report, Result};
use futures::StreamExt;
use futures::{stream, TryStreamExt};
use minijinja::Environment;
use sdk::filetype::lua;
use sdk::filetype::package::Package;
use sdk::murmur::Murmur64;
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use serde::{Deserialize, Serialize};
use string_template::Template;
use time::OffsetDateTime;
use tokio::fs::{self, DirEntry};
use tokio::io::AsyncWriteExt;
use tracing::Instrument;
use super::read_sjson_file;
use crate::controller::app::check_mod_order;
use crate::state::{ActionState, PackageInfo};
pub const MOD_BUNDLE_NAME: &str = "packages/mods";
pub const BOOT_BUNDLE_NAME: &str = "packages/boot";
pub const DML_BUNDLE_NAME: &str = "packages/dml";
pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
pub const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini";
pub const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson";
#[derive(Debug, Serialize, Deserialize)]
pub struct DeploymentData {
pub bundles: Vec<String>,
pub mod_folders: Vec<String>,
#[serde(with = "time::serde::iso8601")]
pub timestamp: OffsetDateTime,
}
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
where
P: AsRef<Path> + std::fmt::Debug,
{
let path = path.as_ref();
let backup_path = {
let mut p = PathBuf::from(path);
let ext = if let Some(ext) = p.extension() {
ext.to_string_lossy().to_string() + ".bak"
} else {
String::from("bak")
};
p.set_extension(ext);
p
};
let file_name = path
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| String::from("file"));
let bin = match fs::read(&backup_path).await {
Ok(bin) => bin,
Err(err) if err.kind() == ErrorKind::NotFound => {
// TODO: This doesn't need to be awaited here, yet.
// I only need to make sure it has finished before writing the changed bundle.
tracing::debug!(
"Backup does not exist. Backing up original {} to '{}'",
file_name,
backup_path.display()
);
fs::copy(path, &backup_path).await.wrap_err_with(|| {
format!(
"Failed to back up {} '{}' to '{}'",
file_name,
path.display(),
backup_path.display()
)
})?;
tracing::debug!("Reading {} from original '{}'", file_name, path.display());
fs::read(path).await.wrap_err_with(|| {
format!("Failed to read {} file: {}", file_name, path.display())
})?
}
Err(err) => {
return Err(err).wrap_err_with(|| {
format!(
"Failed to read {} from backup '{}'",
file_name,
backup_path.display()
)
});
}
};
Ok(bin)
}
#[tracing::instrument(skip_all)]
async fn patch_game_settings(state: Arc<ActionState>) -> Result<()> {
let settings_path = state.game_dir.join("bundle").join(SETTINGS_FILE_PATH);
let settings = read_file_with_backup(&settings_path)
.await
.wrap_err("Failed to read settings.ini")?;
let settings = String::from_utf8(settings).wrap_err("Settings.ini is not valid UTF-8")?;
let mut f = fs::File::create(&settings_path)
.await
.wrap_err_with(|| format!("Failed to open {}", settings_path.display()))?;
let Some(i) = settings.find("boot_script =") else {
eyre::bail!("couldn't find 'boot_script' field");
};
f.write_all(settings[0..i].as_bytes()).await?;
f.write_all(b"boot_script = \"scripts/mod_main\"").await?;
let Some(j) = settings[i..].find('\n') else {
eyre::bail!("couldn't find end of 'boot_script' field");
};
f.write_all(settings[(i + j)..].as_bytes()).await?;
Ok(())
}
#[tracing::instrument(skip_all, fields(package = info.name))]
fn make_package(info: &PackageInfo) -> Result<Package> {
let mut pkg = Package::new(info.name.clone(), PathBuf::new());
for f in &info.files {
let mut it = f.rsplit('.');
let file_type = it
.next()
.ok_or_else(|| eyre::eyre!("missing file extension"))
.and_then(BundleFileType::from_str)
.wrap_err("Invalid file name in package info")?;
let name: String = it.collect();
pkg.add_file(file_type, name);
}
Ok(pkg)
}
#[tracing::instrument]
async fn copy_recursive(
from: impl Into<PathBuf> + std::fmt::Debug,
to: impl AsRef<Path> + std::fmt::Debug,
) -> Result<()> {
let to = to.as_ref();
#[tracing::instrument]
async fn handle_dir(from: PathBuf) -> Result<Vec<(bool, DirEntry)>> {
let mut dir = fs::read_dir(&from)
.await
.wrap_err("Failed to read directory")?;
let mut entries = Vec::new();
while let Some(entry) = dir.next_entry().await? {
let meta = entry.metadata().await.wrap_err_with(|| {
format!("Failed to get metadata for '{}'", entry.path().display())
})?;
entries.push((meta.is_dir(), entry));
}
Ok(entries)
}
let base = from.into();
stream::unfold(vec![base.clone()], |mut state| async {
let from = state.pop()?;
let inner = match handle_dir(from).await {
Ok(entries) => {
for (is_dir, entry) in &entries {
if *is_dir {
state.push(entry.path());
}
}
stream::iter(entries).map(Ok).left_stream()
}
Err(e) => stream::once(async { Err(e) }).right_stream(),
};
Some((inner, state))
})
.flatten()
.try_for_each(|(is_dir, entry)| {
let path = entry.path();
let dest = path
.strip_prefix(&base)
.map(|suffix| to.join(suffix))
.expect("all entries are relative to the directory we are walking");
async move {
if is_dir {
tracing::trace!("Creating directory '{}'", dest.display());
// Instead of trying to filter "already exists" errors out explicitly,
// we just ignore all. It'll fail eventually with the next copy operation.
let _ = fs::create_dir(&dest).await;
Ok(())
} else {
tracing::trace!("Copying file '{}' -> '{}'", path.display(), dest.display());
fs::copy(&path, &dest).await.map(|_| ()).wrap_err_with(|| {
format!(
"Failed to copy file '{}' -> '{}'",
path.display(),
dest.display()
)
})
}
}
})
.await
.map(|_| ())
}
#[tracing::instrument(skip(state))]
async fn copy_mod_folders(state: Arc<ActionState>) -> Result<Vec<String>> {
let game_dir = Arc::clone(&state.game_dir);
let mut tasks = Vec::new();
for mod_info in state
.mods
.iter()
.filter(|m| m.id != "dml" && m.enabled && !m.bundled)
{
let span = tracing::trace_span!("copying legacy mod", name = mod_info.name);
let _enter = span.enter();
let mod_id = mod_info.id.clone();
let mod_dir = Arc::clone(&state.mod_dir);
let game_dir = Arc::clone(&game_dir);
let task = async move {
let from = mod_dir.join(&mod_id);
let to = game_dir.join("mods").join(&mod_id);
tracing::debug!(from = %from.display(), to = %to.display(), "Copying legacy mod '{}'", mod_id);
let _ = fs::create_dir_all(&to).await;
copy_recursive(&from, &to).await.wrap_err_with(|| {
format!(
"Failed to copy legacy mod from '{}' to '{}'",
from.display(),
to.display()
)
})?;
Ok::<_, Report>(mod_id)
};
tasks.push(task);
}
let ids = futures::future::try_join_all(tasks).await?;
Ok(ids)
}
fn build_mod_data_lua(state: Arc<ActionState>) -> Result<String> {
#[derive(Serialize)]
struct TemplateDataMod {
id: String,
name: String,
bundled: bool,
init: String,
data: Option<String>,
localization: Option<String>,
packages: Vec<String>,
}
let mut env = Environment::new();
env.add_template("mod_data.lua", include_str!("../../assets/mod_data.lua.j2"))
.wrap_err("Failed to compile template for `mod_data.lua`")?;
let tmpl = env
.get_template("mod_data.lua")
.wrap_err("Failed to get template `mod_data.lua`")?;
let data: Vec<TemplateDataMod> = state
.mods
.iter()
.filter_map(|m| {
if m.id == "dml" || !m.enabled {
return None;
}
Some(TemplateDataMod {
id: m.id.clone(),
name: m.name.clone(),
bundled: m.bundled,
init: m.resources.init.to_string_lossy().to_string(),
data: m
.resources
.data
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
localization: m
.resources
.localization
.as_ref()
.map(|p| p.to_string_lossy().to_string()),
packages: m.packages.iter().map(|p| p.name.clone()).collect(),
})
})
.collect();
let lua = tmpl
.render(minijinja::context!(mods => data))
.wrap_err("Failed to render template `mod_data.lua`")?;
tracing::debug!("mod_data.lua:\n{}", lua);
Ok(lua)
}
#[tracing::instrument(skip_all)]
async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string());
let mut tasks = Vec::new();
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let mut bundles = Vec::new();
{
tracing::trace!("Building mod data script");
let span = tracing::debug_span!("Building mod data script");
let _enter = span.enter();
let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?;
tracing::trace!("Compiling mod data script");
let file =
lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?;
tracing::trace!("Compile mod data script");
mod_bundle.add_file(file);
}
tracing::trace!("Preparing tasks to deploy bundle files");
for mod_info in state
.mods
.iter()
.filter(|m| m.id != "dml" && m.enabled && m.bundled)
{
let span = tracing::trace_span!("building mod packages", name = mod_info.name);
let _enter = span.enter();
let mod_dir = state.mod_dir.join(&mod_info.id);
for pkg_info in &mod_info.packages {
let span = tracing::trace_span!("building package", name = pkg_info.name);
let _enter = span.enter();
tracing::trace!(
"Building package {} for mod {}",
pkg_info.name,
mod_info.name
);
let pkg = make_package(pkg_info).wrap_err("Failed to make package")?;
let mut variant = BundleFileVariant::new();
let bin = pkg
.to_binary()
.wrap_err("Failed to serialize package to binary")?;
variant.set_data(bin);
let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package);
file.add_variant(variant);
tracing::trace!(
"Compiled package {} for mod {}",
pkg_info.name,
mod_info.name
);
mod_bundle.add_file(file);
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = mod_dir.join(&bundle_name);
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
// Explicitely drop the guard, so that we can move the span
// into the async operation
drop(_enter);
let ctx = state.ctx.clone();
let task = async move {
let bundle = {
let bin = fs::read(&src).await.wrap_err_with(|| {
format!("Failed to read bundle file '{}'", src.display())
})?;
let name = Bundle::get_name_from_path(&ctx, &src);
Bundle::from_binary(&ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?
};
tracing::debug!(
src = %src.display(),
dest = %dest.display(),
"Copying bundle '{}' for mod '{}'",
pkg_name,
mod_name,
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
Ok::<Bundle, color_eyre::Report>(bundle)
}
.instrument(span);
tasks.push(task);
}
}
tracing::debug!("Copying {} mod bundles", tasks.len());
let mut tasks = stream::iter(tasks).buffer_unordered(10);
while let Some(res) = tasks.next().await {
let bundle = res?;
bundles.push(bundle);
}
{
let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64()));
tracing::trace!("Writing mod bundle to '{}'", path.display());
fs::write(&path, mod_bundle.to_binary()?)
.await
.wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?;
}
bundles.push(mod_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all)]
async fn patch_boot_bundle(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())));
let mut bundles = Vec::with_capacity(2);
let mut boot_bundle = async {
let bin = read_file_with_backup(&bundle_path)
.await
.wrap_err("Failed to read boot bundle")?;
Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin)
.wrap_err("Failed to parse boot bundle")
}
.instrument(tracing::trace_span!("read boot bundle"))
.await
.wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?;
{
tracing::trace!("Adding mod package file to boot bundle");
let span = tracing::trace_span!("create mod package file");
let _enter = span.enter();
let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new());
for mod_info in &state.mods {
for pkg_info in &mod_info.packages {
pkg.add_file(BundleFileType::Package, &pkg_info.name);
}
}
pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT);
let mut variant = BundleFileVariant::new();
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
tracing::trace!("Handling DML packages and bundle");
let span = tracing::trace_span!("handle DML");
let _enter = span.enter();
let mut variant = BundleFileVariant::new();
let mod_info = state
.mods
.iter()
.find(|m| m.id == "dml")
.ok_or_else(|| eyre::eyre!("DML not found in mod list"))?;
let pkg_info = mod_info
.packages
.get(0)
.ok_or_else(|| eyre::eyre!("invalid mod package for DML"))
.with_suggestion(|| "Re-download and import the newest version.".to_string())?;
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = state.mod_dir.join(&mod_info.id).join(&bundle_name);
{
let bin = fs::read(&src)
.await
.wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?;
let name = Bundle::get_name_from_path(&state.ctx, &src);
let dml_bundle = Bundle::from_binary(&state.ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?;
bundles.push(dml_bundle);
};
{
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
tracing::debug!(
"Copying bundle {} for mod {}: {} -> {}",
pkg_name,
mod_name,
src.display(),
dest.display()
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
}
let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?;
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
let span = tracing::debug_span!("Importing mod main script");
let _enter = span.enter();
let is_io_enabled = format!("{}", state.is_io_enabled);
let mut data = HashMap::new();
data.insert("is_io_enabled", is_io_enabled.as_str());
let tmpl = include_str!("../../assets/mod_main.lua");
let lua = Template::new(tmpl).render(&data);
tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
let file =
lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
boot_bundle.add_file(file);
}
async {
let bin = boot_bundle
.to_binary()
.wrap_err("Failed to serialize boot bundle")?;
fs::write(&bundle_path, bin)
.await
.wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display()))
}
.instrument(tracing::trace_span!("write boot bundle"))
.await?;
bundles.push(boot_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn patch_bundle_database<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let mut db = {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("Failed to read bundle database")?;
let mut r = Cursor::new(bin);
let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?;
tracing::trace!("Finished parsing bundle database");
db
};
for bundle in bundles.as_ref() {
tracing::trace!("Adding '{}' to bundle database", bundle.name().display());
db.add_bundle(bundle);
}
{
let bin = db
.to_binary()
.wrap_err("Failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})?;
}
Ok(())
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn write_deployment_data<B>(
state: Arc<ActionState>,
bundles: B,
mod_folders: Vec<String>,
) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let info = DeploymentData {
timestamp: OffsetDateTime::now_utc(),
bundles: bundles
.as_ref()
.iter()
.map(|bundle| format!("{:x}", bundle.name().to_murmur64()))
.collect(),
// TODO:
mod_folders,
};
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?;
fs::write(&path, &data)
.await
.wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.game_dir.display(),
mods = state.mods.len()
))]
pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> {
let state = Arc::new(state);
let bundle_dir = state.game_dir.join("bundle");
let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()));
if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999")))
.await
.is_ok()
{
let err = eyre::eyre!("Found dtkit-patch-based mod installation.");
return Err(err)
.with_suggestion(|| {
"If you're a mod author and saved projects directly in 'mods/', \
use DTMT to migrate them to the new project structure."
.to_string()
})
.with_suggestion(|| {
"Click 'Reset Game' to remove the previous mod installation.".to_string()
});
}
let (_, game_info, deployment_info) = tokio::try_join!(
async {
fs::metadata(&bundle_dir)
.await
.wrap_err("Failed to open game bundle directory")
.with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.")
},
async {
tokio::task::spawn_blocking(dtmt_shared::collect_game_info)
.await
.map_err(Report::new)
},
async {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
match read_sjson_file::<_, DeploymentData>(path).await {
Ok(data) => Ok(Some(data)),
Err(err) => {
if let Some(err) = err.downcast_ref::<std::io::Error>()
&& err.kind() == ErrorKind::NotFound
{
Ok(None)
} else {
Err(err).wrap_err("Failed to read deployment data")
}
}
}
}
)
.wrap_err("Failed to gather deployment information")?;
tracing::debug!(?game_info, ?deployment_info);
if let Some(game_info) = game_info {
if deployment_info
.as_ref()
.map(|i| game_info.last_updated > i.timestamp)
.unwrap_or(false)
{
tracing::warn!(
"Game was updated since last mod deployment. \
Attempting to reconcile game files."
);
tokio::try_join!(
async {
let path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let backup_path = path.with_extension("data.bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for bundle database.")
},
async {
let path = bundle_dir.join(boot_bundle_path);
let backup_path = path.with_extension("bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for boot bundle")
}
)
.with_suggestion(|| {
"Reset the game using 'Reset Game', then verify game files.".to_string()
})?;
tracing::info!(
"Successfully re-created game file backups. \
Continuing mod deployment."
);
}
}
check_mod_order(&state)?;
tracing::info!(
"Deploying {} mods to '{}'.",
state.mods.iter().filter(|i| i.enabled).count(),
bundle_dir.display()
);
tracing::info!("Copy legacy mod folders");
let mod_folders = copy_mod_folders(state.clone())
.await
.wrap_err("Failed to copy mod folders")?;
tracing::info!("Build mod bundles");
let mut bundles = build_bundles(state.clone())
.await
.wrap_err("Failed to build mod bundles")?;
tracing::info!("Patch boot bundle");
let mut boot_bundles = patch_boot_bundle(state.clone())
.await
.wrap_err("Failed to patch boot bundle")?;
bundles.append(&mut boot_bundles);
if let Some(info) = &deployment_info {
let bundle_dir = Arc::new(bundle_dir);
// Remove bundles from the previous deployment that don't match the current one.
// I.e. mods that used to be installed/enabled but aren't anymore.
{
let tasks = info.bundles.iter().cloned().filter_map(|file_name| {
let is_being_deployed = bundles.iter().any(|b2| {
let name = format!("{:016x}", b2.name());
file_name == name
});
if !is_being_deployed {
let bundle_dir = bundle_dir.clone();
let task = async move {
let path = bundle_dir.join(&file_name);
tracing::debug!("Removing unused bundle '{}'", file_name);
if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| {
format!("Failed to remove unused bundle '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
// Do the same thing for mod folders
{
let tasks = info.mod_folders.iter().filter_map(|mod_id| {
let is_being_deployed = mod_folders.iter().any(|id| id == mod_id);
if !is_being_deployed {
let path = bundle_dir.join("mods").join(mod_id);
tracing::debug!("Removing unused mod folder '{}'", path.display());
let task = async move {
if let Err(err) = fs::remove_dir_all(&path).await.wrap_err_with(|| {
format!("Failed to remove unused legacy mod '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
}
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("Failed to patch game settings")?;
tracing::info!("Patching bundle database");
patch_bundle_database(state.clone(), &bundles)
.await
.wrap_err("Failed to patch bundle database")?;
tracing::info!("Writing deployment data");
write_deployment_data(state.clone(), &bundles, mod_folders)
.await
.wrap_err("Failed to write deployment data")?;
tracing::info!("Finished deploying mods");
Ok(())
}

624
crates/dtmm/src/controller/game.rs
View file

@ -1,19 +1,46 @@
use std::io::{self, ErrorKind};
use std::collections::HashMap;
use std::io::{self, Cursor, ErrorKind};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Result};
use color_eyre::{eyre, Help, Report, Result};
use futures::stream;
use futures::StreamExt;
use path_slash::PathBufExt;
use sdk::filetype::lua;
use sdk::filetype::package::Package;
use sdk::murmur::Murmur64;
use tokio::fs::{self};
use tokio::io::AsyncWriteExt;
use crate::controller::deploy::{
DeploymentData, BOOT_BUNDLE_NAME, BUNDLE_DATABASE_NAME, DEPLOYMENT_DATA_PATH,
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use crate::state::ActionState;
use serde::{Deserialize, Serialize};
use string_template::Template;
use time::OffsetDateTime;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use tracing::Instrument;
use super::deploy::SETTINGS_FILE_PATH;
use super::read_sjson_file;
use crate::controller::app::check_mod_order;
use crate::state::{ActionState, PackageInfo};
const MOD_BUNDLE_NAME: &str = "packages/mods";
const BOOT_BUNDLE_NAME: &str = "packages/boot";
const DML_BUNDLE_NAME: &str = "packages/dml";
const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini";
const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson";
#[derive(Debug, Serialize, Deserialize)]
struct DeploymentData {
bundles: Vec<String>,
#[serde(with = "time::serde::iso8601")]
timestamp: OffsetDateTime,
}
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
@ -103,6 +130,585 @@ async fn patch_game_settings(state: Arc<ActionState>) -> Result<()> {
Ok(())
}
#[tracing::instrument(skip_all, fields(package = info.name))]
fn make_package(info: &PackageInfo) -> Result<Package> {
let mut pkg = Package::new(info.name.clone(), PathBuf::new());
for f in &info.files {
let mut it = f.rsplit('.');
let file_type = it
.next()
.ok_or_else(|| eyre::eyre!("missing file extension"))
.and_then(BundleFileType::from_str)
.wrap_err("Invalid file name in package info")?;
let name: String = it.collect();
pkg.add_file(file_type, name);
}
Ok(pkg)
}
fn build_mod_data_lua(state: Arc<ActionState>) -> String {
let mut lua = String::from("return {\n");
// DMF is handled explicitely by the loading procedures, as it actually drives most of that
// and should therefore not show up in the load order.
for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) {
lua.push_str(" {\n name = \"");
lua.push_str(&mod_info.name);
lua.push_str("\",\n id = \"");
lua.push_str(&mod_info.id);
lua.push_str("\",\n run = function()\n");
let resources = &mod_info.resources;
if resources.data.is_some() || resources.localization.is_some() {
lua.push_str(" new_mod(\"");
lua.push_str(&mod_info.id);
lua.push_str("\", {\n mod_script = \"");
lua.push_str(&resources.init.to_slash_lossy());
if let Some(data) = resources.data.as_ref() {
lua.push_str("\",\n mod_data = \"");
lua.push_str(&data.to_slash_lossy());
}
if let Some(localization) = &resources.localization {
lua.push_str("\",\n mod_localization = \"");
lua.push_str(&localization.to_slash_lossy());
}
lua.push_str("\",\n })\n");
} else {
lua.push_str(" return dofile(\"");
lua.push_str(&resources.init.to_slash_lossy());
lua.push_str("\")\n");
}
lua.push_str(" end,\n packages = {\n");
for pkg_info in &mod_info.packages {
lua.push_str(" \"");
lua.push_str(&pkg_info.name);
lua.push_str("\",\n");
}
lua.push_str(" },\n },\n");
}
lua.push('}');
tracing::debug!("mod_data_lua:\n{}", lua);
lua
}
#[tracing::instrument(skip_all)]
async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let mut mod_bundle = Bundle::new(MOD_BUNDLE_NAME.to_string());
let mut tasks = Vec::new();
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let mut bundles = Vec::new();
{
tracing::trace!("Building mod data script");
let span = tracing::debug_span!("Building mod data script");
let _enter = span.enter();
let lua = build_mod_data_lua(state.clone());
tracing::trace!("Compiling mod data script");
let file =
lua::compile(MOD_DATA_SCRIPT, &lua).wrap_err("Failed to compile mod data Lua file")?;
tracing::trace!("Compile mod data script");
mod_bundle.add_file(file);
}
tracing::trace!("Preparing tasks to deploy bundle files");
for mod_info in state.mods.iter().filter(|m| m.id != "dml" && m.enabled) {
let span = tracing::trace_span!("building mod packages", name = mod_info.name);
let _enter = span.enter();
let mod_dir = state.mod_dir.join(&mod_info.id);
for pkg_info in &mod_info.packages {
let span = tracing::trace_span!("building package", name = pkg_info.name);
let _enter = span.enter();
tracing::trace!(
"Building package {} for mod {}",
pkg_info.name,
mod_info.name
);
let pkg = make_package(pkg_info).wrap_err("Failed to make package")?;
let mut variant = BundleFileVariant::new();
let bin = pkg
.to_binary()
.wrap_err("Failed to serialize package to binary")?;
variant.set_data(bin);
let mut file = BundleFile::new(pkg_info.name.clone(), BundleFileType::Package);
file.add_variant(variant);
tracing::trace!(
"Compiled package {} for mod {}",
pkg_info.name,
mod_info.name
);
mod_bundle.add_file(file);
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = mod_dir.join(&bundle_name);
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
// Explicitely drop the guard, so that we can move the span
// into the async operation
drop(_enter);
let ctx = state.ctx.clone();
let task = async move {
let bundle = {
let bin = fs::read(&src).await.wrap_err_with(|| {
format!("Failed to read bundle file '{}'", src.display())
})?;
let name = Bundle::get_name_from_path(&ctx, &src);
Bundle::from_binary(&ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?
};
tracing::debug!(
src = %src.display(),
dest = %dest.display(),
"Copying bundle '{}' for mod '{}'",
pkg_name,
mod_name,
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
Ok::<Bundle, color_eyre::Report>(bundle)
}
.instrument(span);
tasks.push(task);
}
}
tracing::debug!("Copying {} mod bundles", tasks.len());
let mut tasks = stream::iter(tasks).buffer_unordered(10);
while let Some(res) = tasks.next().await {
let bundle = res?;
bundles.push(bundle);
}
{
let path = bundle_dir.join(format!("{:x}", mod_bundle.name().to_murmur64()));
tracing::trace!("Writing mod bundle to '{}'", path.display());
fs::write(&path, mod_bundle.to_binary()?)
.await
.wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?;
}
bundles.push(mod_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all)]
async fn patch_boot_bundle(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())));
let mut bundles = Vec::with_capacity(2);
let mut boot_bundle = async {
let bin = read_file_with_backup(&bundle_path)
.await
.wrap_err("Failed to read boot bundle")?;
Bundle::from_binary(&state.ctx, BOOT_BUNDLE_NAME.to_string(), bin)
.wrap_err("Failed to parse boot bundle")
}
.instrument(tracing::trace_span!("read boot bundle"))
.await
.wrap_err_with(|| format!("Failed to read bundle '{}'", BOOT_BUNDLE_NAME))?;
{
tracing::trace!("Adding mod package file to boot bundle");
let span = tracing::trace_span!("create mod package file");
let _enter = span.enter();
let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new());
for mod_info in &state.mods {
for pkg_info in &mod_info.packages {
pkg.add_file(BundleFileType::Package, &pkg_info.name);
}
}
pkg.add_file(BundleFileType::Lua, MOD_DATA_SCRIPT);
let mut variant = BundleFileVariant::new();
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
tracing::trace!("Handling DML packages and bundle");
let span = tracing::trace_span!("handle DML");
let _enter = span.enter();
let mut variant = BundleFileVariant::new();
let mod_info = state
.mods
.iter()
.find(|m| m.id == "dml")
.ok_or_else(|| eyre::eyre!("DML not found in mod list"))?;
let pkg_info = mod_info
.packages
.get(0)
.ok_or_else(|| eyre::eyre!("invalid mod package for DML"))
.with_suggestion(|| "Re-download and import the newest version.".to_string())?;
let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
let src = state.mod_dir.join(&mod_info.id).join(&bundle_name);
{
let bin = fs::read(&src)
.await
.wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?;
let name = Bundle::get_name_from_path(&state.ctx, &src);
let dml_bundle = Bundle::from_binary(&state.ctx, name, bin)
.wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?;
bundles.push(dml_bundle);
};
{
let dest = bundle_dir.join(&bundle_name);
let pkg_name = pkg_info.name.clone();
let mod_name = mod_info.name.clone();
tracing::debug!(
"Copying bundle {} for mod {}: {} -> {}",
pkg_name,
mod_name,
src.display(),
dest.display()
);
// We attempt to remove any previous file, so that the hard link can be created.
// We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
// may be possible despite an error here, or the error will be reported by it anyways.
// TODO: There is a chance that we delete an actual game bundle, but with 64bit
// hashes, it's low enough for now, and the setup required to detect
// "game bundle vs mod bundle" is non-trivial.
let _ = fs::remove_file(&dest).await;
fs::copy(&src, &dest).await.wrap_err_with(|| {
format!(
"Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
src.display(),
dest.display()
)
})?;
}
let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?;
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
boot_bundle.add_file(f);
}
{
let span = tracing::debug_span!("Importing mod main script");
let _enter = span.enter();
let is_io_enabled = format!("{}", state.is_io_enabled);
let mut data = HashMap::new();
data.insert("is_io_enabled", is_io_enabled.as_str());
let tmpl = include_str!("../../assets/mod_main.lua");
let lua = Template::new(tmpl).render(&data);
tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
let file =
lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
boot_bundle.add_file(file);
}
async {
let bin = boot_bundle
.to_binary()
.wrap_err("Failed to serialize boot bundle")?;
fs::write(&bundle_path, bin)
.await
.wrap_err_with(|| format!("Failed to write main bundle: {}", bundle_path.display()))
}
.instrument(tracing::trace_span!("write boot bundle"))
.await?;
bundles.push(boot_bundle);
Ok(bundles)
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn patch_bundle_database<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let mut db = {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("Failed to read bundle database")?;
let mut r = Cursor::new(bin);
let db = BundleDatabase::from_binary(&mut r).wrap_err("Failed to parse bundle database")?;
tracing::trace!("Finished parsing bundle database");
db
};
for bundle in bundles.as_ref() {
tracing::trace!("Adding '{}' to bundle database", bundle.name().display());
db.add_bundle(bundle);
}
{
let bin = db
.to_binary()
.wrap_err("Failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})?;
}
Ok(())
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn write_deployment_data<B>(state: Arc<ActionState>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let info = DeploymentData {
timestamp: OffsetDateTime::now_utc(),
bundles: bundles
.as_ref()
.iter()
.map(|bundle| format!("{:x}", bundle.name().to_murmur64()))
.collect(),
};
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = serde_sjson::to_string(&info).wrap_err("Failed to serizalie deployment data")?;
fs::write(&path, &data)
.await
.wrap_err_with(|| format!("Failed to write deployment data to '{}'", path.display()))?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.game_dir.display(),
mods = state.mods.len()
))]
pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> {
let state = Arc::new(state);
let bundle_dir = state.game_dir.join("bundle");
let boot_bundle_path = format!("{:016x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes()));
if fs::metadata(bundle_dir.join(format!("{boot_bundle_path}.patch_999")))
.await
.is_ok()
{
let err = eyre::eyre!("Found dtkit-patch-based mod installation.");
return Err(err)
.with_suggestion(|| {
"If you're a mod author and saved projects directly in 'mods/', \
use DTMT to migrate them to the new project structure."
.to_string()
})
.with_suggestion(|| {
"Click 'Reset Game' to remove the previous mod installation.".to_string()
});
}
let (_, game_info, deployment_info) = tokio::try_join!(
async {
fs::metadata(&bundle_dir)
.await
.wrap_err("Failed to open game bundle directory")
.with_suggestion(|| "Double-check 'Game Directory' in the Settings tab.")
},
async {
tokio::task::spawn_blocking(dtmt_shared::collect_game_info)
.await
.map_err(Report::new)
},
async {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
match read_sjson_file::<_, DeploymentData>(path)
.await
{
Ok(data) => Ok(Some(data)),
Err(err) => {
if let Some(err) = err.downcast_ref::<std::io::Error>() && err.kind() == ErrorKind::NotFound {
Ok(None)
} else {
Err(err).wrap_err("Failed to read deployment data")
}
}
}
}
)
.wrap_err("Failed to gather deployment information")?;
tracing::debug!(?game_info, ?deployment_info);
if let Some(game_info) = game_info {
if deployment_info
.as_ref()
.map(|i| game_info.last_updated > i.timestamp)
.unwrap_or(false)
{
tracing::warn!(
"Game was updated since last mod deployment. \
Attempting to reconcile game files."
);
tokio::try_join!(
async {
let path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let backup_path = path.with_extension("data.bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for bundle database.")
},
async {
let path = bundle_dir.join(boot_bundle_path);
let backup_path = path.with_extension("bak");
fs::copy(&path, &backup_path)
.await
.wrap_err("Failed to re-create backup for boot bundle")
}
)
.with_suggestion(|| {
"Reset the game using 'Reset Game', then verify game files.".to_string()
})?;
tracing::info!(
"Successfully re-created game file backups. \
Continuing mod deployment."
);
}
}
check_mod_order(&state)?;
tracing::info!(
"Deploying {} mods to '{}'.",
state.mods.iter().filter(|i| i.enabled).count(),
bundle_dir.display()
);
tracing::info!("Build mod bundles");
let mut bundles = build_bundles(state.clone())
.await
.wrap_err("Failed to build mod bundles")?;
tracing::info!("Patch boot bundle");
let mut more_bundles = patch_boot_bundle(state.clone())
.await
.wrap_err("Failed to patch boot bundle")?;
bundles.append(&mut more_bundles);
if let Some(info) = &deployment_info {
let bundle_dir = Arc::new(bundle_dir);
let tasks = info.bundles.iter().cloned().filter_map(|file_name| {
let contains = bundles.iter().any(|b2| {
let name = format!("{:016x}", b2.name());
file_name == name
});
if !contains {
let bundle_dir = bundle_dir.clone();
let task = async move {
let path = bundle_dir.join(&file_name);
tracing::debug!("Removing unused bundle '{}'", file_name);
if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| {
format!("Failed to remove unused bundle '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("Failed to patch game settings")?;
tracing::info!("Patching bundle database");
patch_bundle_database(state.clone(), &bundles)
.await
.wrap_err("Failed to patch bundle database")?;
tracing::info!("Writing deployment data");
write_deployment_data(state.clone(), &bundles)
.await
.wrap_err("Failed to write deployment data")?;
tracing::info!("Finished deploying mods");
Ok(())
}
#[tracing::instrument(skip_all)]
async fn reset_dtkit_patch(state: ActionState) -> Result<()> {
let bundle_dir = state.game_dir.join("bundle");

495
crates/dtmm/src/controller/import.rs
View file

@ -1,495 +0,0 @@
use std::collections::HashMap;
use std::ffi::CStr;
use std::io::{Cursor, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use color_eyre::eyre::{self, Context};
use color_eyre::{Help, Report, Result};
use druid::im::Vector;
use druid::{FileInfo, ImageBuf};
use dtmt_shared::{ModConfig, ModConfigResources};
use luajit2_sys as lua;
use nexusmods::Api as NexusApi;
use tokio::fs;
use zip::ZipArchive;
use crate::state::{ActionState, ModInfo, NexusInfo, PackageInfo};
fn find_archive_file<R: Read + Seek>(
archive: &ZipArchive<R>,
name: impl AsRef<str>,
) -> Option<String> {
let path = archive
.file_names()
.find(|path| path.ends_with(name.as_ref()))
.map(|s| s.to_string());
path
}
// Runs the content of a `.mod` file to extract what data we can get
// from legacy mods.
// 1. Create a global function `new_mod` that stores
// the relevant bits in global variables.
// 2. Run the `.mod` file, which will return a table.
// 3. Run the `run` function from that table.
// 4. Access the global variables from #1.
#[tracing::instrument]
fn parse_mod_id_file(data: &str) -> Result<(String, ModConfigResources)> {
tracing::debug!("Parsing mod file:\n{}", data);
let ret = unsafe {
let state = lua::luaL_newstate();
lua::luaL_openlibs(state);
let run = b"
function fassert() end
function new_mod(id, resources)
_G.id = id
_G.script = resources.mod_script
_G.data = resources.mod_data
_G.localization = resources.mod_localization
end
\0";
match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRSYNTAX => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Invalid syntax: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to create `new_mod`")
}
_ => unreachable!(),
}
match lua::lua_pcall(state, 0, 0, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run buffer: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run buffer")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let name = b".mod\0";
match lua::luaL_loadbuffer(
state,
data.as_ptr() as _,
data.len() as _,
name.as_ptr() as _,
) as u32
{
lua::LUA_OK => {}
lua::LUA_ERRSYNTAX => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Invalid syntax: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to load `.mod` file buffer")
}
_ => unreachable!(),
}
match lua::lua_pcall(state, 0, 1, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run `.mod` file: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run `.mod` file")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let key = b"run\0";
lua::lua_pushstring(state, key.as_ptr() as _);
lua::lua_gettable(state, -2);
match lua::lua_pcall(state, 0, 0, 0) as u32 {
lua::LUA_OK => {}
lua::LUA_ERRRUN => {
let err = lua::lua_tostring(state, -1);
let err = CStr::from_ptr(err).to_string_lossy().to_string();
lua::lua_close(state);
eyre::bail!("Failed to run `.mod.run`: {}", err);
}
lua::LUA_ERRMEM => {
lua::lua_close(state);
eyre::bail!("Failed to allocate sufficient memory to run `.mod.run`")
}
// We don't use an error handler function, so this should be unreachable
lua::LUA_ERRERR => unreachable!(),
_ => unreachable!(),
}
let get_global = |state, key: &[u8]| {
lua::lua_getglobal(state, key.as_ptr() as _);
if lua::lua_isnil(state, -1) != 0 {
return Ok(None);
}
let s = lua::lua_tostring(state, -1);
if s.is_null() {
eyre::bail!("Expected string, got NULL");
}
let ret = CStr::from_ptr(s).to_string_lossy().to_string();
lua::lua_pop(state, 1);
Ok(Some(ret))
};
let mod_id = get_global(state, b"id\0")
.and_then(|s| s.ok_or_else(|| eyre::eyre!("Got `nil`")))
.wrap_err("Failed to get `id`")?;
let resources = ModConfigResources {
init: get_global(state, b"script\0")
.and_then(|s| s.map(PathBuf::from).ok_or_else(|| eyre::eyre!("Got `nil`")))
.wrap_err("Failed to get `script`.")?,
data: get_global(state, b"data\0")
.wrap_err("Failed to get `data`.")?
.map(PathBuf::from),
localization: get_global(state, b"localization\0")
.wrap_err("Failed to get `localization`")?
.map(PathBuf::from),
};
lua::lua_close(state);
(mod_id, resources)
};
Ok(ret)
}
// Extracts the mod configuration from the mod archive.
// This may either be a proper `dtmt.cfg`, or the legacy `<mod_name>.mod` ID file.
//
// It also returns the directory where this file was found, used as root path. This
// allows flexibility in what the directory structure is exactly, since many people
// still end up creating tarbombs and Nexus does its own re-packaging.
#[tracing::instrument(skip(archive))]
fn extract_mod_config<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Result<(ModConfig, String)> {
let legacy_mod_data = if let Some(name) = find_archive_file(archive, ".mod") {
let (mod_id, resources) = {
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read `.mod` file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read `.mod` file from archive")?;
let data = String::from_utf8(buf).wrap_err("`.mod` file is not valid UTF-8")?;
parse_mod_id_file(&data)
.wrap_err("Invalid `.mod` file")
.note(
"The `.mod` file's `run` function may not contain any additional logic \
besides the default.",
)
.suggestion("Contact the mod author to fix this.")?
};
let root = if let Some(index) = name.rfind('/') {
name[..index].to_string()
} else {
String::new()
};
Some((mod_id, resources, root))
} else {
None
};
if let Some(name) = find_archive_file(archive, "dtmt.cfg") {
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read mod config from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read mod config from archive")?;
let data = String::from_utf8(buf).wrap_err("Mod config is not valid UTF-8")?;
let mut cfg: ModConfig = serde_sjson::from_str(&data)
.wrap_err("Failed to deserialize mod config")
.suggestion("Contact the mod author to fix this.")?;
if let Some((mod_id, resources, root)) = legacy_mod_data {
if cfg.id != mod_id {
let err = eyre::eyre!("Mod ID in `dtmt.cfg` does not match mod ID in `.mod` file");
return Err(err).suggestion("Contact the mod author to fix this.");
}
cfg.resources = resources;
Ok((cfg, root))
} else {
let root = name
.strip_suffix("dtmt.cfg")
.expect("String must end with that suffix")
.to_string();
Ok((cfg, root))
}
} else {
eyre::bail!(
"Mod needs a config file or `.mod` file. \
Please get in touch with the author to provide a properly packaged mod."
);
}
}
#[tracing::instrument(skip(archive))]
fn extract_bundled_mod<R: Read + Seek>(
archive: &mut ZipArchive<R>,
root: String,
dest: impl AsRef<Path> + std::fmt::Debug,
) -> Result<Vector<Arc<PackageInfo>>> {
let files: HashMap<String, Vec<String>> = {
let name = archive
.file_names()
.find(|name| name.ends_with("files.sjson"))
.map(|s| s.to_string())
.ok_or_else(|| eyre::eyre!("archive does not contain file index"))?;
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read file index from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
let data = String::from_utf8(buf).wrap_err("File index is not valid UTF-8")?;
serde_sjson::from_str(&data).wrap_err("Failed to deserialize file index")?
};
tracing::trace!(?files);
let dest = dest.as_ref();
tracing::trace!("Extracting mod archive to {}", dest.display());
archive
.extract(dest)
.wrap_err_with(|| format!("Failed to extract archive to {}", dest.display()))?;
let packages = files
.into_iter()
.map(|(name, files)| Arc::new(PackageInfo::new(name, files.into_iter().collect())))
.collect();
tracing::trace!(?packages);
Ok(packages)
}
#[tracing::instrument(skip(archive))]
fn extract_legacy_mod<R: Read + Seek>(
archive: &mut ZipArchive<R>,
root: String,
dest: impl Into<PathBuf> + std::fmt::Debug,
) -> Result<()> {
let dest = dest.into();
let file_count = archive.len();
for i in 0..file_count {
let mut f = archive
.by_index(i)
.wrap_err_with(|| format!("Failed to get file at index {}", i))?;
let Some(name) = f.enclosed_name().map(|p| p.to_path_buf()) else {
let err = eyre::eyre!("File name in archive is not a safe path value.").suggestion(
"Only use well-known applications to create the ZIP archive, \
and don't create paths that point outside the archive directory.",
);
return Err(err);
};
let Ok(suffix) = name.strip_prefix(&root) else {
tracing::warn!(
"Skipping file outside of the mod root directory: {}",
name.display()
);
continue;
};
let name = dest.join(suffix);
if f.is_dir() {
// The majority of errors will actually be "X already exists".
// But rather than filter them invidually, we just ignore all of them.
// If there is a legitimate error of "couldn't create X", it will eventually fail when
// we try to put a file in there.
tracing::trace!("Creating directory '{}'", name.display());
let _ = std::fs::create_dir_all(&name);
} else {
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err_with(|| format!("Failed to read file '{}'", name.display()))?;
tracing::trace!("Writing file '{}'", name.display());
let mut out = std::fs::OpenOptions::new()
.write(true)
.create(true)
.open(&name)
.wrap_err_with(|| format!("Failed to open file '{}'", name.display()))?;
out.write_all(&buf)
.wrap_err_with(|| format!("Failed to write to '{}'", name.display()))?;
}
}
Ok(())
}
#[tracing::instrument(skip(state))]
pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<ModInfo> {
let data = fs::read(&info.path)
.await
.wrap_err_with(|| format!("Failed to read file {}", info.path.display()))?;
let data = Cursor::new(data);
let nexus = if let Some((_, id, version, _)) = info
.path
.file_name()
.and_then(|s| s.to_str())
.and_then(NexusApi::parse_file_name)
{
if !state.nexus_api_key.is_empty() {
let api = NexusApi::new(state.nexus_api_key.to_string())?;
let mod_info = api
.mods_id(id)
.await
.wrap_err_with(|| format!("Failed to query mod {} from Nexus", id))?;
let info = NexusInfo::from(mod_info);
tracing::debug!("{:?}", info);
Some((info, version))
} else {
None
}
} else {
None
};
let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
if tracing::enabled!(tracing::Level::DEBUG) {
let names = archive.file_names().fold(String::new(), |mut s, name| {
s.push('\n');
s.push_str(name);
s
});
tracing::debug!("Archive contents:{}", names);
}
let (mut mod_cfg, root) =
extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?;
tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id);
tracing::debug!(root, ?mod_cfg);
let image = if let Some(path) = &mod_cfg.image {
let name = archive
.file_names()
.find(|name| name.ends_with(&path.display().to_string()))
.map(|s| s.to_string())
.ok_or_else(|| eyre::eyre!("archive does not contain configured image file"))?;
let mut f = archive
.by_name(&name)
.wrap_err("Failed to read image file from archive")?;
let mut buf = Vec::with_capacity(f.size() as usize);
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
let img = match ImageBuf::from_data(&buf) {
Ok(img) => img,
Err(err) => {
let err = Report::msg(err.to_string())
.wrap_err("Invalid image data")
.note("Supported formats are: PNG, JPEG, Bitmap and WebP")
.suggestion("Contact the mod author to fix this");
return Err(err);
}
};
Some(img)
} else {
None
};
tracing::trace!(?image);
let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
let dest = mod_dir.join(&mod_cfg.id);
tracing::trace!("Creating mods directory {}", dest.display());
fs::create_dir_all(&dest)
.await
.wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
let packages = if mod_cfg.bundled {
extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")?
} else {
extract_legacy_mod(&mut archive, root, &dest).wrap_err("Failed to extract legacy mod")?;
if let Some((_, version)) = &nexus {
// We use the version number stored in the `ModInfo` to compare against the `NexusInfo`
// for version checks. So for this one, we can't actually rely on merely shadowing,
// like with the other fields.
mod_cfg.version = version.clone();
}
let data = serde_sjson::to_string(&mod_cfg).wrap_err("Failed to serialize mod config")?;
fs::write(dest.join("dtmt.cfg"), &data)
.await
.wrap_err("Failed to write mod config")?;
Default::default()
};
if let Some((nexus, _)) = &nexus {
let data = serde_sjson::to_string(nexus).wrap_err("Failed to serialize Nexus info")?;
let path = dest.join("nexus.sjson");
fs::write(&path, data.as_bytes())
.await
.wrap_err_with(|| format!("Failed to write Nexus info to '{}'", path.display()))?;
}
let info = ModInfo::new(mod_cfg, packages, image, nexus.map(|(info, _)| info));
Ok(info)
}

2
crates/dtmm/src/controller/mod.rs
View file

@ -5,9 +5,7 @@ use serde::Deserialize;
use tokio::fs;
pub mod app;
pub mod deploy;
pub mod game;
pub mod import;
pub mod worker;
#[tracing::instrument]

4
crates/dtmm/src/controller/worker.rs
View file

@ -13,9 +13,7 @@ use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::RwLock;
use crate::controller::app::*;
use crate::controller::deploy::deploy_mods;
use crate::controller::game::*;
use crate::controller::import::import_mod;
use crate::state::AsyncAction;
use crate::state::ACTION_FINISH_CHECK_UPDATE;
use crate::state::ACTION_FINISH_LOAD_INITIAL;
@ -38,9 +36,7 @@ async fn handle_action(
action_queue: Arc<RwLock<UnboundedReceiver<AsyncAction>>>,
) {
while let Some(action) = action_queue.write().await.recv().await {
if cfg!(debug_assertions) && !matches!(action, AsyncAction::Log(_)) {
tracing::debug!(?action);
}
let event_sink = event_sink.clone();
match action {

1
crates/dtmm/src/main.rs
View file

@ -1,7 +1,6 @@
#![recursion_limit = "256"]
#![feature(let_chains)]
#![feature(arc_unwrap_or_clone)]
#![feature(iterator_try_collect)]
#![windows_subsystem = "windows"]
use std::path::PathBuf;

4
crates/dtmm/src/state/data.rs
View file

@ -73,7 +73,6 @@ impl From<dtmt_shared::ModDependency> for ModDependency {
#[derive(Clone, Data, Debug, Lens, serde::Serialize, serde::Deserialize)]
pub(crate) struct NexusInfo {
pub id: u64,
pub name: String,
pub version: String,
pub author: String,
pub summary: Arc<String>,
@ -84,7 +83,6 @@ impl From<NexusMod> for NexusInfo {
fn from(value: NexusMod) -> Self {
Self {
id: value.mod_id,
name: value.name,
version: value.version,
author: value.author,
summary: Arc::new(value.summary),
@ -111,7 +109,6 @@ pub(crate) struct ModInfo {
#[data(ignore)]
pub resources: ModResourceInfo,
pub depends: Vector<ModDependency>,
pub bundled: bool,
#[data(ignore)]
pub nexus: Option<NexusInfo>,
}
@ -132,7 +129,6 @@ impl ModInfo {
version: cfg.version,
enabled: false,
packages,
bundled: cfg.bundled,
image,
categories: cfg.categories.into_iter().collect(),
resources: ModResourceInfo {

1
crates/dtmm/src/state/delegate.rs
View file

@ -411,7 +411,6 @@ impl AppDelegate<State> for Delegate {
state.config_path = Arc::new(config.path);
state.data_dir = Arc::new(config.data_dir);
state.game_dir = Arc::new(config.game_dir.unwrap_or_default());
state.nexus_api_key = Arc::new(config.nexus_api_key.unwrap_or_default());
state.is_io_enabled = config.unsafe_io;
}

11
crates/dtmm/src/ui/window/main.rs
View file

@ -135,13 +135,8 @@ fn build_mod_list() -> impl Widget<State> {
})
.lens(lens!((usize, Arc<ModInfo>, bool), 1).then(ModInfo::enabled.in_arc()));
let name = Label::dynamic(|info: &Arc<ModInfo>, _| {
info.nexus
.as_ref()
.map(|n| n.name.clone())
.unwrap_or_else(|| info.name.clone())
})
.lens(lens!((usize, Arc<ModInfo>, bool), 1));
let name =
Label::raw().lens(lens!((usize, Arc<ModInfo>, bool), 1).then(ModInfo::name.in_arc()));
let version = {
let icon = {
@ -150,7 +145,7 @@ fn build_mod_list() -> impl Widget<State> {
let tree = theme::icons::recolor_icon(tree, true, COLOR_YELLOW_LIGHT);
Svg::new(tree).fix_height(druid::theme::TEXT_SIZE_NORMAL)
Svg::new(Arc::new(tree)).fix_height(druid::theme::TEXT_SIZE_NORMAL)
};
Either::new(

8
crates/dtmm/src/util/config.rs
View file

@ -125,9 +125,6 @@ where
.wrap_err_with(|| format!("Invalid config file {}", path.display()))?;
cfg.path = path;
tracing::debug!("Read config file '{}': {:?}", cfg.path.display(), cfg);
Ok(cfg)
}
Err(err) if err.kind() == ErrorKind::NotFound => {
@ -136,11 +133,6 @@ where
.wrap_err_with(|| format!("Failed to read config file {}", path.display()))?;
}
tracing::debug!(
"Config file not found at '{}', creating default.",
path.display()
);
{
let parent = default_path
.parent()

5
crates/dtmt/src/cmd/dictionary.rs
View file

@ -145,10 +145,7 @@ pub(crate) async fn run(mut ctx: sdk::Context, matches: &ArgMatches) -> Result<(
.get_one::<HashGroup>("group")
.expect("required argument not found");
let r: BufReader<Box<dyn tokio::io::AsyncRead + std::marker::Unpin>> = if let Some(name) =
path.file_name()
&& name == "-"
{
let r: BufReader<Box<dyn tokio::io::AsyncRead + std::marker::Unpin>> = if let Some(name) = path.file_name() && name == "-" {
let f = tokio::io::stdin();
BufReader::new(Box::new(f))
} else {

1
crates/dtmt/src/cmd/migrate.rs
View file

@ -350,7 +350,6 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
localization: mod_file.localization,
},
depends: vec![ModDependency::ID(String::from("DMF"))],
bundled: true,
};
tracing::debug!(?dtmt_cfg);

13
lib/dtmt-shared/src/lib.rs
View file

@ -30,17 +30,6 @@ pub enum ModDependency {
Config { id: String, order: ModOrder },
}
// A bit dumb, but serde doesn't support literal values with the
// `default` attribute, only paths.
fn default_true() -> bool {
true
}
// Similarly dumb, as the `skip_serializing_if` attribute needs a function
fn is_true(val: &bool) -> bool {
*val
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
pub struct ModConfig {
#[serde(skip)]
@ -62,8 +51,6 @@ pub struct ModConfig {
pub resources: ModConfigResources,
#[serde(default)]
pub depends: Vec<ModDependency>,
#[serde(default = "default_true", skip_serializing_if = "is_true")]
pub bundled: bool,
}
pub const STEAMAPP_ID: u32 = 1361210;

4
lib/nexusmods/src/lib.rs
View file

@ -112,7 +112,7 @@ impl Api {
RE.captures(name.as_ref()).and_then(|cap| {
let name = cap.name("name").map(|s| s.as_str().to_string())?;
let mod_id = cap.name("mod_id").and_then(|s| s.as_str().parse().ok())?;
let version = cap.name("version").map(|s| s.as_str().replace('-', "."))?;
let version = cap.name("version").map(|s| s.as_str().to_string())?;
let updated = cap
.name("updated")
.and_then(|s| s.as_str().parse().ok())
@ -154,7 +154,7 @@ impl Api {
self.mods_download_link(nxm.mod_id, nxm.file_id, nxm.key, nxm.expires)
)?;
let Some(download_url) = download_info.first().map(|i| i.uri.clone()) else {
let Some(download_url) = download_info.get(0).map(|i| i.uri.clone()) else {
return Err(Error::InvalidNXM("no download link", url));
};

7
lib/sdk/src/bundle/mod.rs
View file

@ -227,9 +227,12 @@ impl Bundle {
let _enter = span.enter();
tracing::trace!(num_files = self.files.len());
self.files.iter().try_fold(Vec::new(), |mut data, file| {
self.files
.iter()
.fold(Ok::<Vec<u8>, Report>(Vec::new()), |data, file| {
let mut data = data?;
data.append(&mut file.to_binary()?);
Ok::<_, Report>(data)
Ok(data)
})?
};

4
lib/sdk/src/filetype/lua.rs
View file

@ -53,8 +53,8 @@ where
let mut buf = vec![0u8; length];
r.read_exact(&mut buf)?;
let mut s =
String::from_utf8(buf).wrap_err("Invalid byte sequence for LuaJIT bytecode name")?;
let mut s = String::from_utf8(buf)
.wrap_err_with(|| format!("Invalid byte sequence for LuaJIT bytecode name"))?;
// Remove the leading `@`
s.remove(0);
s