Improve deployments and resets #50

Merged
lucas merged 6 commits from feat/deployment-improvements into master 2023-03-07 19:50:51 +01:00
12 changed files with 481 additions and 32 deletions

3
.gitmodules vendored
View file

@ -1,3 +1,6 @@
[submodule "lib/serde_sjson"]
path = lib/serde_sjson
url = git@git.sclu1034.dev:lucas/serde_sjson.git
[submodule "lib/steamlocate-rs"]
path = lib/steamlocate-rs
url = git@github.com:sclu1034/steamlocate-rs.git

View file

@ -7,6 +7,8 @@
- dtmt: split `build` into `build` and `package`
- dtmt: implement deploying built bundles
- dtmm: indicate when a deployment is necessary
- dtmm: check for Steam game update before deployment
- dtmm: remove unused bundles from previous deployment
=== Fixed

231
Cargo.lock generated
View file

@ -483,6 +483,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86ec7a15cbe22e59248fc7eadb1907dab5ba09372595da4d73dd805ed4417dfe"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cace84e55f07e7301bae1c519df89cdad8cc3cd868413d3fdbdeca9ff3db484"
[[package]]
name = "crc32fast"
version = "1.3.2"
@ -557,6 +572,15 @@ dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs"
version = "3.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-next"
version = "2.0.0"
@ -680,6 +704,7 @@ dependencies = [
"sdk",
"serde",
"serde_sjson",
"time",
"tokio",
"tokio-stream",
"tracing",
@ -725,7 +750,9 @@ dependencies = [
name = "dtmt-shared"
version = "0.1.0"
dependencies = [
"color-eyre",
"serde",
"steamlocate",
"time",
"tracing",
"tracing-error",
@ -750,6 +777,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "enum_primitive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4551092f4d519593039259a9ed8daedf0da12e5109c5280338073eaeb81180"
dependencies = [
"num-traits 0.1.43",
]
[[package]]
name = "errno"
version = "0.2.8"
@ -1372,6 +1408,17 @@ dependencies = [
"bitflags",
]
[[package]]
name = "keyvalues-parser"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d990301996c856ea07a84bc291e76f1273db52683663efc05c8d355976897e5"
dependencies = [
"pest",
"pest_derive",
"thiserror",
]
[[package]]
name = "kurbo"
version = "0.9.1"
@ -1533,6 +1580,12 @@ dependencies = [
"memoffset 0.6.5",
]
[[package]]
name = "nom"
version = "1.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5b8c256fd9471521bcb84c3cdba98921497f1a331cbc15b8030fc63b82050ce"
[[package]]
name = "nom"
version = "7.1.3"
@ -1551,7 +1604,7 @@ checksum = "b1e299bf5ea7b212e811e71174c5d1a5d065c4c0ad0c8691ecb1f97e3e66025e"
dependencies = [
"bytecount",
"memchr",
"nom",
"nom 7.1.3",
]
[[package]]
@ -1564,6 +1617,91 @@ dependencies = [
"winapi",
]
[[package]]
name = "num"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b7a8e9be5e039e2ff869df49155f1c06bd01ade2117ec783e56ab0932b67a8f"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits 0.2.15",
]
[[package]]
name = "num-bigint"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3"
dependencies = [
"autocfg",
"num-integer",
"num-traits 0.2.15",
]
[[package]]
name = "num-complex"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747d632c0c558b87dbabbe6a82f3b4ae03720d0646ac5b7b4dae89394be5f2c5"
dependencies = [
"num-traits 0.2.15",
]
[[package]]
name = "num-integer"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
dependencies = [
"autocfg",
"num-traits 0.2.15",
]
[[package]]
name = "num-iter"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
dependencies = [
"autocfg",
"num-integer",
"num-traits 0.2.15",
]
[[package]]
name = "num-rational"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07"
dependencies = [
"autocfg",
"num-bigint",
"num-integer",
"num-traits 0.2.15",
]
[[package]]
name = "num-traits"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
dependencies = [
"num-traits 0.2.15",
]
[[package]]
name = "num-traits"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.15.0"
@ -1728,6 +1866,50 @@ dependencies = [
"sha2",
]
[[package]]
name = "pest"
version = "2.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7"
dependencies = [
"thiserror",
"ucd-trie",
]
[[package]]
name = "pest_derive"
version = "2.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a81186863f3d0a27340815be8f2078dd8050b14cd71913db9fbda795e5f707d7"
dependencies = [
"pest",
"pest_generator",
]
[[package]]
name = "pest_generator"
version = "2.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75a1ef20bf3193c15ac345acb32e26b3dc3223aff4d77ae4fc5359567683796b"
dependencies = [
"pest",
"pest_meta",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "pest_meta"
version = "2.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e3b284b1f13a20dc5ebc90aff59a51b8d7137c221131b52a7260c08cbc1cc80"
dependencies = [
"once_cell",
"pest",
"sha2",
]
[[package]]
name = "piet"
version = "0.6.2"
@ -2098,7 +2280,7 @@ dependencies = [
name = "serde_sjson"
version = "0.2.4"
dependencies = [
"nom",
"nom 7.1.3",
"nom_locate",
"serde",
]
@ -2168,6 +2350,42 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
[[package]]
name = "steamid-ng"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffb049f8faa2cba570c5366dbaf88ee5849725b16edb771848639fac92e33673"
dependencies = [
"enum_primitive",
"lazy_static",
"num",
"regex",
"serde",
"serde_derive",
"thiserror",
]
[[package]]
name = "steamlocate"
version = "1.1.1"
dependencies = [
"crc",
"dirs",
"keyvalues-parser",
"steamid-ng",
"steamy-vdf",
"winreg",
]
[[package]]
name = "steamy-vdf"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "533127ad49314bfe71c3d3fd36b3ebac3d24f40618092e70e1cfe8362c7fac79"
dependencies = [
"nom 1.2.4",
]
[[package]]
name = "str-buf"
version = "1.0.6"
@ -2791,6 +3009,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "winreg"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
dependencies = [
"winapi",
]
[[package]]
name = "wio"
version = "0.2.2"

View file

@ -24,3 +24,4 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
zip = "0.6.4"
tokio-stream = { version = "0.1.12", features = ["fs"] }
path-slash = "0.2.1"
time = { version = "0.3.20", features = ["serde", "serde-well-known", "local-offset"] }

View file

@ -8,7 +8,6 @@ use color_eyre::{Help, Result};
use druid::im::Vector;
use druid::FileInfo;
use dtmt_shared::ModConfig;
use serde::Deserialize;
use tokio::fs::{self, DirEntry};
use tokio::runtime::Runtime;
use tokio_stream::wrappers::ReadDirStream;
@ -18,6 +17,8 @@ use zip::ZipArchive;
use crate::state::{ModInfo, PackageInfo, State};
use crate::util::config::{ConfigSerialize, LoadOrderEntry};
use super::read_sjson_file;
#[tracing::instrument(skip(state))]
pub(crate) async fn import_mod(state: State, info: FileInfo) -> Result<ModInfo> {
let data = fs::read(&info.path)
@ -144,16 +145,6 @@ pub(crate) async fn save_settings(state: State) -> Result<()> {
})
}
async fn read_sjson_file<P, T>(path: P) -> Result<T>
where
T: for<'a> Deserialize<'a>,
P: AsRef<Path> + std::fmt::Debug,
{
let buf = fs::read(path).await.wrap_err("failed to read file")?;
let data = String::from_utf8(buf).wrap_err("invalid UTF8")?;
serde_sjson::from_str(&data).wrap_err("failed to deserialize")
}
#[tracing::instrument(skip_all,fields(
name = ?res.as_ref().map(|entry| entry.file_name())
))]

View file

@ -5,7 +5,7 @@ use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Help, Result};
use color_eyre::{eyre, Help, Report, Result};
use futures::stream;
use futures::StreamExt;
use path_slash::PathBufExt;
@ -15,10 +15,13 @@ use sdk::murmur::Murmur64;
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use serde::{Deserialize, Serialize};
use time::OffsetDateTime;
use tokio::fs;
use tokio::io::AsyncWriteExt;
use tracing::Instrument;
use super::read_sjson_file;
use crate::state::{PackageInfo, State};
const MOD_BUNDLE_NAME: &str = "packages/mods";
@ -28,6 +31,14 @@ const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
const SETTINGS_FILE_PATH: &str = "application_settings/settings_common.ini";
const DEPLOYMENT_DATA_PATH: &str = "dtmm-deployment.sjson";
#[derive(Debug, Serialize, Deserialize)]
struct DeploymentData {
bundles: Vec<String>,
#[serde(with = "time::serde::iso8601")]
timestamp: OffsetDateTime,
}
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
@ -449,8 +460,11 @@ async fn patch_boot_bundle(state: Arc<State>) -> Result<Vec<Bundle>> {
Ok(bundles)
}
#[tracing::instrument(skip_all, fields(bundles = bundles.len()))]
async fn patch_bundle_database(state: Arc<State>, bundles: Vec<Bundle>) -> Result<()> {
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn patch_bundle_database<B>(state: Arc<State>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
@ -464,9 +478,9 @@ async fn patch_bundle_database(state: Arc<State>, bundles: Vec<Bundle>) -> Resul
db
};
for bundle in bundles {
for bundle in bundles.as_ref() {
tracing::trace!("Adding '{}' to bundle database", bundle.name().display());
db.add_bundle(&bundle);
db.add_bundle(bundle);
}
{
@ -484,6 +498,29 @@ async fn patch_bundle_database(state: Arc<State>, bundles: Vec<Bundle>) -> Resul
Ok(())
}
#[tracing::instrument(skip_all, fields(bundles = bundles.as_ref().len()))]
async fn write_deployment_data<B>(state: Arc<State>, bundles: B) -> Result<()>
where
B: AsRef<[Bundle]>,
{
let info = DeploymentData {
timestamp: OffsetDateTime::now_utc(),
bundles: bundles
.as_ref()
.iter()
.map(|bundle| format!("{:x}", bundle.name().to_murmur64()))
.collect(),
};
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = serde_sjson::to_string(&info).wrap_err("failed to serizalie deployment data")?;
fs::write(&path, &data)
.await
.wrap_err_with(|| format!("failed to write deployment data to '{}'", path.display()))?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.game_dir.display(),
mods = state.mods.len()
@ -499,9 +536,45 @@ pub(crate) async fn deploy_mods(state: State) -> Result<()> {
}
}
let (game_info, deployment_info) = tokio::try_join!(
async {
tokio::task::spawn_blocking(dtmt_shared::collect_game_info)
.await
.map_err(Report::new)
},
async {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
match read_sjson_file::<_, DeploymentData>(path)
.await
{
Ok(data) => Ok(Some(data)),
Err(err) => {
if let Some(err) = err.downcast_ref::<std::io::Error>() && err.kind() == ErrorKind::NotFound {
Ok(None)
} else {
Err(err).wrap_err("failed to read deployment data")
}
}
}
}
)
.wrap_err("failed to gather deployment information")?;
let game_info = game_info.wrap_err("failed to collect Steam info")?;
tracing::debug!(?game_info, ?deployment_info);
if deployment_info
.as_ref()
.map(|i| game_info.last_updated > i.timestamp)
.unwrap_or(false)
{
eyre::bail!("Game was updated since last mod deployment. Please reset first.");
}
tracing::info!(
"Deploying {} mods to {}",
state.mods.len(),
state.mods.iter().filter(|i| i.enabled).count(),
state.game_dir.join("bundle").display()
);
@ -516,16 +589,55 @@ pub(crate) async fn deploy_mods(state: State) -> Result<()> {
.wrap_err("failed to patch boot bundle")?;
bundles.append(&mut more_bundles);
if let Some(info) = &deployment_info {
let bundle_dir = Arc::new(state.game_dir.join("bundle"));
let tasks = info
.bundles
.iter()
.cloned()
.map(|v| (v, bundle_dir.clone()))
.filter_map(|(file_name, bundle_dir)| {
let contains = bundles.iter().any(|b2| {
let name = b2.name().to_murmur64().to_string();
file_name == name
});
if !contains {
let task = async move {
let path = bundle_dir.join(&file_name);
tracing::debug!("Removing unused bundle '{}'", file_name);
if let Err(err) = fs::remove_file(&path).await.wrap_err_with(|| {
format!("failed to remove unused bundle '{}'", path.display())
}) {
tracing::error!("{:?}", err);
}
};
Some(task)
} else {
None
}
});
futures::future::join_all(tasks).await;
}
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("failed to patch game settings")?;
tracing::info!("Patching bundle database");
patch_bundle_database(state.clone(), bundles)
patch_bundle_database(state.clone(), &bundles)
.await
.wrap_err("failed to patch bundle database")?;
tracing::info!("Writing deployment data");
write_deployment_data(state.clone(), &bundles)
.await
.wrap_err("failed to write deployment data")?;
tracing::info!("Finished deploying mods");
Ok(())
}
@ -538,6 +650,40 @@ pub(crate) async fn reset_mod_deployment(state: State) -> Result<()> {
tracing::info!("Resetting mod deployment in {}", bundle_dir.display());
tracing::debug!("Reading mod deployment");
let info: DeploymentData = {
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
let data = match fs::read(&path).await {
Ok(data) => data,
Err(err) if err.kind() == ErrorKind::NotFound => {
tracing::info!("No deployment to reset");
return Ok(());
}
Err(err) => {
return Err(err).wrap_err_with(|| {
format!("failed to read deployment info at '{}'", path.display())
});
}
};
let data = String::from_utf8(data).wrap_err("invalid UTF8 in deployment data")?;
serde_sjson::from_str(&data).wrap_err("invalid SJSON in deployment data")?
};
for name in info.bundles {
let path = bundle_dir.join(name);
match fs::remove_file(&path).await {
Ok(_) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => {
tracing::error!("Failed to remove '{}': {:?}", path.display(), err);
}
};
}
for p in paths {
let path = bundle_dir.join(p);
let backup = bundle_dir.join(&format!("{}.bak", p));
@ -555,9 +701,13 @@ pub(crate) async fn reset_mod_deployment(state: State) -> Result<()> {
tracing::debug!("Deleting backup: {}", backup.display());
fs::remove_file(&backup)
.await
.wrap_err_with(|| format!("failed to remove '{}'", backup.display()))
match fs::remove_file(&backup).await {
Ok(_) => Ok(()),
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
Err(err) => {
Err(err).wrap_err_with(|| format!("failed to remove '{}'", backup.display()))
}
}
}
.await;
@ -570,6 +720,17 @@ pub(crate) async fn reset_mod_deployment(state: State) -> Result<()> {
}
}
{
let path = state.game_dir.join(DEPLOYMENT_DATA_PATH);
if let Err(err) = fs::remove_file(&path).await {
tracing::error!(
"Failed to remove deployment data '{}': {:?}",
path.display(),
err
);
}
}
tracing::info!("Reset finished");
Ok(())

View file

@ -0,0 +1,23 @@
use std::path::Path;
use color_eyre::{eyre::Context, Result};
use serde::Deserialize;
use tokio::fs;
pub mod app;
pub mod game;
pub mod worker;
#[tracing::instrument]
async fn read_sjson_file<P, T>(path: P) -> Result<T>
where
T: for<'a> Deserialize<'a>,
P: AsRef<Path> + std::fmt::Debug,
{
let path = path.as_ref();
let buf = fs::read(path)
.await
.wrap_err_with(|| format!("failed to read file '{}'", path.display()))?;
let data = String::from_utf8(buf).wrap_err("invalid UTF8")?;
serde_sjson::from_str(&data).wrap_err("failed to deserialize SJSON")
}

View file

@ -17,11 +17,7 @@ use crate::controller::app::load_mods;
use crate::controller::worker::work_thread;
use crate::state::{Delegate, State};
mod controller {
pub mod app;
pub mod game;
pub mod worker;
}
mod controller;
mod state;
mod util {
pub mod config;
@ -64,10 +60,14 @@ fn main() -> Result<()> {
let config = util::config::read_config(&default_config_path, &matches)
.wrap_err("failed to read config file")?;
let game_info = dtmt_shared::collect_game_info()?;
tracing::debug!(?config, ?game_info);
let initial_state = {
let mut state = State::new(
config.path,
config.game_dir.unwrap_or_default(),
config.game_dir.unwrap_or(game_info.path),
config.data_dir.unwrap_or_default(),
);
state.mods = load_mods(state.get_mod_dir(), config.mod_order.iter())

View file

@ -6,7 +6,9 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
color-eyre = "0.6.2"
serde = "1.0.152"
steamlocate = { path = "../../lib/steamlocate-rs", version = "*" }
time = { version = "0.3.19", features = ["formatting", "local-offset", "macros"] }
tracing = "0.1.37"
tracing-error = "0.2.0"

View file

@ -1,8 +1,13 @@
mod log;
use std::path::PathBuf;
use color_eyre::eyre;
use color_eyre::Result;
mod log;
pub use log::*;
use steamlocate::SteamDir;
use time::OffsetDateTime;
#[derive(Clone, Debug, Default, serde::Deserialize)]
pub struct ModConfigResources {
@ -26,3 +31,36 @@ pub struct ModConfig {
#[serde(default)]
pub depends: Vec<String>,
}
pub const STEAMAPP_ID: u32 = 1361210;
#[derive(Debug)]
pub struct GameInfo {
pub path: PathBuf,
pub last_updated: OffsetDateTime,
}
pub fn collect_game_info() -> Result<GameInfo> {
let mut dir = if let Some(dir) = SteamDir::locate() {
dir
} else {
eyre::bail!("Failed to locate Steam installation")
};
let found = dir
.app(&STEAMAPP_ID)
.and_then(|app| app.vdf.get("LastUpdated").map(|v| (app.path.clone(), v)));
let Some((path, last_updated)) = found else {
eyre::bail!("Failed to find game installation");
};
let Some(last_updated) = last_updated
.as_value()
.and_then(|v| v.to::<i64>())
.and_then(|v| OffsetDateTime::from_unix_timestamp(v).ok()) else {
eyre::bail!("Couldn't read 'LastUpdate'.");
};
Ok(GameInfo { path, last_updated })
}

@ -1 +1 @@
Subproject commit e94218d8f52a51529c83af33a99cc17f66caae2e
Subproject commit 81213f792767ddf1e7be60b066c87f7b137ca0a7

1
lib/steamlocate-rs Submodule

@ -0,0 +1 @@
Subproject commit 4d6898f632e20ea15d47b0b071daa4f3fa6c9574