feat(dtmm): Implement rudimentary mod deployment

This commit is contained in:
Lucas Schwiderski 2023-02-18 10:20:10 +01:00
parent cb9f154f1e
commit e65579d8aa
Signed by: lucas
GPG key ID: AA12679AAA6DF4D8
13 changed files with 660 additions and 22 deletions

4
Cargo.lock generated
View file

@ -658,12 +658,16 @@ dependencies = [
name = "dtmm"
version = "0.1.0"
dependencies = [
"bitflags",
"clap",
"color-eyre",
"confy",
"druid",
"futures",
"oodle-sys",
"sdk",
"serde",
"serde_sjson",
"tokio",
"toml",
"tracing",

View file

@ -6,13 +6,17 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bitflags = "1.3.2"
clap = { version = "4.0.15", features = ["color", "derive", "std", "cargo", "unicode"] }
color-eyre = "0.6.2"
confy = "0.5.1"
druid = { git = "https://github.com/linebender/druid.git", features = ["im"] }
futures = "0.3.25"
sdk = { path = "../../lib/sdk", version = "0.2.0" }
serde = "1.0.152"
tokio = "1.23.0"
serde = { version = "1.0.152", features = ["derive"] }
serde_sjson = { path = "../../lib/serde_sjson", version = "*" }
oodle-sys = { path = "../../lib/oodle-sys", version = "*" }
tokio = { version = "1.23.0", features = ["rt", "fs", "tracing", "sync"] }
toml = "0.5.10"
tracing = "0.1.37"
tracing-error = "0.2.0"

View file

@ -0,0 +1,47 @@
use druid::widget::{Button, Controller};
use druid::{Data, Env, Event, EventCtx, LifeCycle, LifeCycleCtx, UpdateCtx, Widget};
pub struct DisabledButtonController;
impl<T: Data> Controller<T, Button<T>> for DisabledButtonController {
fn event(
&mut self,
child: &mut Button<T>,
ctx: &mut EventCtx,
event: &Event,
data: &mut T,
env: &Env,
) {
if !ctx.is_disabled() {
ctx.set_disabled(true);
ctx.request_paint();
}
child.event(ctx, event, data, env)
}
fn lifecycle(
&mut self,
child: &mut Button<T>,
ctx: &mut LifeCycleCtx,
event: &LifeCycle,
data: &T,
env: &Env,
) {
child.lifecycle(ctx, event, data, env)
}
fn update(
&mut self,
child: &mut Button<T>,
ctx: &mut UpdateCtx,
old_data: &T,
data: &T,
env: &Env,
) {
if !ctx.is_disabled() {
ctx.set_disabled(true);
ctx.request_paint();
}
child.update(ctx, old_data, data, env)
}
}

373
crates/dtmm/src/engine.rs Normal file
View file

@ -0,0 +1,373 @@
use std::ffi::CString;
use std::io::{Cursor, ErrorKind};
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use color_eyre::eyre::Context;
use color_eyre::{eyre, Result};
use futures::stream;
use futures::StreamExt;
use sdk::filetype::lua;
use sdk::filetype::package::Package;
use sdk::murmur::Murmur64;
use sdk::{
Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary,
};
use tokio::io::AsyncWriteExt;
use tokio::{fs, try_join};
use tracing::Instrument;
use crate::state::{PackageInfo, State};
const MOD_BUNDLE_NAME: &str = "packages/mods";
const BOOT_BUNDLE_NAME: &str = "packages/boot";
const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
#[tracing::instrument]
async fn read_file_with_backup<P>(path: P) -> Result<Vec<u8>>
where
P: AsRef<Path> + std::fmt::Debug,
{
let path = path.as_ref();
let backup_path = {
let mut p = PathBuf::from(path);
let ext = if let Some(ext) = p.extension() {
ext.to_string_lossy().to_string() + ".bak"
} else {
String::from("bak")
};
p.set_extension(ext);
p
};
let file_name = path
.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| String::from("file"));
let bin = match fs::read(&backup_path).await {
Ok(bin) => bin,
Err(err) if err.kind() == ErrorKind::NotFound => {
// TODO: This doesn't need to be awaited here, yet.
// I only need to make sure it has finished before writing the changed bundle.
tracing::debug!(
"Backup does not exist. Backing up original {} to '{}'",
file_name,
backup_path.display()
);
fs::copy(path, &backup_path).await.wrap_err_with(|| {
format!(
"failed to back up {} '{}' to '{}'",
file_name,
path.display(),
backup_path.display()
)
})?;
tracing::debug!("Reading {} from original '{}'", file_name, path.display());
fs::read(path).await.wrap_err_with(|| {
format!("failed to read {} file: {}", file_name, path.display())
})?
}
Err(err) => {
return Err(err).wrap_err_with(|| {
format!(
"failed to read {} from backup '{}'",
file_name,
backup_path.display()
)
});
}
};
Ok(bin)
}
#[tracing::instrument(skip_all)]
async fn patch_game_settings(state: Arc<State>) -> Result<()> {
let settings_path = state
.get_game_dir()
.join("bundle/application_settings/settings_common.ini");
let settings = read_file_with_backup(&settings_path)
.await
.wrap_err("failed to read settings.ini")?;
let settings = String::from_utf8(settings).wrap_err("settings.ini is not valid UTF-8")?;
let mut f = fs::File::create(&settings_path)
.await
.wrap_err_with(|| format!("failed to open {}", settings_path.display()))?;
let Some(i) = settings.find("boot_script =") else {
eyre::bail!("couldn't find 'boot_script' field");
};
f.write_all(settings[0..i].as_bytes()).await?;
f.write_all(b"boot_script = \"scripts/mod_main\"").await?;
let Some(j) = settings[i..].find('\n') else {
eyre::bail!("couldn't find end of 'boot_script' field");
};
f.write_all(settings[(i + j)..].as_bytes()).await?;
tracing::info!("Patched game settings");
Ok(())
}
#[tracing::instrument(skip_all, fields(package = info.get_name()))]
fn make_package(info: &PackageInfo) -> Result<Package> {
let mut pkg = Package::new(info.get_name().clone(), PathBuf::new());
for f in info.get_files().iter() {
let mut it = f.rsplit('.');
let file_type = it
.next()
.ok_or_else(|| eyre::eyre!("missing file extension"))
.and_then(BundleFileType::from_str)
.wrap_err("invalid file name in package info")?;
let name: String = it.collect();
pkg.add_file(file_type, name);
}
Ok(pkg)
}
#[tracing::instrument(skip_all)]
async fn build_bundles(state: Arc<State>) -> Result<()> {
let mut bundle = Bundle::new(MOD_BUNDLE_NAME.into());
let mut tasks = Vec::new();
let bundle_dir = Arc::new(state.get_game_dir().join("bundle"));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let mut db = {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("failed to read bundle database")?;
let mut r = Cursor::new(bin);
let db = BundleDatabase::from_binary(&mut r).wrap_err("failed to parse bundle database")?;
tracing::trace!("Finished parsing bundle database");
db
};
for mod_info in state.get_mods() {
let span = tracing::trace_span!("building mod packages", name = mod_info.get_name());
let _enter = span.enter();
let mod_dir = state.get_mod_dir().join(mod_info.get_name());
for pkg_info in mod_info.get_packages() {
let span = tracing::trace_span!("building package", name = pkg_info.get_name());
let _enter = span.enter();
let pkg = make_package(pkg_info).wrap_err("failed to make package")?;
let mut variant = BundleFileVariant::new();
let bin = pkg
.to_binary()
.wrap_err("failed to serialize package to binary")?;
variant.set_data(bin);
let mut file = BundleFile::new(pkg_info.get_name().clone(), BundleFileType::Package);
file.add_variant(variant);
bundle.add_file(file);
let src = mod_dir.join(pkg_info.get_name());
let dest = bundle_dir.clone();
let pkg_name = pkg_info.get_name().clone();
let mod_name = mod_info.get_name().clone();
tracing::trace!(
"Adding package {} for mod {} to bundle database",
pkg_info.get_name(),
mod_info.get_name()
);
// Explicitely drop the guard, so that we can move the span
// into the async operation
drop(_enter);
let task = async move {
tracing::debug!(
"Copying bundle {} for mod {}: {} -> {}",
pkg_name,
mod_name,
src.display(),
dest.display()
);
fs::hard_link(&src, dest.as_ref()).await.wrap_err_with(|| {
format!("failed to hard link bundle {pkg_name} for mod {mod_name}")
})
}
.instrument(span);
tasks.push(task);
}
}
tracing::debug!("Copying {} mod bundles", tasks.len());
let mut tasks = stream::iter(tasks).buffer_unordered(10);
while let Some(res) = tasks.next().await {
res?;
}
db.add_bundle(&bundle);
{
let path = bundle_dir.join(format!("{:x}", Murmur64::hash(bundle.name())));
tracing::trace!("Writing mod bundle to '{}'", path.display());
fs::write(&path, bundle.to_binary()?)
.await
.wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?;
}
{
tracing::trace!("Writing bundle database to '{}'", database_path.display());
let bin = db
.to_binary()
.wrap_err("failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})?;
}
Ok(())
}
#[tracing::instrument(skip_all)]
async fn patch_boot_bundle(state: Arc<State>) -> Result<()> {
let bundle_dir = Arc::new(state.get_game_dir().join("bundle"));
let bundle_path = bundle_dir.join(format!("{:x}", Murmur64::hash(BOOT_BUNDLE_NAME.as_bytes())));
let database_path = bundle_dir.join(BUNDLE_DATABASE_NAME);
let (mut db, mut bundle) = try_join!(
async {
let bin = read_file_with_backup(&database_path)
.await
.wrap_err("failed to read bundle database")?;
let mut r = Cursor::new(bin);
BundleDatabase::from_binary(&mut r).wrap_err("failed to parse bundle database")
}
.instrument(tracing::trace_span!("read bundle database")),
async {
let bin = read_file_with_backup(&bundle_path)
.await
.wrap_err("failed to read boot bundle")?;
Bundle::from_binary(&state.get_ctx(), BOOT_BUNDLE_NAME.to_string(), bin)
.wrap_err("failed to parse boot bundle")
}
.instrument(tracing::trace_span!("read boot bundle"))
)?;
{
tracing::trace!("Adding mod package file to boot bundle");
let span = tracing::trace_span!("create mod package file");
let _enter = span.enter();
let mut pkg = Package::new(MOD_BUNDLE_NAME.to_string(), PathBuf::new());
for mod_info in state.get_mods() {
for pkg_info in mod_info.get_packages() {
pkg.add_file(BundleFileType::Package, pkg_info.get_name());
}
}
let mut variant = BundleFileVariant::new();
variant.set_data(pkg.to_binary()?);
let mut f = BundleFile::new(MOD_BUNDLE_NAME.to_string(), BundleFileType::Package);
f.add_variant(variant);
bundle.add_file(f);
}
{
tracing::trace!("Adding main mod Lua file to boot bundle");
let span = tracing::trace_span!("create mod boot script file");
let _enter = span.enter();
// TODO: Build actual boot script
let lua = CString::new(
r#"
print("dtmm says hello!")
require("scripts/main")
"#,
)
.expect("invalid C string");
let f = lua::compile(MOD_BOOT_SCRIPT.to_string(), &lua)
.wrap_err("failed to compile mod boot script")?;
// TODO:
bundle.add_file(f);
}
db.add_bundle(&bundle);
try_join!(
async {
let bin = bundle
.to_binary()
.wrap_err("failed to serialize boot bundle")?;
fs::write(&bundle_path, bin)
.await
.wrap_err_with(|| format!("failed to write main bundle: {}", bundle_path.display()))
}
.instrument(tracing::trace_span!("write boot bundle")),
async {
let bin = db
.to_binary()
.wrap_err("failed to serialize bundle database")?;
fs::write(&database_path, bin).await.wrap_err_with(|| {
format!(
"failed to write bundle database to '{}'",
database_path.display()
)
})
}
.instrument(tracing::trace_span!("write bundle database"))
)?;
Ok(())
}
#[tracing::instrument(skip_all, fields(
game_dir = %state.get_game_dir().display(),
mods = state.get_mods().len()
))]
pub(crate) async fn deploy_mods(state: State) -> Result<()> {
let state = Arc::new(state);
tracing::info!(
"Deploying {} mods to {}",
state.get_mods().len(),
state.get_game_dir().join("bundle").display()
);
tracing::info!("Build mod bundles");
build_bundles(state.clone())
.await
.wrap_err("failed to build mod bundles")?;
tracing::info!("Patch boot bundle");
patch_boot_bundle(state.clone())
.await
.wrap_err("failed to patch boot bundle")?;
tracing::info!("Patch game settings");
patch_game_settings(state.clone())
.await
.wrap_err("failed to patch game settings")?;
// TODO: Build mod order data
// TODO: Handle DMF
tracing::info!("Finished deploying mods");
Ok(())
}

View file

@ -1,24 +1,73 @@
#![recursion_limit = "256"]
#![feature(let_chains)]
use std::sync::Arc;
use clap::command;
use clap::Arg;
use color_eyre::Report;
use color_eyre::Result;
use druid::AppLauncher;
use druid::ExtEventSink;
use druid::Target;
use tokio::runtime::Runtime;
use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::RwLock;
use tracing_error::ErrorLayer;
use tracing_subscriber::prelude::*;
use tracing_subscriber::EnvFilter;
use crate::state::State;
use crate::engine::deploy_mods;
use crate::state::{AsyncAction, Delegate, State, COMMAND_FINISH_DEPLOY};
mod controller;
mod engine;
mod main_window;
mod state;
mod theme;
mod widget;
fn work_thread(
event_sink: Arc<RwLock<ExtEventSink>>,
action_queue: Arc<RwLock<UnboundedReceiver<AsyncAction>>>,
) -> Result<()> {
let rt = Runtime::new()?;
rt.block_on(async {
while let Some(action) = action_queue.write().await.recv().await {
let event_sink = event_sink.clone();
match action {
AsyncAction::DeployMods(state) => tokio::spawn(async move {
if let Err(err) = deploy_mods(state).await {
tracing::error!("Failed to deploy mods: {:?}", err);
}
event_sink
.write()
.await
.submit_command(COMMAND_FINISH_DEPLOY, (), Target::Auto)
.expect("failed to send command");
}),
};
}
});
Ok(())
}
#[tracing::instrument]
#[tokio::main]
async fn main() -> Result<()> {
color_eyre::install()?;
let _matches = command!().get_matches();
let matches = command!()
.arg(Arg::new("oodle").long("oodle").help(
"The oodle library to load. This may either be:\n\
- A library name that will be searched for in the system's default paths.\n\
- A file path relative to the current working directory.\n\
- An absolute file path.",
))
.get_matches();
{
let filter_layer =
@ -47,9 +96,27 @@ async fn main() -> Result<()> {
}
}
unsafe {
oodle_sys::init(matches.get_one::<String>("oodle"));
}
let initial_state = State::new();
AppLauncher::with_window(main_window::new())
.launch(initial_state)
.map_err(Report::new)
let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
let delegate = Delegate::new(sender);
let launcher = AppLauncher::with_window(main_window::new()).delegate(delegate);
let event_sink = launcher.get_external_handle();
std::thread::spawn(move || {
let event_sink = Arc::new(RwLock::new(event_sink));
let receiver = Arc::new(RwLock::new(receiver));
loop {
if let Err(err) = work_thread(event_sink.clone(), receiver.clone()) {
tracing::error!("Work thread failed, restarting: {:?}", err);
}
}
});
launcher.launch(initial_state).map_err(Report::new)
}

View file

@ -7,7 +7,7 @@ use druid::{lens, Insets, LensExt, Widget, WidgetExt, WindowDesc};
use crate::state::{
ModInfo, PathBufFormatter, State, StateController, View, ACTION_DELETE_SELECTED_MOD,
ACTION_SELECTED_MOD_DOWN, ACTION_SELECTED_MOD_UP, ACTION_SELECT_MOD,
ACTION_SELECTED_MOD_DOWN, ACTION_SELECTED_MOD_UP, ACTION_SELECT_MOD, COMMAND_START_DEPLOY,
};
use crate::theme;
use crate::widget::ExtraWidgetExt;
@ -48,11 +48,13 @@ fn build_top_bar() -> impl Widget<State> {
)
.with_child(
Flex::row()
.with_child(Button::new("Deploy Mods").on_click(
|_ctx, _state: &mut State, _env| {
todo!();
},
))
.with_child(
Button::new("Deploy Mods")
.on_click(|ctx, _state: &mut State, _env| {
ctx.submit_command(COMMAND_START_DEPLOY);
})
.disabled_if(|data, _| !data.can_deploy_mods()),
)
.with_default_spacer()
.with_child(
Button::new("Run Game").on_click(|_ctx, _state: &mut State, _env| {

View file

@ -5,7 +5,7 @@ use druid::im::Vector;
use druid::text::Formatter;
use druid::widget::Controller;
use druid::{
AppDelegate, Data, DelegateCtx, Env, Event, EventCtx, Handled, Lens, Selector, Target,
AppDelegate, Command, Data, DelegateCtx, Env, Event, EventCtx, Handled, Lens, Selector, Target,
Widget,
};
use tokio::sync::mpsc::UnboundedSender;
@ -15,6 +15,9 @@ pub const ACTION_SELECTED_MOD_UP: Selector = Selector::new("dtmm.action.selected
pub const ACTION_SELECTED_MOD_DOWN: Selector = Selector::new("dtmm.action.selected-mod-down");
pub const ACTION_DELETE_SELECTED_MOD: Selector = Selector::new("dtmm.action.delete-selected-mod");
pub const COMMAND_FINISH_DEPLOY: Selector = Selector::new("dtmm.command.finish-deploy");
pub const COMMAND_START_DEPLOY: Selector = Selector::new("dtmm.command.start-deploy");
#[derive(Copy, Clone, Data, Debug, PartialEq)]
pub(crate) enum View {
Mods,
@ -147,6 +150,10 @@ impl State {
self.selected_mod_index.map(|i| i > 0).unwrap_or(false)
}
pub fn can_deploy_mods(&self) -> bool {
!self.is_deployment_in_progress
}
pub(crate) fn get_game_dir(&self) -> &PathBuf {
&self.game_dir
}
@ -298,6 +305,52 @@ impl<W: Widget<State>> Controller<State, W> for StateController {
}
}
pub(crate) enum AsyncAction {
DeployMods(State),
}
pub(crate) struct Delegate {
sender: UnboundedSender<AsyncAction>,
}
impl Delegate {
pub fn new(sender: UnboundedSender<AsyncAction>) -> Self {
Self { sender }
}
}
impl AppDelegate<State> for Delegate {
#[tracing::instrument(name = "Delegate", skip_all)]
fn command(
&mut self,
_ctx: &mut DelegateCtx,
_target: Target,
cmd: &Command,
data: &mut State,
_env: &Env,
) -> Handled {
if cmd.is(COMMAND_START_DEPLOY) {
if self
.sender
.send(AsyncAction::DeployMods(data.clone()))
.is_ok()
{
data.is_deployment_in_progress = true;
} else {
tracing::error!("Failed to queue action to deploy mods");
}
Handled::Yes
} else if cmd.is(COMMAND_FINISH_DEPLOY) {
data.is_deployment_in_progress = false;
Handled::Yes
} else {
tracing::debug!("Unknown command: {:?}", cmd);
Handled::No
}
}
}
pub(crate) struct PathBufFormatter;
impl PathBufFormatter {

View file

@ -0,0 +1,73 @@
use druid::widget::{Controller, Flex};
use druid::{Data, Widget};
pub struct TableSelect<T> {
widget: Flex<T>,
controller: TableSelectController<T>,
}
impl<T: Data> TableSelect<T> {
pub fn new(values: impl IntoIterator<Item = (impl Widget<T> + 'static)>) -> Self {
todo!();
}
}
impl<T: Data> Widget<T> for TableSelect<T> {
fn event(
&mut self,
ctx: &mut druid::EventCtx,
event: &druid::Event,
data: &mut T,
env: &druid::Env,
) {
todo!()
}
fn lifecycle(
&mut self,
ctx: &mut druid::LifeCycleCtx,
event: &druid::LifeCycle,
data: &T,
env: &druid::Env,
) {
todo!()
}
fn update(&mut self, ctx: &mut druid::UpdateCtx, old_data: &T, data: &T, env: &druid::Env) {
todo!()
}
fn layout(
&mut self,
ctx: &mut druid::LayoutCtx,
bc: &druid::BoxConstraints,
data: &T,
env: &druid::Env,
) -> druid::Size {
todo!()
}
fn paint(&mut self, ctx: &mut druid::PaintCtx, data: &T, env: &druid::Env) {
todo!()
}
}
struct TableSelectController<T> {
inner: T,
}
impl<T: Data> TableSelectController<T> {}
impl<T: Data> Controller<T, Flex<T>> for TableSelectController<T> {}
pub struct TableItem<T> {
inner: dyn Widget<T>,
}
impl<T: Data> TableItem<T> {
pub fn new(inner: impl Widget<T>) -> Self {
todo!();
}
}
impl<T: Data> Widget<T> for TableItem<T> {}

View file

@ -50,14 +50,15 @@ impl BundleDatabase {
self.stored_files.entry(hash).or_default().push(file);
// TODO: Resource hashes
for f in bundle.files() {
let file_name = FileName {
extension: f.file_type(),
name: Murmur64::hash(f.name(false, None).as_bytes()),
};
// TODO: Compute actual resource hash
self.resource_hashes.insert(hash, 0);
self.bundle_contents
.entry(hash)
.or_default()

View file

@ -12,8 +12,6 @@ use crate::binary::sync::*;
use crate::filetype::*;
use crate::murmur::{HashGroup, IdString64, Murmur64};
use super::EntryHeader;
#[derive(Debug, Hash, PartialEq, Eq, Copy, Clone)]
pub enum BundleFileType {
Animation,
@ -801,6 +799,12 @@ impl BundleFile {
}
}
impl PartialEq for BundleFile {
fn eq(&self, other: &Self) -> bool {
self.name == other.name && self.file_type == other.file_type
}
}
pub struct UserFile {
// TODO: Might be able to avoid some allocations with a Cow here
data: Vec<u8>,

View file

@ -91,10 +91,6 @@ impl Bundle {
}
}
pub fn get_file<S: AsRef<str>>(&self, name: S) -> Option<&BundleFile> {
self.files.iter().find(|f| f.base_name().eq(name.as_ref()))
}
#[tracing::instrument(skip(ctx, binary), fields(len_binary = binary.as_ref().len()))]
pub fn from_binary<B>(ctx: &crate::Context, name: String, binary: B) -> Result<Self>
where

View file

@ -104,6 +104,14 @@ impl DerefMut for Package {
}
impl Package {
pub fn new(name: String, root: PathBuf) -> Self {
Self {
_name: name,
_root: root,
inner: Default::default(),
}
}
fn len(&self) -> usize {
self.values().fold(0, |total, files| total + files.len())
}

View file

@ -66,6 +66,12 @@ impl fmt::UpperHex for Murmur64 {
}
}
impl fmt::LowerHex for Murmur64 {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::LowerHex::fmt(&self.0, f)
}
}
impl fmt::Display for Murmur64 {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::UpperHex::fmt(&self.0, f)