From 8cf08e07381cf1342d8c64775d9e00a996a30815 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 9 Mar 2023 21:29:18 +0100 Subject: [PATCH] feat(dtmt): Implement watch command Closes #61. --- Cargo.lock | 110 +++++++++++++++ crates/dtmt/Cargo.toml | 1 + crates/dtmt/src/cmd/build.rs | 59 +++++--- crates/dtmt/src/cmd/package.rs | 76 ++++++----- crates/dtmt/src/cmd/watch.rs | 238 +++++++++++++++++++++++++++++++-- crates/dtmt/src/main.rs | 2 +- 6 files changed, 422 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a46c278..d1b4bce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -525,6 +525,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.15" @@ -749,6 +759,7 @@ dependencies = [ "glob", "libloading", "nanorand", + "notify", "oodle-sys", "path-clean", "path-slash", @@ -878,6 +889,18 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.45.0", +] + [[package]] name = "flate2" version = "1.0.25" @@ -949,6 +972,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.3.26" @@ -1358,6 +1390,26 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "instant" version = "0.1.12" @@ -1461,6 +1513,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "kqueue" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8fc60ba15bf51257aa9807a48a61013db043fcf3a78cb0d916e8e396dcad98" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587" +dependencies = [ + "bitflags", + "libc", +] + [[package]] name = "kurbo" version = "0.9.1" @@ -1649,6 +1721,24 @@ dependencies = [ "nom 7.1.3", ] +[[package]] +name = "notify" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ea850aa68a06e48fdb069c0ec44d0d64c8dbffa49bf3b6f7f0a901fdea1ba9" +dependencies = [ + "bitflags", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "mio", + "walkdir", + "windows-sys 0.42.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2277,6 +2367,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2920,6 +3019,17 @@ dependencies = [ "quote", ] +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 6ed04de..f89c27c 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -31,6 +31,7 @@ zip = "0.6.3" path-clean = "1.0.1" path-slash = "0.2.1" async-recursion = "1.0.2" +notify = "5.1.0" [dev-dependencies] tempfile = "3.3.0" diff --git a/crates/dtmt/src/cmd/build.rs b/crates/dtmt/src/cmd/build.rs index b8f23a8..9bdddf7 100644 --- a/crates/dtmt/src/cmd/build.rs +++ b/crates/dtmt/src/cmd/build.rs @@ -260,26 +260,16 @@ pub(crate) async fn read_project_config(dir: Option) -> Result Result<()> { - unsafe { - oodle_sys::init(matches.get_one::("oodle")); - } - - let cfg = read_project_config(matches.get_one::("directory").cloned()).await?; - - let game_dir = matches - .get_one::("deploy") - .map(|p| p.join("bundle")); - - let out_path = matches - .get_one::("out") - .expect("parameter should have default value"); - - tracing::debug!(?cfg, ?game_dir, ?out_path); - - let game_dir = Arc::new(game_dir); - let cfg = Arc::new(cfg); +pub(crate) async fn build( + cfg: &ModConfig, + out_path: P1, + game_dir: Arc>, +) -> Result<()> +where + P1: AsRef, + P2: AsRef, +{ + let out_path = out_path.as_ref(); fs::create_dir_all(out_path) .await @@ -340,7 +330,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> .wrap_err_with(|| format!("Failed to write bundle to '{}'", path.display()))?; if let Some(game_dir) = game_dir.as_ref() { - let path = game_dir.join(&name); + let path = game_dir.as_ref().join(&name); tracing::trace!( "Deploying bundle {} to '{}'", @@ -393,8 +383,33 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> tracing::info!("Compiled bundles written to '{}'", out_path.display()); if let Some(game_dir) = game_dir.as_ref() { - tracing::info!("Deployed bundles to '{}'", game_dir.display()); + tracing::info!("Deployed bundles to '{}'", game_dir.as_ref().display()); } Ok(()) } + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + unsafe { + oodle_sys::init(matches.get_one::("oodle")); + } + + let cfg = read_project_config(matches.get_one::("directory").cloned()).await?; + + let game_dir = matches + .get_one::("deploy") + .map(|p| p.join("bundle")); + + let out_path = matches + .get_one::("out") + .expect("parameter should have default value"); + + tracing::debug!(?cfg, ?game_dir, ?out_path); + + let game_dir = Arc::new(game_dir); + + build(&cfg, out_path, game_dir).await?; + + Ok(()) +} diff --git a/crates/dtmt/src/cmd/package.rs b/crates/dtmt/src/cmd/package.rs index f4d990e..e922b6e 100644 --- a/crates/dtmt/src/cmd/package.rs +++ b/crates/dtmt/src/cmd/package.rs @@ -1,11 +1,12 @@ use std::io::{Cursor, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::{Context, Result}; use color_eyre::Help; -use path_slash::PathBufExt; +use dtmt_shared::ModConfig; +use path_slash::{PathBufExt, PathExt}; use tokio::fs; use tokio::sync::Mutex; use tokio_stream::wrappers::ReadDirStream; @@ -41,18 +42,28 @@ pub(crate) fn command_definition() -> Command { Arg::new("out") .long("out") .short('o') - .default_value(".") .value_parser(value_parser!(PathBuf)) - .help("The path to write the packaged file to. May be a directory or a file name."), + .help( + "The path to write the packaged file to. Will default to a file in the \ + current working directory", + ), ) } #[async_recursion::async_recursion] -async fn process_directory( +async fn process_directory( zip: Arc>>, - path: PathBuf, - prefix: PathBuf, -) -> Result<()> { + path: P1, + prefix: P2, +) -> Result<()> +where + P1: AsRef + std::marker::Send, + P2: AsRef + std::marker::Send, + W: std::io::Write + std::io::Seek + std::marker::Send, +{ + let path = path.as_ref(); + let prefix = prefix.as_ref(); + zip.lock() .await .add_directory(prefix.to_slash_lossy(), Default::default())?; @@ -88,33 +99,18 @@ async fn process_directory Result<()> { - let cfg = read_project_config(matches.get_one::("project").cloned()).await?; - - let dest = { - let mut path = matches - .get_one::("out") - .cloned() - .unwrap_or_else(|| PathBuf::from(".")); - - if path.extension().is_none() { - path.push(format!("{}.zip", cfg.id)) - } - - path - }; +pub(crate) async fn package(cfg: &ModConfig, path: P1, dest: P2) -> Result<()> +where + P1: AsRef, + P2: AsRef, +{ + let path = path.as_ref(); + let dest = dest.as_ref(); let data = Cursor::new(Vec::new()); let zip = ZipWriter::new(data); let zip = Arc::new(Mutex::new(zip)); - let path = cfg.dir.join( - matches - .get_one::("directory") - .expect("parameter has default value"), - ); - process_directory(zip.clone(), path, PathBuf::from(&cfg.id)) .await .wrap_err("Failed to add directory to archive")?; @@ -135,7 +131,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> let data = zip.finish()?; - fs::write(&dest, data.into_inner()) + fs::write(dest, data.into_inner()) .await .wrap_err_with(|| format!("Failed to write mod archive to '{}'", dest.display())) .with_suggestion(|| "Make sure that parent directories exist.".to_string())?; @@ -143,3 +139,21 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> tracing::info!("Mod archive written to {}", dest.display()); Ok(()) } + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + let cfg = read_project_config(matches.get_one::("project").cloned()).await?; + + let dest = matches + .get_one::("out") + .map(path_clean::clean) + .unwrap_or_else(|| PathBuf::from(format!("{}.zip", cfg.id))); + + let path = cfg.dir.join( + matches + .get_one::("directory") + .expect("parameter has default value"), + ); + + package(&cfg, path, dest).await +} diff --git a/crates/dtmt/src/cmd/watch.rs b/crates/dtmt/src/cmd/watch.rs index 508cef9..54e2274 100644 --- a/crates/dtmt/src/cmd/watch.rs +++ b/crates/dtmt/src/cmd/watch.rs @@ -1,24 +1,242 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; -use clap::{value_parser, Arg, ArgMatches, Command}; -use color_eyre::eyre::Result; +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; +use color_eyre::eyre::{Context, Result}; +use dtmt_shared::ModConfig; +use notify::{Event, Watcher}; -pub(crate) fn _command_definition() -> Command { +use crate::cmd::build::{build, read_project_config}; + +use super::package::package; + +pub(crate) fn command_definition() -> Command { Command::new("watch") - .about("Re-build the given directory on file changes.") + .about("Watch for file system changes and re-build the mod archive.") + .arg( + Arg::new("debounce") + .default_value("150") + .value_parser(value_parser!(u64)) + .help( + "The delay to debounce events by. This avoids continously \ + rebuilding on rapid file changes, such as version control checkouts.", + ), + ) .arg( Arg::new("directory") .required(false) - .default_value(".") .value_parser(value_parser!(PathBuf)) .help( "The path to the project to build. \ - If omitted, the current working directory is used.", + If omitted, the current working directory is used.", + ), + ) + .arg(Arg::new("oodle").long("oodle").help( + "The oodle library to load. This may either be:\n\ + - A library name that will be searched for in the system's default paths.\n\ + - A file path relative to the current working directory.\n\ + - An absolute file path.", + )) + .arg( + Arg::new("out") + .long("out") + .short('o') + .default_value("out") + .value_parser(value_parser!(PathBuf)) + .help("The directory to write output files to."), + ) + .arg( + Arg::new("deploy") + .long("deploy") + .short('d') + .value_parser(value_parser!(PathBuf)) + .help( + "If the path to the game (without the trailing '/bundle') is specified, \ + deploy the newly built bundles. \ + This will not adjust the bundle database or package files, so if files are \ + added or removed, you will have to import into DTMM and re-deploy there.", + ), + ) + .arg( + Arg::new("archive") + .long("archive") + .short('a') + .value_parser(value_parser!(PathBuf)) + .help( + "The path to write the packaged file to. Will default to a file in the \ + current working directory", + ), + ) + .arg( + Arg::new("ignore") + .long("ignore") + .short('i') + .value_parser(value_parser!(PathBuf)) + .action(ArgAction::Append) + .help( + "A directory or file path to ignore. May be specified multiple times. \ + The values of 'out' and 'archive' are ignored automatically.", ), ) } -#[tracing::instrument(skip_all)] -pub(crate) async fn run(_ctx: sdk::Context, _matches: &ArgMatches) -> Result<()> { - unimplemented!() +async fn compile( + cfg: &ModConfig, + out_path: P1, + archive_path: P2, + game_dir: Arc>, +) -> Result<()> +where + P1: AsRef + std::marker::Copy, + P2: AsRef, + P3: AsRef, +{ + build(cfg, out_path, game_dir) + .await + .wrap_err("Failed to build bundles")?; + package(cfg, out_path, archive_path) + .await + .wrap_err("Failed to package bundles") +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + unsafe { + oodle_sys::init(matches.get_one::("oodle")); + } + + let cfg = read_project_config(matches.get_one::("directory").cloned()) + .await + .wrap_err("failed to load project config")?; + tracing::debug!(?cfg); + let cfg = Arc::new(cfg); + + let game_dir = matches + .get_one::("deploy") + .map(path_clean::clean) + .map(|p| if p.is_absolute() { p } else { cfg.dir.join(p) }) + .map(|p| p.join("bundle")); + + let out_path = matches + .get_one::("out") + .map(path_clean::clean) + .map(|p| if p.is_absolute() { p } else { cfg.dir.join(p) }) + .expect("parameter should have default value"); + + let archive_path = matches + .get_one::("archive") + .map(path_clean::clean) + .map(|p| if p.is_absolute() { p } else { cfg.dir.join(p) }) + .unwrap_or_else(|| cfg.dir.join(format!("{}.zip", cfg.id))); + + let ignored = { + let mut ignored: Vec<_> = matches + .get_many::("ignore") + .unwrap_or_default() + .map(path_clean::clean) + .map(|p| if p.is_absolute() { p } else { cfg.dir.join(p) }) + .collect(); + + ignored.push(out_path.clone()); + ignored.push(archive_path.clone()); + + ignored + }; + + if tracing::enabled!(tracing::Level::INFO) { + let list = ignored.iter().fold(String::new(), |mut s, p| { + s.push_str("\n - "); + s.push_str(&p.display().to_string()); + s + }); + + tracing::info!("Ignoring:{}", list); + } + + let game_dir = Arc::new(game_dir); + + let duration = + Duration::from_millis(matches.get_one::("debounce").copied().unwrap_or(150)); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel(); + + let mut watcher = notify::recommended_watcher(move |res: Result| { + let ignored = match &res { + Ok(evt) => evt.paths.iter().any(|p1| { + let p1 = path_clean::clean(p1); + ignored.iter().any(|p2| p1.starts_with(p2)) + }), + Err(_) => false, + }; + + tracing::trace!(?res, ignored, "Received file system event"); + + if !ignored { + if let Err(err) = tx.send(res) { + tracing::error!("Failed to send file system event: {:?}", err); + } + } + }) + .wrap_err("failed to create file system watcher")?; + + tracing::info!("Starting file watcher on '{}'", cfg.dir.display()); + + let path = cfg.dir.clone(); + watcher + .watch(&path, notify::RecursiveMode::Recursive) + .wrap_err_with(|| { + format!( + "failed to watch directory for file changes: {}", + path.display() + ) + })?; + + tracing::trace!("Starting debounce loop"); + + let mut dirty = false; + loop { + // While we could just always await on the timeout, splitting things like this + // optimizes the case when no events happen for a while. Rather than being woken every + // `duration` just to do nothing, this way we always wait for a new event first until + // we start the debounce timeouts. + if dirty { + match tokio::time::timeout(duration, rx.recv()).await { + // The error is the wanted case, as it signals that we haven't received an + // event within `duration`, which es what the debounce is supposed to wait for. + Err(_) => { + tracing::trace!("Received debounce timeout, running build"); + if let Err(err) = + compile(&cfg, &out_path, &archive_path, game_dir.clone()).await + { + tracing::error!("Failed to build mod archive: {:?}", err); + } + dirty = false; + } + Ok(None) => { + break; + } + // We received a value before the timeout, so we reset it + Ok(_) => { + tracing::trace!("Received value before timeout, resetting"); + } + } + } else { + match rx.recv().await { + Some(_) => { + tracing::trace!("Received event, starting debounce"); + dirty = true; + } + None => { + break; + } + } + } + } + + tracing::trace!("Event channel closed"); + if let Err(err) = compile(&cfg, &out_path, &archive_path, game_dir.clone()).await { + tracing::error!("Failed to build mod archive: {:?}", err); + } + + Ok(()) } diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index 8c7b40f..3e1fba2 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -55,7 +55,7 @@ async fn main() -> Result<()> { .subcommand(cmd::murmur::command_definition()) .subcommand(cmd::new::command_definition()) .subcommand(cmd::package::command_definition()) - // .subcommand(cmd::watch::command_definition()) + .subcommand(cmd::watch::command_definition()) .get_matches(); dtmt_shared::create_tracing_subscriber(); -- 2.45.3