From 22ea4d75a1468e611318c8f1b087000c4e1fdb7f Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 2 Mar 2023 08:52:49 +0100 Subject: [PATCH 1/4] fix: Remove obsolete CLI parameter With Oodle being linked statically, there is no need for a parameter to speficy the library location. --- crates/dtmm/src/main.rs | 6 ------ crates/dtmt/src/cmd/build.rs | 24 ++++++++---------------- crates/dtmt/src/cmd/bundle/mod.rs | 8 +------- 3 files changed, 9 insertions(+), 29 deletions(-) diff --git a/crates/dtmm/src/main.rs b/crates/dtmm/src/main.rs index 9ce6192..a89dde5 100644 --- a/crates/dtmm/src/main.rs +++ b/crates/dtmm/src/main.rs @@ -37,12 +37,6 @@ fn main() -> Result<()> { tracing::trace!(default_config_path = %default_config_path.display()); let matches = command!() - .arg(Arg::new("oodle").long("oodle").help( - "The oodle library to load. This may either be:\n\ - - A library name that will be searched for in the system's default paths.\n\ - - A file path relative to the current working directory.\n\ - - An absolute file path.", - )) .arg( Arg::new("config") .long("config") diff --git a/crates/dtmt/src/cmd/build.rs b/crates/dtmt/src/cmd/build.rs index f077094..97f192c 100644 --- a/crates/dtmt/src/cmd/build.rs +++ b/crates/dtmt/src/cmd/build.rs @@ -17,23 +17,15 @@ use crate::mods::archive::Archive; const PROJECT_CONFIG_NAME: &str = "dtmt.cfg"; pub(crate) fn command_definition() -> Command { - Command::new("build") - .about("Build a project") - .arg( - Arg::new("directory") - .required(false) - .value_parser(value_parser!(PathBuf)) - .help( - "The path to the project to build. \ + Command::new("build").about("Build a project").arg( + Arg::new("directory") + .required(false) + .value_parser(value_parser!(PathBuf)) + .help( + "The path to the project to build. \ If omitted, dtmt will search from the current working directory upward.", - ), - ) - .arg(Arg::new("oodle").long("oodle").help( - "The oodle library to load. This may either be:\n\ - - A library name that will be searched for in the system's default paths.\n\ - - A file path relative to the current working directory.\n\ - - An absolute file path.", - )) + ), + ) } #[tracing::instrument] diff --git a/crates/dtmt/src/cmd/bundle/mod.rs b/crates/dtmt/src/cmd/bundle/mod.rs index 6baf860..0e7c9f7 100644 --- a/crates/dtmt/src/cmd/bundle/mod.rs +++ b/crates/dtmt/src/cmd/bundle/mod.rs @@ -1,4 +1,4 @@ -use clap::{Arg, ArgMatches, Command}; +use clap::{ArgMatches, Command}; use color_eyre::eyre::Result; mod decompress; @@ -10,12 +10,6 @@ pub(crate) fn command_definition() -> Command { Command::new("bundle") .subcommand_required(true) .about("Manipulate the game's bundle files") - .arg(Arg::new("oodle").long("oodle").help( - "The oodle library to load. This may either be:\n\ - - A library name that will be searched for in the system's default paths.\n\ - - A file path relative to the current working directory.\n\ - - An absolute file path.", - )) .subcommand(decompress::command_definition()) .subcommand(extract::command_definition()) .subcommand(inject::command_definition()) -- 2.45.3 From 5df3aa1cb7985421feabbc98793b0786c2d6424f Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 2 Mar 2023 10:55:23 +0100 Subject: [PATCH 2/4] feat(dtmt): Split build command Closes #40. --- crates/dtmt/src/cmd/build.rs | 197 ++++++++++++++++++-------------- crates/dtmt/src/cmd/package.rs | 128 +++++++++++++++++++++ crates/dtmt/src/main.rs | 11 +- crates/dtmt/src/mods/archive.rs | 98 ---------------- 4 files changed, 245 insertions(+), 189 deletions(-) create mode 100644 crates/dtmt/src/cmd/package.rs delete mode 100644 crates/dtmt/src/mods/archive.rs diff --git a/crates/dtmt/src/cmd/build.rs b/crates/dtmt/src/cmd/build.rs index 97f192c..a0db3f2 100644 --- a/crates/dtmt/src/cmd/build.rs +++ b/crates/dtmt/src/cmd/build.rs @@ -1,3 +1,5 @@ +use std::collections::{HashMap, HashSet}; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -8,24 +10,36 @@ use dtmt_shared::ModConfig; use futures::future::try_join_all; use futures::StreamExt; use sdk::filetype::package::Package; +use sdk::murmur::IdString64; use sdk::{Bundle, BundleFile}; use tokio::fs::{self, File}; use tokio::io::AsyncReadExt; - -use crate::mods::archive::Archive; +use tokio::sync::Mutex; const PROJECT_CONFIG_NAME: &str = "dtmt.cfg"; +type FileIndexMap = HashMap>; + pub(crate) fn command_definition() -> Command { - Command::new("build").about("Build a project").arg( - Arg::new("directory") - .required(false) - .value_parser(value_parser!(PathBuf)) - .help( - "The path to the project to build. \ + Command::new("build") + .about("Build a project") + .arg( + Arg::new("directory") + .required(false) + .value_parser(value_parser!(PathBuf)) + .help( + "The path to the project to build. \ If omitted, dtmt will search from the current working directory upward.", - ), - ) + ), + ) + .arg( + Arg::new("out") + .long("out") + .short('o') + .default_value("out") + .value_parser(value_parser!(PathBuf)) + .help("The directory to write output files to."), + ) } #[tracing::instrument] @@ -173,74 +187,79 @@ fn normalize_file_path>(path: P) -> Result { Ok(path) } -#[tracing::instrument(skip_all)] -pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { - let cfg = { - let dir = matches.get_one::("directory").cloned(); - let mut cfg = find_project_config(dir).await?; +#[tracing::instrument] +pub(crate) async fn read_project_config(dir: Option) -> Result { + let mut cfg = find_project_config(dir).await?; - cfg.resources.init = normalize_file_path(cfg.resources.init) - .wrap_err("invalid config field 'resources.init'") + cfg.resources.init = normalize_file_path(cfg.resources.init) + .wrap_err("invalid config field 'resources.init'") + .with_suggestion(|| { + "Specify a file path relative to and child path of the \ + directory where 'dtmt.cfg' is." + .to_string() + }) + .with_suggestion(|| { + "Use 'dtmt new' in a separate directory to generate \ + a valid mod template." + .to_string() + })?; + + if let Some(path) = cfg.resources.data { + let path = normalize_file_path(path) + .wrap_err("invalid config field 'resources.data'") .with_suggestion(|| { "Specify a file path relative to and child path of the \ - directory where 'dtmt.cfg' is." + directory where 'dtmt.cfg' is." .to_string() }) .with_suggestion(|| { "Use 'dtmt new' in a separate directory to generate \ - a valid mod template." + a valid mod template." .to_string() })?; + cfg.resources.data = Some(path); + } - if let Some(path) = cfg.resources.data { - let path = normalize_file_path(path) - .wrap_err("invalid config field 'resources.data'") - .with_suggestion(|| { - "Specify a file path relative to and child path of the \ - directory where 'dtmt.cfg' is." - .to_string() - }) - .with_suggestion(|| { - "Use 'dtmt new' in a separate directory to generate \ - a valid mod template." - .to_string() - })?; - cfg.resources.data = Some(path); - } - - if let Some(path) = cfg.resources.localization { - let path = normalize_file_path(path) - .wrap_err("invalid config field 'resources.localization'") - .with_suggestion(|| { - "Specify a file path relative to and child path of the \ + if let Some(path) = cfg.resources.localization { + let path = normalize_file_path(path) + .wrap_err("invalid config field 'resources.localization'") + .with_suggestion(|| { + "Specify a file path relative to and child path of the \ directory where 'dtmt.cfg' is." - .to_string() - }) - .with_suggestion(|| { - "Use 'dtmt new' in a separate directory to generate \ + .to_string() + }) + .with_suggestion(|| { + "Use 'dtmt new' in a separate directory to generate \ a valid mod template." - .to_string() - })?; - cfg.resources.localization = Some(path); - } + .to_string() + })?; + cfg.resources.localization = Some(path); + } - cfg - }; + Ok(cfg) +} - let dest = { - let mut path = PathBuf::from(&cfg.id); - path.set_extension("zip"); - Arc::new(path) - }; +#[tracing::instrument(skip_all)] +pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + let cfg = read_project_config(matches.get_one::("directory").cloned()).await?; + tracing::debug!(?cfg); let cfg = Arc::new(cfg); - tracing::debug!(?cfg); + let out_path = matches + .get_one::("out") + .expect("parameter should have default value"); + + fs::create_dir_all(out_path) + .await + .wrap_err_with(|| format!("failed to create output directory '{}'", out_path.display()))?; + + let file_map = Arc::new(Mutex::new(FileIndexMap::new())); let tasks = cfg .packages .iter() - .map(|path| (path, cfg.clone())) - .map(|(path, cfg)| async move { + .map(|path| (path, cfg.clone(), file_map.clone())) + .map(|(path, cfg, file_map)| async move { if path.extension().is_some() { eyre::bail!( "Package name must be specified without file extension: {}", @@ -248,45 +267,53 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> ); } - build_package(path, &cfg.dir).await.wrap_err_with(|| { + let bundle = build_package(path, &cfg.dir).await.wrap_err_with(|| { format!( "failed to build package {} in {}", path.display(), cfg.dir.display() ) - }) + })?; + + let bundle_name = match bundle.name() { + IdString64::Hash(_) => { + eyre::bail!("bundle name must be known as string. got hash") + } + IdString64::String(s) => s.clone(), + }; + + { + let mut file_map = file_map.lock().await; + let map_entry = file_map.entry(bundle_name).or_default(); + + for file in bundle.files() { + map_entry.insert(file.name(false, None)); + } + } + + let name = bundle.name().to_murmur64(); + let path = out_path.join(name.to_string().to_ascii_lowercase()); + + let data = bundle.to_binary()?; + fs::write(&path, data) + .await + .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display())) }); - let bundles = try_join_all(tasks) + try_join_all(tasks) .await .wrap_err("failed to build mod bundles")?; - let config_file = { - let path = cfg.dir.join("dtmt.cfg"); - fs::read(&path) - .await - .wrap_err_with(|| format!("failed to read mod config at {}", path.display()))? - }; - { - let dest = dest.clone(); - let id = cfg.id.clone(); - tokio::task::spawn_blocking(move || { - let mut archive = Archive::new(id); - - archive.add_config(config_file); - - for bundle in bundles { - archive.add_bundle(bundle); - } - - archive - .write(dest.as_ref()) - .wrap_err("failed to write mod archive") - }) - .await??; + let file_map = file_map.lock().await; + let data = serde_sjson::to_string(file_map.deref())?; + let path = out_path.join("files.sjson"); + fs::write(&path, data) + .await + .wrap_err_with(|| format!("failed to write file index to '{}'", path.display()))?; } - tracing::info!("Mod archive written to {}", dest.display()); + tracing::info!("Compiled bundles written to '{}'", out_path.display()); + Ok(()) } diff --git a/crates/dtmt/src/cmd/package.rs b/crates/dtmt/src/cmd/package.rs new file mode 100644 index 0000000..44184f6 --- /dev/null +++ b/crates/dtmt/src/cmd/package.rs @@ -0,0 +1,128 @@ +use std::ffi::OsString; +use std::io::{Cursor, Write}; +use std::path::PathBuf; + +use clap::{value_parser, Arg, ArgMatches, Command}; +use color_eyre::eyre::{Context, Result}; +use color_eyre::Help; +use tokio::fs::{self, DirEntry}; +use tokio_stream::wrappers::ReadDirStream; +use tokio_stream::StreamExt; +use zip::ZipWriter; + +use crate::cmd::build::read_project_config; + +pub(crate) fn command_definition() -> Command { + Command::new("package") + .about("Package compiled bundles for distribution") + .arg( + Arg::new("project") + .required(false) + .value_parser(value_parser!(PathBuf)) + .help( + "The path to the project to build. \ + If omitted, dtmt will search from the current working directory upward.", + ), + ) + .arg( + Arg::new("directory") + .long("directory") + .short('d') + .default_value("out") + .value_parser(value_parser!(PathBuf)) + .help( + "The path to the directory were the compiled bundles were written to. \ + This is the same directory as `dtmt build -o`", + ), + ) + .arg( + Arg::new("out") + .long("out") + .short('o') + .default_value(".") + .value_parser(value_parser!(PathBuf)) + .help("The path to write the packaged file to. May be a directory or a file name."), + ) +} + +async fn process_dir_entry(res: Result) -> Result<(OsString, Vec)> { + let entry = res?; + let path = entry.path(); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("failed to read '{}'", path.display()))?; + + Ok((entry.file_name(), data)) +} + +#[tracing::instrument(skip_all)] +pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { + let cfg = read_project_config(matches.get_one::("project").cloned()).await?; + + let dest = { + let mut path = matches + .get_one::("out") + .cloned() + .unwrap_or_else(|| PathBuf::from(".")); + + if path.extension().is_none() { + path.push(format!("{}.zip", cfg.id)) + } + + path + }; + + let data = Cursor::new(Vec::new()); + let mut zip = ZipWriter::new(data); + + zip.add_directory(&cfg.id, Default::default())?; + + let base_path = PathBuf::from(cfg.id); + + { + let name = base_path.join("dtmt.cfg"); + let path = cfg.dir.join("dtmt.cfg"); + + let data = fs::read(&path) + .await + .wrap_err_with(|| format!("failed to read mod config at {}", path.display()))?; + + zip.start_file(name.to_string_lossy(), Default::default())?; + zip.write_all(&data)?; + } + + { + let path = cfg.dir.join( + matches + .get_one::("directory") + .expect("parameter has default value"), + ); + let read_dir = fs::read_dir(&path) + .await + .wrap_err_with(|| format!("failed to read directory '{}'", path.display()))?; + + let stream = ReadDirStream::new(read_dir) + .map(|res| res.wrap_err("failed to read dir entry")) + .then(process_dir_entry); + tokio::pin!(stream); + + while let Some(res) = stream.next().await { + let (name, data) = res?; + + let name = base_path.join(name); + zip.start_file(name.to_string_lossy(), Default::default())?; + zip.write_all(&data)?; + } + }; + + let data = zip.finish()?; + + fs::write(&dest, data.into_inner()) + .await + .wrap_err_with(|| format!("failed to write mod archive to '{}'", dest.display())) + .with_suggestion(|| "Make sure that parent directories exist.".to_string())?; + + tracing::info!("Mod archive written to {}", dest.display()); + Ok(()) +} diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index dc4853e..d4df73c 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -20,14 +20,11 @@ mod cmd { pub mod dictionary; pub mod murmur; pub mod new; + pub mod package; mod util; pub mod watch; } -mod mods { - pub mod archive; -} - #[derive(Default, Deserialize, Serialize)] struct GlobalConfig { game_dir: Option, @@ -56,6 +53,7 @@ async fn main() -> Result<()> { .subcommand(cmd::dictionary::command_definition()) .subcommand(cmd::murmur::command_definition()) .subcommand(cmd::new::command_definition()) + .subcommand(cmd::package::command_definition()) // .subcommand(cmd::watch::command_definition()) .get_matches(); @@ -126,12 +124,13 @@ async fn main() -> Result<()> { }; match matches.subcommand() { + Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?, Some(("bundle", sub_matches)) => cmd::bundle::run(ctx, sub_matches).await?, + Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?, Some(("murmur", sub_matches)) => cmd::murmur::run(ctx, sub_matches).await?, Some(("new", sub_matches)) => cmd::new::run(ctx, sub_matches).await?, - Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?, + Some(("package", sub_matches)) => cmd::package::run(ctx, sub_matches).await?, Some(("watch", sub_matches)) => cmd::watch::run(ctx, sub_matches).await?, - Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?, _ => unreachable!( "clap is configured to require a subcommand, and they're all handled above" ), diff --git a/crates/dtmt/src/mods/archive.rs b/crates/dtmt/src/mods/archive.rs deleted file mode 100644 index 37fec19..0000000 --- a/crates/dtmt/src/mods/archive.rs +++ /dev/null @@ -1,98 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf}; - -use color_eyre::eyre::{self, Context}; -use color_eyre::Result; -use sdk::murmur::IdString64; -use sdk::Bundle; -use zip::ZipWriter; - -pub struct Archive { - name: String, - bundles: Vec, - config_file: Option>, -} - -impl Archive { - pub fn new(name: String) -> Self { - Self { - name, - bundles: Vec::new(), - config_file: None, - } - } - - pub fn add_bundle(&mut self, bundle: Bundle) { - self.bundles.push(bundle) - } - - pub fn add_config(&mut self, content: Vec) { - self.config_file = Some(content); - } - - pub fn write

(&self, path: P) -> Result<()> - where - P: AsRef, - { - let config_file = self - .config_file - .as_ref() - .ok_or_else(|| eyre::eyre!("Config file is missing in mod archive"))?; - - let f = File::create(path.as_ref()).wrap_err_with(|| { - format!( - "failed to open file for reading: {}", - path.as_ref().display() - ) - })?; - let mut zip = ZipWriter::new(f); - - zip.add_directory(&self.name, Default::default())?; - - let base_path = PathBuf::from(&self.name); - - { - let name = base_path.join("dtmt.cfg"); - zip.start_file(name.to_string_lossy(), Default::default())?; - zip.write_all(config_file)?; - } - - let mut file_map = HashMap::new(); - - for bundle in self.bundles.iter() { - let bundle_name = match bundle.name() { - IdString64::Hash(_) => eyre::bail!("bundle name must be known as string. got hash"), - IdString64::String(s) => s, - }; - - let map_entry: &mut HashSet<_> = file_map.entry(bundle_name).or_default(); - - for file in bundle.files() { - map_entry.insert(file.name(false, None)); - } - - let name = bundle.name().to_murmur64(); - let path = base_path.join(name.to_string().to_ascii_lowercase()); - - zip.start_file(path.to_string_lossy(), Default::default())?; - - let data = bundle.to_binary()?; - zip.write_all(&data)?; - } - - { - let data = serde_sjson::to_string(&file_map)?; - zip.start_file( - base_path.join("files.sjson").to_string_lossy(), - Default::default(), - )?; - zip.write_all(data.as_bytes())?; - } - - zip.finish()?; - - Ok(()) - } -} -- 2.45.3 From 865dcae43930034e47988a779f031c401c338d93 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 2 Mar 2023 11:38:08 +0100 Subject: [PATCH 3/4] feat(dtmt): Add parameter to deploy built bundles Ref: #40. --- crates/dtmt/src/cmd/build.rs | 64 +++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/dtmt/src/cmd/build.rs b/crates/dtmt/src/cmd/build.rs index a0db3f2..f4a92bf 100644 --- a/crates/dtmt/src/cmd/build.rs +++ b/crates/dtmt/src/cmd/build.rs @@ -40,6 +40,18 @@ pub(crate) fn command_definition() -> Command { .value_parser(value_parser!(PathBuf)) .help("The directory to write output files to."), ) + .arg( + Arg::new("deploy") + .long("deploy") + .short('d') + .value_parser(value_parser!(PathBuf)) + .help( + "If the path to the game (without the trailing '/bundle') is specified, \ + deploy the newly built bundles. \ + This will not adjust the bundle database or package files, so if files are \ + added or removed, you will have to import into DTMM and re-deploy there.", + ), + ) } #[tracing::instrument] @@ -242,13 +254,20 @@ pub(crate) async fn read_project_config(dir: Option) -> Result Result<()> { let cfg = read_project_config(matches.get_one::("directory").cloned()).await?; - tracing::debug!(?cfg); - let cfg = Arc::new(cfg); + + let game_dir = matches + .get_one::("deploy") + .map(|p| p.join("bundle")); let out_path = matches .get_one::("out") .expect("parameter should have default value"); + tracing::debug!(?cfg, ?game_dir, ?out_path); + + let game_dir = Arc::new(game_dir); + let cfg = Arc::new(cfg); + fs::create_dir_all(out_path) .await .wrap_err_with(|| format!("failed to create output directory '{}'", out_path.display()))?; @@ -258,8 +277,11 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> let tasks = cfg .packages .iter() - .map(|path| (path, cfg.clone(), file_map.clone())) - .map(|(path, cfg, file_map)| async move { + // The closure below would capture the `Arc`s before they could be cloned, + // so instead we need to clone them in a non-move block and inject them + // via parameters. + .map(|path| (path, cfg.clone(), file_map.clone(), game_dir.clone())) + .map(|(path, cfg, file_map, game_dir)| async move { if path.extension().is_some() { eyre::bail!( "Package name must be specified without file extension: {}", @@ -291,13 +313,33 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> } } - let name = bundle.name().to_murmur64(); - let path = out_path.join(name.to_string().to_ascii_lowercase()); - + let name = bundle.name().to_murmur64().to_string().to_ascii_lowercase(); + let path = out_path.join(&name); let data = bundle.to_binary()?; - fs::write(&path, data) + + tracing::trace!( + "Writing bundle {} to '{}'", + bundle.name().display(), + path.display() + ); + fs::write(&path, &data) .await - .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display())) + .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?; + + if let Some(game_dir) = game_dir.as_ref() { + let path = game_dir.join(&name); + + tracing::trace!( + "Deploying bundle {} to '{}'", + bundle.name().display(), + path.display() + ); + fs::write(&path, &data) + .await + .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?; + } + + Ok(()) }); try_join_all(tasks) @@ -315,5 +357,9 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> tracing::info!("Compiled bundles written to '{}'", out_path.display()); + if let Some(game_dir) = game_dir.as_ref() { + tracing::info!("Deployed bundles to '{}'", game_dir.display()); + } + Ok(()) } -- 2.45.3 From dd2ebcf4dffce0d49b83c53a1266b2ed4034ae70 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 2 Mar 2023 11:45:07 +0100 Subject: [PATCH 4/4] docs: Update changelog --- CHANGELOG.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 7c100b3..e32ff71 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -2,6 +2,11 @@ == [Unreleased] +=== Added + +- dtmt: split `build` into `build` and `package` +- dtmt: implement deploying built bundles + == 2023-03-01 === Added -- 2.45.3