Merge pull request 'Split build command' (#41) from feat/split-build into master

Reviewed-on: #41

CHANGELOG.adoc

@@ -2,6 +2,11 @@
 
 == [Unreleased]
 
+=== Added
+
+- dtmt: split `build` into `build` and `package`
+- dtmt: implement deploying built bundles
+
 == 2023-03-01
 
 === Added

crates/dtmm/src/main.rs

@@ -37,12 +37,6 @@ fn main() -> Result<()> {
     tracing::trace!(default_config_path = %default_config_path.display());
 
     let matches = command!()
-        .arg(Arg::new("oodle").long("oodle").help(
-            "The oodle library to load. This may either be:\n\
-                        - A library name that will be searched for in the system's default paths.\n\
-                        - A file path relative to the current working directory.\n\
-                        - An absolute file path.",
-        ))
         .arg(
             Arg::new("config")
                 .long("config")

crates/dtmt/src/cmd/build.rs

@@ -1,3 +1,5 @@
+use std::collections::{HashMap, HashSet};
+use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
@@ -8,14 +10,16 @@ use dtmt_shared::ModConfig;
 use futures::future::try_join_all;
 use futures::StreamExt;
 use sdk::filetype::package::Package;
+use sdk::murmur::IdString64;
 use sdk::{Bundle, BundleFile};
 use tokio::fs::{self, File};
 use tokio::io::AsyncReadExt;
-
-use crate::mods::archive::Archive;
+use tokio::sync::Mutex;
 
 const PROJECT_CONFIG_NAME: &str = "dtmt.cfg";
 
+type FileIndexMap = HashMap<String, HashSet<String>>;
+
 pub(crate) fn command_definition() -> Command {
     Command::new("build")
         .about("Build a project")
@@ -28,12 +32,26 @@ pub(crate) fn command_definition() -> Command {
                         If omitted, dtmt will search from the current working directory upward.",
                 ),
         )
-        .arg(Arg::new("oodle").long("oodle").help(
-            "The oodle library to load. This may either be:\n\
-                - A library name that will be searched for in the system's default paths.\n\
-                - A file path relative to the current working directory.\n\
-                - An absolute file path.",
-        ))
+        .arg(
+            Arg::new("out")
+                .long("out")
+                .short('o')
+                .default_value("out")
+                .value_parser(value_parser!(PathBuf))
+                .help("The directory to write output files to."),
+        )
+        .arg(
+            Arg::new("deploy")
+                .long("deploy")
+                .short('d')
+                .value_parser(value_parser!(PathBuf))
+                .help(
+                    "If the path to the game (without the trailing '/bundle') is specified, \
+                        deploy the newly built bundles. \
+                        This will not adjust the bundle database or package files, so if files are \
+                        added or removed, you will have to import into DTMM and re-deploy there.",
+                ),
+        )
 }
 
 #[tracing::instrument]
@@ -181,10 +199,8 @@ fn normalize_file_path<P: AsRef<Path>>(path: P) -> Result<PathBuf> {
     Ok(path)
 }
 
-#[tracing::instrument(skip_all)]
-pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
-    let cfg = {
-        let dir = matches.get_one::<PathBuf>("directory").cloned();
+#[tracing::instrument]
+pub(crate) async fn read_project_config(dir: Option<PathBuf>) -> Result<ModConfig> {
     let mut cfg = find_project_config(dir).await?;
 
     cfg.resources.init = normalize_file_path(cfg.resources.init)
@@ -232,23 +248,40 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
         cfg.resources.localization = Some(path);
     }
 
-        cfg
-    };
+    Ok(cfg)
+}
 
-    let dest = {
-        let mut path = PathBuf::from(&cfg.id);
-        path.set_extension("zip");
-        Arc::new(path)
-    };
+#[tracing::instrument(skip_all)]
+pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
+    let cfg = read_project_config(matches.get_one::<PathBuf>("directory").cloned()).await?;
+
+    let game_dir = matches
+        .get_one::<PathBuf>("deploy")
+        .map(|p| p.join("bundle"));
+
+    let out_path = matches
+        .get_one::<PathBuf>("out")
+        .expect("parameter should have default value");
+
+    tracing::debug!(?cfg, ?game_dir, ?out_path);
+
+    let game_dir = Arc::new(game_dir);
     let cfg = Arc::new(cfg);
 
-    tracing::debug!(?cfg);
+    fs::create_dir_all(out_path)
+        .await
+        .wrap_err_with(|| format!("failed to create output directory '{}'", out_path.display()))?;
+
+    let file_map = Arc::new(Mutex::new(FileIndexMap::new()));
 
     let tasks = cfg
         .packages
         .iter()
-        .map(|path| (path, cfg.clone()))
-        .map(|(path, cfg)| async move {
+        // The closure below would capture the `Arc`s before they could be cloned,
+        // so instead we need to clone them in a non-move block and inject them
+        // via parameters.
+        .map(|path| (path, cfg.clone(), file_map.clone(), game_dir.clone()))
+        .map(|(path, cfg, file_map, game_dir)| async move {
             if path.extension().is_some() {
                 eyre::bail!(
                     "Package name must be specified without file extension: {}",
@@ -256,45 +289,77 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
                 );
             }
 
-            build_package(path, &cfg.dir).await.wrap_err_with(|| {
+            let bundle = build_package(path, &cfg.dir).await.wrap_err_with(|| {
                 format!(
                     "failed to build package {} in {}",
                     path.display(),
                     cfg.dir.display()
                 )
-            })
-        });
+            })?;
 
-    let bundles = try_join_all(tasks)
-        .await
-        .wrap_err("failed to build mod bundles")?;
-
-    let config_file = {
-        let path = cfg.dir.join("dtmt.cfg");
-        fs::read(&path)
-            .await
-            .wrap_err_with(|| format!("failed to read mod config at {}", path.display()))?
+            let bundle_name = match bundle.name() {
+                IdString64::Hash(_) => {
+                    eyre::bail!("bundle name must be known as string. got hash")
+                }
+                IdString64::String(s) => s.clone(),
             };
 
             {
-        let dest = dest.clone();
-        let id = cfg.id.clone();
-        tokio::task::spawn_blocking(move || {
-            let mut archive = Archive::new(id);
+                let mut file_map = file_map.lock().await;
+                let map_entry = file_map.entry(bundle_name).or_default();
 
-            archive.add_config(config_file);
-
-            for bundle in bundles {
-                archive.add_bundle(bundle);
+                for file in bundle.files() {
+                    map_entry.insert(file.name(false, None));
+                }
             }
 
-            archive
-                .write(dest.as_ref())
-                .wrap_err("failed to write mod archive")
-        })
-        .await??;
+            let name = bundle.name().to_murmur64().to_string().to_ascii_lowercase();
+            let path = out_path.join(&name);
+            let data = bundle.to_binary()?;
+
+            tracing::trace!(
+                "Writing bundle {} to '{}'",
+                bundle.name().display(),
+                path.display()
+            );
+            fs::write(&path, &data)
+                .await
+                .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?;
+
+            if let Some(game_dir) = game_dir.as_ref() {
+                let path = game_dir.join(&name);
+
+                tracing::trace!(
+                    "Deploying bundle {} to '{}'",
+                    bundle.name().display(),
+                    path.display()
+                );
+                fs::write(&path, &data)
+                    .await
+                    .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?;
+            }
+
+            Ok(())
+        });
+
+    try_join_all(tasks)
+        .await
+        .wrap_err("failed to build mod bundles")?;
+
+    {
+        let file_map = file_map.lock().await;
+        let data = serde_sjson::to_string(file_map.deref())?;
+        let path = out_path.join("files.sjson");
+        fs::write(&path, data)
+            .await
+            .wrap_err_with(|| format!("failed to write file index to '{}'", path.display()))?;
+    }
+
+    tracing::info!("Compiled bundles written to '{}'", out_path.display());
+
+    if let Some(game_dir) = game_dir.as_ref() {
+        tracing::info!("Deployed bundles to '{}'", game_dir.display());
     }
 
-    tracing::info!("Mod archive written to {}", dest.display());
     Ok(())
 }

crates/dtmt/src/cmd/bundle/mod.rs

@@ -1,4 +1,4 @@
-use clap::{Arg, ArgMatches, Command};
+use clap::{ArgMatches, Command};
 use color_eyre::eyre::Result;
 
 mod decompress;
@@ -10,12 +10,6 @@ pub(crate) fn command_definition() -> Command {
     Command::new("bundle")
         .subcommand_required(true)
         .about("Manipulate the game's bundle files")
-        .arg(Arg::new("oodle").long("oodle").help(
-            "The oodle library to load. This may either be:\n\
-                        - A library name that will be searched for in the system's default paths.\n\
-                        - A file path relative to the current working directory.\n\
-                        - An absolute file path.",
-        ))
         .subcommand(decompress::command_definition())
         .subcommand(extract::command_definition())
         .subcommand(inject::command_definition())

crates/dtmt/src/cmd/package.rs

@@ -0,0 +1,128 @@
+use std::ffi::OsString;
+use std::io::{Cursor, Write};
+use std::path::PathBuf;
+
+use clap::{value_parser, Arg, ArgMatches, Command};
+use color_eyre::eyre::{Context, Result};
+use color_eyre::Help;
+use tokio::fs::{self, DirEntry};
+use tokio_stream::wrappers::ReadDirStream;
+use tokio_stream::StreamExt;
+use zip::ZipWriter;
+
+use crate::cmd::build::read_project_config;
+
+pub(crate) fn command_definition() -> Command {
+    Command::new("package")
+        .about("Package compiled bundles for distribution")
+        .arg(
+            Arg::new("project")
+                .required(false)
+                .value_parser(value_parser!(PathBuf))
+                .help(
+                    "The path to the project to build. \
+                        If omitted, dtmt will search from the current working directory upward.",
+                ),
+        )
+        .arg(
+            Arg::new("directory")
+                .long("directory")
+                .short('d')
+                .default_value("out")
+                .value_parser(value_parser!(PathBuf))
+                .help(
+                    "The path to the directory were the compiled bundles were written to. \
+                        This is the same directory as `dtmt build -o`",
+                ),
+        )
+        .arg(
+            Arg::new("out")
+                .long("out")
+                .short('o')
+                .default_value(".")
+                .value_parser(value_parser!(PathBuf))
+                .help("The path to write the packaged file to. May be a directory or a file name."),
+        )
+}
+
+async fn process_dir_entry(res: Result<DirEntry>) -> Result<(OsString, Vec<u8>)> {
+    let entry = res?;
+    let path = entry.path();
+
+    let data = fs::read(&path)
+        .await
+        .wrap_err_with(|| format!("failed to read '{}'", path.display()))?;
+
+    Ok((entry.file_name(), data))
+}
+
+#[tracing::instrument(skip_all)]
+pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> {
+    let cfg = read_project_config(matches.get_one::<PathBuf>("project").cloned()).await?;
+
+    let dest = {
+        let mut path = matches
+            .get_one::<PathBuf>("out")
+            .cloned()
+            .unwrap_or_else(|| PathBuf::from("."));
+
+        if path.extension().is_none() {
+            path.push(format!("{}.zip", cfg.id))
+        }
+
+        path
+    };
+
+    let data = Cursor::new(Vec::new());
+    let mut zip = ZipWriter::new(data);
+
+    zip.add_directory(&cfg.id, Default::default())?;
+
+    let base_path = PathBuf::from(cfg.id);
+
+    {
+        let name = base_path.join("dtmt.cfg");
+        let path = cfg.dir.join("dtmt.cfg");
+
+        let data = fs::read(&path)
+            .await
+            .wrap_err_with(|| format!("failed to read mod config at {}", path.display()))?;
+
+        zip.start_file(name.to_string_lossy(), Default::default())?;
+        zip.write_all(&data)?;
+    }
+
+    {
+        let path = cfg.dir.join(
+            matches
+                .get_one::<PathBuf>("directory")
+                .expect("parameter has default value"),
+        );
+        let read_dir = fs::read_dir(&path)
+            .await
+            .wrap_err_with(|| format!("failed to read directory '{}'", path.display()))?;
+
+        let stream = ReadDirStream::new(read_dir)
+            .map(|res| res.wrap_err("failed to read dir entry"))
+            .then(process_dir_entry);
+        tokio::pin!(stream);
+
+        while let Some(res) = stream.next().await {
+            let (name, data) = res?;
+
+            let name = base_path.join(name);
+            zip.start_file(name.to_string_lossy(), Default::default())?;
+            zip.write_all(&data)?;
+        }
+    };
+
+    let data = zip.finish()?;
+
+    fs::write(&dest, data.into_inner())
+        .await
+        .wrap_err_with(|| format!("failed to write mod archive to '{}'", dest.display()))
+        .with_suggestion(|| "Make sure that parent directories exist.".to_string())?;
+
+    tracing::info!("Mod archive written to {}", dest.display());
+    Ok(())
+}

crates/dtmt/src/main.rs

@@ -20,14 +20,11 @@ mod cmd {
     pub mod dictionary;
     pub mod murmur;
     pub mod new;
+    pub mod package;
     mod util;
     pub mod watch;
 }
 
-mod mods {
-    pub mod archive;
-}
-
 #[derive(Default, Deserialize, Serialize)]
 struct GlobalConfig {
     game_dir: Option<PathBuf>,
@@ -56,6 +53,7 @@ async fn main() -> Result<()> {
         .subcommand(cmd::dictionary::command_definition())
         .subcommand(cmd::murmur::command_definition())
         .subcommand(cmd::new::command_definition())
+        .subcommand(cmd::package::command_definition())
         // .subcommand(cmd::watch::command_definition())
         .get_matches();
 
@@ -126,12 +124,13 @@ async fn main() -> Result<()> {
     };
 
     match matches.subcommand() {
+        Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?,
         Some(("bundle", sub_matches)) => cmd::bundle::run(ctx, sub_matches).await?,
+        Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?,
         Some(("murmur", sub_matches)) => cmd::murmur::run(ctx, sub_matches).await?,
         Some(("new", sub_matches)) => cmd::new::run(ctx, sub_matches).await?,
-        Some(("build", sub_matches)) => cmd::build::run(ctx, sub_matches).await?,
+        Some(("package", sub_matches)) => cmd::package::run(ctx, sub_matches).await?,
         Some(("watch", sub_matches)) => cmd::watch::run(ctx, sub_matches).await?,
-        Some(("dictionary", sub_matches)) => cmd::dictionary::run(ctx, sub_matches).await?,
         _ => unreachable!(
             "clap is configured to require a subcommand, and they're all handled above"
         ),

crates/dtmt/src/mods/archive.rs

@@ -1,98 +0,0 @@
-use std::collections::{HashMap, HashSet};
-use std::fs::File;
-use std::io::Write;
-use std::path::{Path, PathBuf};
-
-use color_eyre::eyre::{self, Context};
-use color_eyre::Result;
-use sdk::murmur::IdString64;
-use sdk::Bundle;
-use zip::ZipWriter;
-
-pub struct Archive {
-    name: String,
-    bundles: Vec<Bundle>,
-    config_file: Option<Vec<u8>>,
-}
-
-impl Archive {
-    pub fn new(name: String) -> Self {
-        Self {
-            name,
-            bundles: Vec::new(),
-            config_file: None,
-        }
-    }
-
-    pub fn add_bundle(&mut self, bundle: Bundle) {
-        self.bundles.push(bundle)
-    }
-
-    pub fn add_config(&mut self, content: Vec<u8>) {
-        self.config_file = Some(content);
-    }
-
-    pub fn write<P>(&self, path: P) -> Result<()>
-    where
-        P: AsRef<Path>,
-    {
-        let config_file = self
-            .config_file
-            .as_ref()
-            .ok_or_else(|| eyre::eyre!("Config file is missing in mod archive"))?;
-
-        let f = File::create(path.as_ref()).wrap_err_with(|| {
-            format!(
-                "failed to open file for reading: {}",
-                path.as_ref().display()
-            )
-        })?;
-        let mut zip = ZipWriter::new(f);
-
-        zip.add_directory(&self.name, Default::default())?;
-
-        let base_path = PathBuf::from(&self.name);
-
-        {
-            let name = base_path.join("dtmt.cfg");
-            zip.start_file(name.to_string_lossy(), Default::default())?;
-            zip.write_all(config_file)?;
-        }
-
-        let mut file_map = HashMap::new();
-
-        for bundle in self.bundles.iter() {
-            let bundle_name = match bundle.name() {
-                IdString64::Hash(_) => eyre::bail!("bundle name must be known as string. got hash"),
-                IdString64::String(s) => s,
-            };
-
-            let map_entry: &mut HashSet<_> = file_map.entry(bundle_name).or_default();
-
-            for file in bundle.files() {
-                map_entry.insert(file.name(false, None));
-            }
-
-            let name = bundle.name().to_murmur64();
-            let path = base_path.join(name.to_string().to_ascii_lowercase());
-
-            zip.start_file(path.to_string_lossy(), Default::default())?;
-
-            let data = bundle.to_binary()?;
-            zip.write_all(&data)?;
-        }
-
-        {
-            let data = serde_sjson::to_string(&file_map)?;
-            zip.start_file(
-                base_path.join("files.sjson").to_string_lossy(),
-                Default::default(),
-            )?;
-            zip.write_all(data.as_bytes())?;
-        }
-
-        zip.finish()?;
-
-        Ok(())
-    }
-}