use std::collections::{HashMap, HashSet}; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::{self, Context, Result}; use color_eyre::{Help, Report}; use dtmt_shared::ModConfig; use futures::future::try_join_all; use futures::StreamExt; use sdk::filetype::package::Package; use sdk::murmur::IdString64; use sdk::{Bundle, BundleFile}; use tokio::fs::{self, File}; use tokio::io::AsyncReadExt; use tokio::sync::Mutex; const PROJECT_CONFIG_NAME: &str = "dtmt.cfg"; type FileIndexMap = HashMap>; pub(crate) fn command_definition() -> Command { Command::new("build") .about("Build a project") .arg( Arg::new("directory") .required(false) .value_parser(value_parser!(PathBuf)) .help( "The path to the project to build. \ If omitted, dtmt will search from the current working directory upward.", ), ) .arg( Arg::new("out") .long("out") .short('o') .default_value("out") .value_parser(value_parser!(PathBuf)) .help("The directory to write output files to."), ) .arg( Arg::new("deploy") .long("deploy") .short('d') .value_parser(value_parser!(PathBuf)) .help( "If the path to the game (without the trailing '/bundle') is specified, \ deploy the newly built bundles. \ This will not adjust the bundle database or package files, so if files are \ added or removed, you will have to import into DTMM and re-deploy there.", ), ) } #[tracing::instrument] async fn find_project_config(dir: Option) -> Result { let (path, mut file) = if let Some(path) = dir { let file = File::open(&path.join(PROJECT_CONFIG_NAME)) .await .wrap_err_with(|| format!("failed to open file: {}", path.display())) .with_suggestion(|| { format!( "Make sure the file at '{}' exists and is readable", path.display() ) })?; (path, file) } else { let mut dir = std::env::current_dir()?; loop { let path = dir.join(PROJECT_CONFIG_NAME); match File::open(&path).await { Ok(file) => break (dir, file), Err(err) if err.kind() == std::io::ErrorKind::NotFound => { if let Some(parent) = dir.parent() { // TODO: Re-write with recursion to avoid allocating the `PathBuf`. dir = parent.to_path_buf(); } else { eyre::bail!("Could not find project root"); } } Err(err) => { let err = Report::new(err) .wrap_err(format!("failed to open file: {}", path.display())); return Err(err); } } } }; let mut buf = String::new(); file.read_to_string(&mut buf) .await .wrap_err("invalid UTF-8")?; let mut cfg: ModConfig = serde_sjson::from_str(&buf).wrap_err("failed to deserialize mod config")?; cfg.dir = path; Ok(cfg) } #[tracing::instrument(skip_all)] async fn compile_package_files

(pkg: &Package, root: P) -> Result> where P: AsRef + std::fmt::Debug, { let root = Arc::new(root.as_ref()); let tasks = pkg .iter() .flat_map(|(file_type, paths)| { paths.iter().map(|path| { ( *file_type, path, // Cloning the `Arc` here solves the issue that in the next `.map`, I need to // `move` the closure parameters, but can't `move` `root` before it was cloned. root.clone(), ) }) }) .map(|(file_type, path, root)| async move { let sjson = fs::read_to_string(&path).await?; let mut path = path.clone(); path.set_extension(""); BundleFile::from_sjson( path.to_string_lossy().to_string(), file_type, sjson, root.as_ref(), ) .await }); let results = futures::stream::iter(tasks) .buffer_unordered(10) .collect::>>() .await; results.into_iter().collect() } #[tracing::instrument(skip_all, fields(files = files.len()))] fn compile_bundle(name: String, files: Vec) -> Result { let mut bundle = Bundle::new(name); for file in files { bundle.add_file(file); } Ok(bundle) } #[tracing::instrument] async fn build_package(package: P1, root: P2) -> Result where P1: AsRef + std::fmt::Debug, P2: AsRef + std::fmt::Debug, { let root = root.as_ref(); let package = package.as_ref(); let mut path = root.join(package); path.set_extension("package"); let sjson = fs::read_to_string(&path) .await .wrap_err_with(|| format!("failed to read file {}", path.display()))?; let pkg_name = package.to_string_lossy().to_string(); let pkg = Package::from_sjson(sjson, pkg_name.clone(), root) .await .wrap_err_with(|| format!("invalid package file {}", &pkg_name))?; compile_package_files(&pkg, root) .await .wrap_err("failed to compile package") .and_then(|files| compile_bundle(pkg_name, files)) .wrap_err("failed to build bundle") } fn normalize_file_path>(path: P) -> Result { let path = path.as_ref(); if path.is_absolute() || path.has_root() { let err = eyre::eyre!("path is absolute: {}", path.display()); return Err(err).with_suggestion(|| "Specify a relative file path.".to_string()); } let path = path_clean::clean(path); if path.starts_with("..") { eyre::bail!("path starts with a parent component: {}", path.display()); } Ok(path) } #[tracing::instrument] pub(crate) async fn read_project_config(dir: Option) -> Result { let mut cfg = find_project_config(dir).await?; cfg.resources.init = normalize_file_path(cfg.resources.init) .wrap_err("invalid config field 'resources.init'") .with_suggestion(|| { "Specify a file path relative to and child path of the \ directory where 'dtmt.cfg' is." .to_string() }) .with_suggestion(|| { "Use 'dtmt new' in a separate directory to generate \ a valid mod template." .to_string() })?; if let Some(path) = cfg.resources.data { let path = normalize_file_path(path) .wrap_err("invalid config field 'resources.data'") .with_suggestion(|| { "Specify a file path relative to and child path of the \ directory where 'dtmt.cfg' is." .to_string() }) .with_suggestion(|| { "Use 'dtmt new' in a separate directory to generate \ a valid mod template." .to_string() })?; cfg.resources.data = Some(path); } if let Some(path) = cfg.resources.localization { let path = normalize_file_path(path) .wrap_err("invalid config field 'resources.localization'") .with_suggestion(|| { "Specify a file path relative to and child path of the \ directory where 'dtmt.cfg' is." .to_string() }) .with_suggestion(|| { "Use 'dtmt new' in a separate directory to generate \ a valid mod template." .to_string() })?; cfg.resources.localization = Some(path); } Ok(cfg) } #[tracing::instrument(skip_all)] pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()> { let cfg = read_project_config(matches.get_one::("directory").cloned()).await?; let game_dir = matches .get_one::("deploy") .map(|p| p.join("bundle")); let out_path = matches .get_one::("out") .expect("parameter should have default value"); tracing::debug!(?cfg, ?game_dir, ?out_path); let game_dir = Arc::new(game_dir); let cfg = Arc::new(cfg); fs::create_dir_all(out_path) .await .wrap_err_with(|| format!("failed to create output directory '{}'", out_path.display()))?; let file_map = Arc::new(Mutex::new(FileIndexMap::new())); let tasks = cfg .packages .iter() // The closure below would capture the `Arc`s before they could be cloned, // so instead we need to clone them in a non-move block and inject them // via parameters. .map(|path| (path, cfg.clone(), file_map.clone(), game_dir.clone())) .map(|(path, cfg, file_map, game_dir)| async move { if path.extension().is_some() { eyre::bail!( "Package name must be specified without file extension: {}", path.display() ); } let bundle = build_package(path, &cfg.dir).await.wrap_err_with(|| { format!( "failed to build package {} in {}", path.display(), cfg.dir.display() ) })?; let bundle_name = match bundle.name() { IdString64::Hash(_) => { eyre::bail!("bundle name must be known as string. got hash") } IdString64::String(s) => s.clone(), }; { let mut file_map = file_map.lock().await; let map_entry = file_map.entry(bundle_name).or_default(); for file in bundle.files() { map_entry.insert(file.name(false, None)); } } let name = bundle.name().to_murmur64().to_string().to_ascii_lowercase(); let path = out_path.join(&name); let data = bundle.to_binary()?; tracing::trace!( "Writing bundle {} to '{}'", bundle.name().display(), path.display() ); fs::write(&path, &data) .await .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?; if let Some(game_dir) = game_dir.as_ref() { let path = game_dir.join(&name); tracing::trace!( "Deploying bundle {} to '{}'", bundle.name().display(), path.display() ); fs::write(&path, &data) .await .wrap_err_with(|| format!("failed to write bundle to '{}'", path.display()))?; } Ok(()) }); try_join_all(tasks) .await .wrap_err("failed to build mod bundles")?; { let file_map = file_map.lock().await; let data = serde_sjson::to_string(file_map.deref())?; let path = out_path.join("files.sjson"); fs::write(&path, data) .await .wrap_err_with(|| format!("failed to write file index to '{}'", path.display()))?; } tracing::info!("Compiled bundles written to '{}'", out_path.display()); if let Some(game_dir) = game_dir.as_ref() { tracing::info!("Deployed bundles to '{}'", game_dir.display()); } Ok(()) }