Implement name overrides #181

Merged
lucas merged 2 commits from feat/name-overrides into master 2024-07-19 09:35:17 +02:00
10 changed files with 116 additions and 97 deletions
Showing only changes of commit 95fc6c160b - Show all commits

View file

@ -324,11 +324,11 @@ async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
let mut bundles = Vec::new();
let mut add_lua_asset = |name, data: &str| {
let mut add_lua_asset = |name: &str, data: &str| {
let span = tracing::info_span!("Compiling Lua", name, data_len = data.len());
let _enter = span.enter();
let file = lua::compile(name, data).wrap_err("Failed to compile Lua")?;
let file = lua::compile(name.to_string(), data).wrap_err("Failed to compile Lua")?;
mod_bundle.add_file(file);
@ -517,8 +517,8 @@ async fn patch_boot_bundle(
.wrap_err("Failed to render template `mod_main.lua`")?;
tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua);
let file =
lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?;
let file = lua::compile(MOD_BOOT_SCRIPT.to_string(), lua)
.wrap_err("Failed to compile mod main Lua file")?;
boot_bundle.add_file(file);
}

View file

@ -297,6 +297,7 @@ fn extract_mod_config<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Result<(Mo
packages: Vec::new(),
resources,
depends: Vec::new(),
name_overrides: Default::default(),
};
Ok((cfg, root))

View file

@ -103,38 +103,41 @@ async fn find_project_config(dir: Option<PathBuf>) -> Result<ModConfig> {
}
#[tracing::instrument(skip_all)]
async fn compile_package_files<P>(pkg: &Package, root: P) -> Result<Vec<BundleFile>>
where
P: AsRef<Path> + std::fmt::Debug,
{
let root = Arc::new(root.as_ref());
async fn compile_package_files(pkg: &Package, cfg: &ModConfig) -> Result<Vec<BundleFile>> {
let root = Arc::new(&cfg.dir);
let name_overrides = &cfg.name_overrides;
let tasks = pkg
.iter()
.flat_map(|(file_type, paths)| {
paths.iter().map(|path| {
.flat_map(|(file_type, names)| {
names.iter().map(|name| {
(
*file_type,
path,
name,
// Cloning the `Arc` here solves the issue that in the next `.map`, I need to
// `move` the closure parameters, but can't `move` `root` before it was cloned.
root.clone(),
)
})
})
.map(|(file_type, path, root)| async move {
let sjson = fs::read_to_string(&path).await?;
let mut path = path.clone();
path.set_extension("");
BundleFile::from_sjson(
path.to_slash_lossy().to_string(),
file_type,
sjson,
root.as_ref(),
)
.map(|(file_type, name, root)| async move {
let path = PathBuf::from(name);
let sjson = fs::read_to_string(&path)
.await
.wrap_err_with(|| format!("Failed to read file '{}'", path.display()))?;
let name = path.with_extension("").to_slash_lossy().to_string();
let name = if let Some(new_name) = name_overrides.get(&name) {
let new_name = match u64::from_str_radix(new_name, 16) {
Ok(hash) => IdString64::from(hash),
Err(_) => IdString64::from(new_name.clone()),
};
tracing::info!("Overriding '{}' -> '{}'", name, new_name.display());
new_name
} else {
IdString64::from(name.clone())
};
BundleFile::from_sjson(name, file_type, sjson, root.as_ref()).await
});
let results = futures::stream::iter(tasks)
@ -146,12 +149,11 @@ where
}
#[tracing::instrument]
async fn build_package<P1, P2>(package: P1, root: P2) -> Result<Bundle>
where
P1: AsRef<Path> + std::fmt::Debug,
P2: AsRef<Path> + std::fmt::Debug,
{
let root = root.as_ref();
async fn build_package(
cfg: &ModConfig,
package: impl AsRef<Path> + std::fmt::Debug,
) -> Result<Bundle> {
let root = &cfg.dir;
let package = package.as_ref();
let mut path = root.join(package);
@ -165,7 +167,7 @@ where
.await
.wrap_err_with(|| format!("Invalid package file {}", &pkg_name))?;
let files = compile_package_files(&pkg, root).await?;
let files = compile_package_files(&pkg, cfg).await?;
let mut bundle = Bundle::new(pkg_name);
for file in files {
bundle.add_file(file);
@ -254,14 +256,14 @@ pub(crate) async fn read_project_config(dir: Option<PathBuf>) -> Result<ModConfi
Ok(cfg)
}
pub(crate) async fn build<P1, P2>(
#[tracing::instrument]
pub(crate) async fn build<P>(
cfg: &ModConfig,
out_path: P1,
game_dir: Arc<Option<P2>>,
out_path: impl AsRef<Path> + std::fmt::Debug,
game_dir: Arc<Option<P>>,
) -> Result<()>
where
P1: AsRef<Path>,
P2: AsRef<Path>,
P: AsRef<Path> + std::fmt::Debug,
{
let out_path = out_path.as_ref();
@ -286,7 +288,7 @@ where
);
}
let bundle = build_package(path, &cfg.dir).await.wrap_err_with(|| {
let bundle = build_package(&cfg, path).await.wrap_err_with(|| {
format!(
"Failed to build package '{}' at '{}'",
path.display(),

View file

@ -351,6 +351,7 @@ pub(crate) async fn run(_ctx: sdk::Context, matches: &ArgMatches) -> Result<()>
},
depends: vec![ModDependency::ID(String::from("DMF"))],
bundled: true,
name_overrides: HashMap::new(),
};
tracing::debug!(?dtmt_cfg);

View file

@ -77,17 +77,14 @@ pub(crate) fn command_definition() -> Command {
)
}
async fn compile<P1, P2, P3>(
#[tracing::instrument]
async fn compile(
cfg: &ModConfig,
out_path: P1,
archive_path: P2,
game_dir: Arc<Option<P3>>,
) -> Result<()>
where
P1: AsRef<Path> + std::marker::Copy,
P2: AsRef<Path>,
P3: AsRef<Path>,
{
out_path: impl AsRef<Path> + std::fmt::Debug,
archive_path: impl AsRef<Path> + std::fmt::Debug,
game_dir: Arc<Option<impl AsRef<Path> + std::fmt::Debug>>,
) -> Result<()> {
let out_path = out_path.as_ref();
build(cfg, out_path, game_dir)
.await
.wrap_err("Failed to build bundles")?;

View file

@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::path::PathBuf;
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
@ -67,6 +68,8 @@ pub struct ModConfig {
pub depends: Vec<ModDependency>,
#[serde(default = "default_true", skip_serializing_if = "is_true")]
pub bundled: bool,
#[serde(default)]
pub name_overrides: HashMap<String, String>,
}
pub const STEAMAPP_ID: u32 = 1361210;

View file

@ -120,7 +120,7 @@ pub struct BundleFile {
}
impl BundleFile {
pub fn new(name: String, file_type: BundleFileType) -> Self {
pub fn new(name: impl Into<IdString64>, file_type: BundleFileType) -> Self {
Self {
file_type,
name: name.into(),
@ -252,20 +252,15 @@ impl BundleFile {
Ok(w.into_inner())
}
#[tracing::instrument(name = "File::from_sjson", skip(sjson))]
pub async fn from_sjson<P, S>(
name: String,
#[tracing::instrument("File::from_sjson", skip(sjson, name), fields(name = %name.display()))]
pub async fn from_sjson(
name: IdString64,
file_type: BundleFileType,
sjson: S,
root: P,
) -> Result<Self>
where
P: AsRef<Path> + std::fmt::Debug,
S: AsRef<str>,
{
sjson: impl AsRef<str>,
root: impl AsRef<Path> + std::fmt::Debug,
) -> Result<Self> {
match file_type {
BundleFileType::Lua => lua::compile(name.clone(), sjson)
.wrap_err_with(|| format!("Failed to compile Lua file '{}'", name)),
BundleFileType::Lua => lua::compile(name, sjson).wrap_err("Failed to compile Lua file"),
BundleFileType::Unknown(_) => {
eyre::bail!("Unknown file type. Cannot compile from SJSON");
}
@ -304,10 +299,7 @@ impl BundleFile {
s
}
pub fn matches_name<S>(&self, name: S) -> bool
where
S: Into<IdString64>,
{
pub fn matches_name(&self, name: impl Into<IdString64>) -> bool {
let name = name.into();
if self.name == name {
return true;

View file

@ -15,6 +15,7 @@ use tokio::fs;
use crate::binary::sync::ReadExt;
use crate::binary::sync::WriteExt;
use crate::bundle::file::{BundleFileVariant, UserFile};
use crate::murmur::IdString64;
use crate::{BundleFile, BundleFileType};
const BITSQUID_LUAJIT_HEADER: u32 = 0x8253461B;
@ -117,17 +118,13 @@ where
}
#[tracing::instrument(skip_all)]
pub fn compile<S, C>(name: S, code: C) -> Result<BundleFile>
where
S: Into<String>,
C: AsRef<str>,
{
pub fn compile(name: impl Into<IdString64>, code: impl AsRef<str>) -> Result<BundleFile> {
let name = name.into();
let code = code.as_ref();
tracing::trace!(
"Compiling '{}', {} bytes of code",
name,
name.display(),
code.as_bytes().len()
);
@ -135,8 +132,8 @@ where
let state = lua::luaL_newstate();
lua::luaL_openlibs(state);
let name = CString::new(format!("@{name}").into_bytes())
.wrap_err_with(|| format!("Cannot convert name into CString: {}", name))?;
let name = CString::new(format!("@{}", name.display()).into_bytes())
.wrap_err_with(|| format!("Cannot convert name into CString: {}", name.display()))?;
match lua::luaL_loadbuffer(
state,
code.as_ptr() as _,

View file

@ -7,13 +7,12 @@ use std::str::FromStr;
use async_recursion::async_recursion;
use color_eyre::eyre::{self, Context};
use color_eyre::Result;
use path_slash::PathBufExt;
use tokio::fs;
use crate::binary::sync::{ReadExt, WriteExt};
use crate::bundle::file::UserFile;
use crate::bundle::filetype::BundleFileType;
use crate::murmur::{HashGroup, Murmur64};
use crate::murmur::{HashGroup, IdString64, Murmur64};
#[tracing::instrument]
#[async_recursion]
@ -91,12 +90,12 @@ where
Ok(paths)
}
type PackageType = HashMap<BundleFileType, HashSet<PathBuf>>;
type PackageType = HashMap<BundleFileType, HashSet<String>>;
type PackageDefinition = HashMap<String, HashSet<String>>;
#[derive(Default)]
pub struct Package {
_name: String,
_name: IdString64,
_root: PathBuf,
inner: PackageType,
flags: u8,
@ -117,9 +116,9 @@ impl DerefMut for Package {
}
impl Package {
pub fn new(name: String, root: PathBuf) -> Self {
pub fn new(name: impl Into<IdString64>, root: PathBuf) -> Self {
Self {
_name: name,
_name: name.into(),
_root: root,
inner: Default::default(),
flags: 1,
@ -130,17 +129,22 @@ impl Package {
self.values().fold(0, |total, files| total + files.len())
}
pub fn add_file<P: Into<PathBuf>>(&mut self, file_type: BundleFileType, name: P) {
pub fn add_file(&mut self, file_type: BundleFileType, name: impl Into<String>) {
self.inner.entry(file_type).or_default().insert(name.into());
}
#[tracing::instrument("Package::from_sjson", skip(sjson), fields(sjson_len = sjson.as_ref().len()))]
pub async fn from_sjson<P, S>(sjson: S, name: String, root: P) -> Result<Self>
pub async fn from_sjson<P, S>(
sjson: S,
name: impl Into<IdString64> + std::fmt::Debug,
root: P,
) -> Result<Self>
where
P: AsRef<Path> + std::fmt::Debug,
S: AsRef<str>,
{
let root = root.as_ref();
let name = name.into();
let definition: PackageDefinition = serde_sjson::from_str(sjson.as_ref())?;
let mut inner: PackageType = Default::default();
@ -174,7 +178,11 @@ impl Package {
continue;
};
inner.entry(t).or_default().insert(path);
tracing::debug!("Adding file {}", path.display());
inner
.entry(t)
.or_default()
.insert(path.display().to_string());
}
}
}
@ -193,11 +201,9 @@ impl Package {
pub fn to_sjson(&self) -> Result<String> {
let mut map: PackageDefinition = Default::default();
for (t, paths) in self.iter() {
for path in paths.iter() {
map.entry(t.ext_name())
.or_default()
.insert(path.display().to_string());
for (t, names) in self.iter() {
for name in names.iter() {
map.entry(t.ext_name()).or_default().insert(name.clone());
}
}
@ -223,11 +229,11 @@ impl Package {
for _ in 0..file_count {
let t = BundleFileType::from(r.read_u64()?);
let hash = Murmur64::from(r.read_u64()?);
let path = ctx.lookup_hash(hash, HashGroup::Filename);
let name = ctx.lookup_hash(hash, HashGroup::Filename);
inner
.entry(t)
.or_default()
.insert(PathBuf::from(path.display().to_string()));
.insert(name.display().to_string());
}
let flags = r.read_u8()?;
@ -240,7 +246,7 @@ impl Package {
let pkg = Self {
inner,
_name: name,
_name: name.into(),
_root: PathBuf::new(),
flags,
};
@ -256,12 +262,10 @@ impl Package {
w.write_u32(0x2b)?;
w.write_u32(self.values().flatten().count() as u32)?;
for (t, paths) in self.iter() {
for path in paths.iter() {
for (t, names) in self.iter() {
for name in names.iter() {
w.write_u64(t.hash().into())?;
let hash = Murmur64::hash(path.to_slash_lossy().as_bytes());
w.write_u64(hash.into())?;
w.write_u64(Murmur64::hash(name.as_bytes()).into())?;
}
}

View file

@ -1,3 +1,7 @@
use std::path::Path;
use path_slash::PathExt;
use self::util::{parse_hex32, parse_hex64};
use super::*;
@ -263,11 +267,23 @@ impl IdString64 {
IdString64::String(_) => false,
}
}
// Would love to have this as a proper `impl From`, but
// rustc will complain that it overlaps with the `impl From<Into<String>>`.
pub fn from_path(p: impl AsRef<Path>) -> Self {
Self::String(p.as_ref().to_slash_lossy().to_string())
}
}
impl<S: Into<String>> From<S> for IdString64 {
fn from(value: S) -> Self {
Self::String(value.into())
impl From<String> for IdString64 {
fn from(value: String) -> Self {
Self::String(value)
}
}
impl From<u64> for IdString64 {
fn from(value: u64) -> Self {
Self::Hash(value.into())
}
}
@ -283,6 +299,12 @@ impl From<IdString64> for Murmur64 {
}
}
impl Default for IdString64 {
fn default() -> Self {
Self::Hash(0.into())
}
}
impl PartialEq for IdString64 {
fn eq(&self, other: &Self) -> bool {
self.to_murmur64() == other.to_murmur64()