From 535a30a7ca6f77195da6bf54ab4d9b9120ff3334 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Tue, 14 May 2024 00:50:01 +0200 Subject: [PATCH 01/19] Add simpler shell parser This obsoletes `shlex`. The quoting turned out unnecessary, and the splitting supported a lot more than we need. It also forced unncessary allocations: The splitting doesn't add any characters and keeps UTF-8 intact, so returning slices from the input is perfectly possible. Though this particular implementation will only come to use in the future, as `CmdLine` still requires that the slices are cloned. Still, the custom implementation performs about 3x faster. --- crates/dtmt/Cargo.toml | 5 +- crates/dtmt/src/cmd/bundle/extract.rs | 32 +++-- crates/dtmt/src/main.rs | 2 + crates/dtmt/src/shell_parse.rs | 189 ++++++++++++++++++++++++++ 4 files changed, 214 insertions(+), 14 deletions(-) create mode 100644 crates/dtmt/src/shell_parse.rs diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 69bbc31..688066f 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -33,7 +33,10 @@ path-slash = "0.2.1" async-recursion = "1.0.2" notify = "5.1.0" luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } -shlex = "1.2.0" +shlex = { version = "1.2.0", optional = true } [dev-dependencies] tempfile = "3.3.0" + +[features] +shlex-bench = ["dep:shlex"] diff --git a/crates/dtmt/src/cmd/bundle/extract.rs b/crates/dtmt/src/cmd/bundle/extract.rs index 5e1c03b..9a0f1dd 100644 --- a/crates/dtmt/src/cmd/bundle/extract.rs +++ b/crates/dtmt/src/cmd/bundle/extract.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; -use color_eyre::eyre::{self, Context, Result}; +use color_eyre::eyre::{self, bail, Context, Result}; use color_eyre::{Help, Report}; use futures::future::try_join_all; use futures::StreamExt; @@ -12,7 +12,9 @@ use sdk::{Bundle, BundleFile, CmdLine}; use tokio::fs; use crate::cmd::util::resolve_bundle_paths; +use crate::shell_parse::ShellParser; +#[inline] fn parse_glob_pattern(s: &str) -> Result { match Pattern::new(s) { Ok(p) => Ok(p), @@ -20,6 +22,7 @@ fn parse_glob_pattern(s: &str) -> Result { } } +#[inline] fn flatten_name(s: &str) -> String { s.replace('/', "_") } @@ -131,26 +134,29 @@ async fn parse_command_line_template(tmpl: &String) -> Result { let mut cmd = if matches!(fs::try_exists(tmpl).await, Ok(true)) { let path = PathBuf::from(tmpl); if path.file_name() == Some(OsStr::new("main.py")) { - let arg = path.display().to_string(); let mut cmd = CmdLine::new("python"); - cmd.arg(shlex::quote(&arg).to_string()); + cmd.arg(path); cmd } else { CmdLine::new(path) } } else { - let Some(args) = shlex::split(tmpl) else { - eyre::bail!("Invalid shell syntax"); - }; + let mut parsed = ShellParser::new(tmpl.as_bytes()); + // Safety: The initial `tmpl` was a `&String` (i.e. valid UTF-8), and `shlex` does not + // insert or remove characters, nor does it split UTF-8 characters. + // So the resulting byte stream is still valid UTF-8. + let mut cmd = CmdLine::new(unsafe { + let bytes = parsed.next().expect("Template is not empty"); + String::from_utf8_unchecked(bytes.to_vec()) + }); - // We already checked that the template is not empty - let mut cmd = CmdLine::new(args[0].clone()); - let mut it = args.iter(); - // Skip the first one, that's the command name - it.next(); + while let Some(arg) = parsed.next() { + // Safety: See above. + cmd.arg(unsafe { String::from_utf8_unchecked(arg.to_vec()) }); + } - for arg in it { - cmd.arg(arg); + if parsed.errored { + bail!("Invalid command line template"); } cmd diff --git a/crates/dtmt/src/main.rs b/crates/dtmt/src/main.rs index bd419e7..2e10b17 100644 --- a/crates/dtmt/src/main.rs +++ b/crates/dtmt/src/main.rs @@ -1,6 +1,7 @@ #![feature(io_error_more)] #![feature(let_chains)] #![feature(result_flattening)] +#![feature(test)] #![windows_subsystem = "console"] use std::path::PathBuf; @@ -27,6 +28,7 @@ mod cmd { mod util; pub mod watch; } +mod shell_parse; #[derive(Default, Deserialize, Serialize)] struct GlobalConfig { diff --git a/crates/dtmt/src/shell_parse.rs b/crates/dtmt/src/shell_parse.rs new file mode 100644 index 0000000..6f35a5f --- /dev/null +++ b/crates/dtmt/src/shell_parse.rs @@ -0,0 +1,189 @@ +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +enum ParserState { + Start, + Word, + SingleQuote, + DoubleQuote, +} + +pub struct ShellParser<'a> { + bytes: &'a [u8], + offset: usize, + pub errored: bool, +} + +impl<'a> ShellParser<'a> { + pub fn new(bytes: &'a [u8]) -> Self { + Self { + bytes, + offset: 0, + errored: false, + } + } + + fn parse_word(&mut self) -> Option<&'a [u8]> { + // The start of the current word. Certain leading characters should be ignored, + // so this might change. + let mut start = self.offset; + let mut state = ParserState::Start; + + while self.offset < self.bytes.len() { + let c = self.bytes[self.offset]; + self.offset += 1; + + match state { + ParserState::Start => match c { + // Ignore leading whitespace + b' ' | b'\t' | b'\n' => start += 1, + b'\'' => { + state = ParserState::SingleQuote; + start += 1; + } + b'"' => { + state = ParserState::DoubleQuote; + start += 1; + } + _ => { + state = ParserState::Word; + } + }, + ParserState::Word => match c { + // Unquoted whitespace ends the current word + b' ' | b'\t' | b'\n' => { + return Some(&self.bytes[start..self.offset - 1]); + } + _ => {} + }, + ParserState::SingleQuote => match c { + b'\'' => { + return Some(&self.bytes[start..(self.offset - 1)]); + } + _ => {} + }, + ParserState::DoubleQuote => match c { + b'"' => { + return Some(&self.bytes[start..(self.offset - 1)]); + } + _ => {} + }, + } + } + + match state { + ParserState::Start => None, + ParserState::Word => Some(&self.bytes[start..self.offset]), + ParserState::SingleQuote | ParserState::DoubleQuote => { + self.errored = true; + None + } + } + } +} + +impl<'a> Iterator for ShellParser<'a> { + type Item = &'a [u8]; + + fn next(&mut self) -> Option { + self.parse_word() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_one_word() { + let mut it = ShellParser::new(b"hello"); + assert_eq!(it.next(), Some("hello".as_bytes())); + assert_eq!(it.next(), None); + } + + #[test] + fn test_one_single() { + let mut it = ShellParser::new(b"'hello'"); + assert_eq!(it.next(), Some("hello".as_bytes())); + assert_eq!(it.next(), None); + } + + #[test] + fn test_open_quote() { + let mut it = ShellParser::new(b"'hello"); + assert_eq!(it.next(), None); + assert!(it.errored) + } + + #[test] + fn test_ww2ogg() { + let mut it = ShellParser::new( + b"ww2ogg.exe --pcb \"/usr/share/ww2ogg/packed_cookbook_aoTuV_603.bin\"", + ); + assert_eq!(it.next(), Some("ww2ogg.exe".as_bytes())); + assert_eq!(it.next(), Some("--pcb".as_bytes())); + assert_eq!( + it.next(), + Some("/usr/share/ww2ogg/packed_cookbook_aoTuV_603.bin".as_bytes()) + ); + assert_eq!(it.next(), None); + } +} + +#[cfg(test)] +mod bench { + extern crate test; + + use super::*; + #[cfg(feature = "shlex-bench")] + use shlex::bytes::Shlex; + use test::Bencher; + + mod ww2ogg { + use super::*; + + #[bench] + fn custom(b: &mut Bencher) { + let val = test::black_box( + b"ww2ogg.exe --pcb \"/usr/share/ww2ogg/packed_cookbook_aoTuV_603.bin\"", + ); + b.iter(|| { + let it = ShellParser::new(val); + let _: Vec<_> = test::black_box(it.collect()); + }) + } + + #[cfg(feature = "shlex-bench")] + #[bench] + fn shlex(b: &mut Bencher) { + let val = test::black_box( + b"ww2ogg.exe --pcb \"/usr/share/ww2ogg/packed_cookbook_aoTuV_603.bin\"", + ); + b.iter(|| { + let it = Shlex::new(val); + let _: Vec<_> = test::black_box(it.collect()); + }) + } + } + + mod one_single { + use super::*; + + #[bench] + fn custom(b: &mut Bencher) { + let val = test::black_box(b"'hello'"); + b.iter(|| { + let it = ShellParser::new(val); + let _: Vec<_> = test::black_box(it.collect()); + }) + } + + #[cfg(feature = "shlex-bench")] + #[bench] + fn shlex(b: &mut Bencher) { + let val = test::black_box(b"'hello'"); + b.iter(|| { + let it = Shlex::new(val); + let _: Vec<_> = test::black_box(it.collect()); + }) + } + } +} From dfd51513dade1eb4ad39f5bfecb19469f7b34fee Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 16:30:39 +0200 Subject: [PATCH 02/19] Update strip-ansi-escapes Removes duplicate dependency of arrayvec. --- Cargo.lock | 21 +++++++-------------- crates/dtmm/Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4da0fb9..ed230d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -117,12 +117,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.4" @@ -1999,7 +1993,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a53776d271cfb873b17c618af0298445c88afc52837f3e948fa3fafd131f449" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", ] [[package]] @@ -2008,7 +2002,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd85a5776cd9500c2e2059c8c76c3b01528566b7fcbaf8098b55a33fc298849b" dependencies = [ - "arrayvec 0.7.4", + "arrayvec", "serde", ] @@ -3346,9 +3340,9 @@ dependencies = [ [[package]] name = "strip-ansi-escapes" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" +checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" dependencies = [ "vte", ] @@ -3543,7 +3537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8493a203431061e901613751931f047d1971337153f96d0e5e363d6dbf6a67" dependencies = [ "arrayref", - "arrayvec 0.7.4", + "arrayvec", "bytemuck", "cfg-if", "png", @@ -4047,11 +4041,10 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "vte" -version = "0.10.1" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" +checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197" dependencies = [ - "arrayvec 0.5.2", "utf8parse", "vte_generate_state_changes", ] diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 05dc160..9e4960a 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -30,7 +30,7 @@ sdk = { path = "../../lib/sdk", version = "*" } serde = { version = "1.0.152", features = ["derive", "rc"] } serde_sjson = { path = "../../lib/serde_sjson", version = "*" } string_template = "0.2.1" -strip-ansi-escapes = "0.1.1" +strip-ansi-escapes = "0.2.0" time = { version = "0.3.20", features = ["serde", "serde-well-known", "local-offset"] } tokio = { version = "1.23.0", features = ["rt", "fs", "tracing", "sync"] } tokio-stream = { version = "0.1.12", features = ["fs"] } From 6030917ade914885b3ef413e9fa222908ead8e5c Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 16:37:28 +0200 Subject: [PATCH 03/19] Update steamlocate The actual update already happened, but `cargo oudated` cannot handle the suffix, so we must update the `Cargo.toml` as well. --- lib/dtmt-shared/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dtmt-shared/Cargo.toml b/lib/dtmt-shared/Cargo.toml index 57b4a8c..b547dbe 100644 --- a/lib/dtmt-shared/Cargo.toml +++ b/lib/dtmt-shared/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" ansi_term = "0.12.1" color-eyre = "0.6.2" serde = "1.0.152" -steamlocate = "2.0.0-alpha.0" +steamlocate = "2.0.0-beta.2" time = { version = "0.3.19", features = ["formatting", "local-offset", "macros"] } tracing = "0.1.37" tracing-error = "0.2.0" From 21df6cfc5c9a2276620f8b78ec7318c8b769da75 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 18:52:58 +0200 Subject: [PATCH 04/19] Update reqwest --- Cargo.lock | 205 ++++++++++++++++++++++++++++++++------- Cargo.toml | 1 + lib/nexusmods/Cargo.toml | 2 +- 3 files changed, 173 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed230d8..d6dd126 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,9 +205,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" @@ -524,13 +524,20 @@ dependencies = [ name = "color-eyre" version = "0.6.2" dependencies = [ + "ansi-parser", "backtrace", "color-spantrace", "eyre", "indenter", "once_cell", "owo-colors", + "pretty_assertions", + "thiserror", + "tracing", "tracing-error", + "tracing-subscriber", + "url", + "wasm-bindgen-test", ] [[package]] @@ -742,6 +749,12 @@ dependencies = [ "serde", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1596,9 +1609,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.26" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" dependencies = [ "bytes", "fnv", @@ -1678,9 +1691,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -1689,12 +1702,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", "pin-project-lite", ] @@ -1704,47 +1729,60 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "hyper" -version = "0.14.28" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-tls" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", + "http-body-util", "hyper", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -2672,6 +2710,26 @@ dependencies = [ "xi-unicode", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.62", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2709,6 +2767,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -2867,11 +2935,11 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -2879,8 +2947,10 @@ dependencies = [ "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-tls", + "hyper-util", "ipnet", "js-sys", "log", @@ -2902,7 +2972,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.50.0", + "winreg 0.52.0", ] [[package]] @@ -2983,13 +3053,20 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustybuzz" version = "0.6.0" @@ -3054,6 +3131,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -3703,6 +3786,28 @@ dependencies = [ "winnow 0.6.8", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -3715,6 +3820,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4150,6 +4256,31 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm-bindgen-test" +version = "0.3.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9bf62a58e0780af3e852044583deee40983e5886da43a271dd772379987667b" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.62", +] + [[package]] name = "web-sys" version = "0.3.69" @@ -4434,9 +4565,9 @@ dependencies = [ [[package]] name = "winreg" -version = "0.50.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "937f3df7948156640f46aacef17a70db0de5917bda9c92b0f751f3a955b588fc" dependencies = [ "cfg-if", "windows-sys 0.48.0", @@ -4444,9 +4575,9 @@ dependencies = [ [[package]] name = "winreg" -version = "0.51.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "937f3df7948156640f46aacef17a70db0de5917bda9c92b0f751f3a955b588fc" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", @@ -4479,6 +4610,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7a2a501ed189703dba8b08142f057e887dfc4b2cc4db2d343ac6376ba3e0b9" +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zip" version = "0.6.6" diff --git a/Cargo.toml b/Cargo.toml index 11831e4..d44a5a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "lib/sdk", "lib/serde_sjson", "lib/luajit2-sys", + "lib/color-eyre", ] exclude = ["lib/color-eyre"] diff --git a/lib/nexusmods/Cargo.toml b/lib/nexusmods/Cargo.toml index d7967ef..2f90a46 100644 --- a/lib/nexusmods/Cargo.toml +++ b/lib/nexusmods/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" futures = "0.3.26" lazy_static = "1.4.0" regex = "1.7.1" -reqwest = { version = "0.11.14" } +reqwest = { version = "0.12.4" } serde = { version = "1.0.152", features = ["derive"] } serde_json = "1.0.94" thiserror = "1.0.39" From bac75e1c9aad1f9a3737e7e78c3e8d5e15b901db Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 18:58:08 +0200 Subject: [PATCH 05/19] Update confy --- Cargo.lock | 36 ++++++++---------------------------- crates/dtmm/Cargo.toml | 2 +- crates/dtmt/Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d6dd126..7f3772a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,14 +572,14 @@ checksum = "9226dbc05df4fb986f48d730b001532580883c4c06c5d1c213f4b34c1c157178" [[package]] name = "confy" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37668cb35145dcfaa1931a5f37fde375eeae8068b4c0d2f289da28a270b2d2c" +checksum = "45b1f4c00870f07dc34adcac82bb6a72cc5aabca8536ba1797e01df51d2ce9a0" dependencies = [ "directories", "serde", "thiserror", - "toml 0.5.11", + "toml", ] [[package]] @@ -768,11 +768,11 @@ dependencies = [ [[package]] name = "directories" -version = "4.0.1" +version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51c5d4ddabd36886dd3e1438cb358cdcb0d7c499cb99cb4ac2e38e18b5cb210" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ - "dirs-sys 0.3.7", + "dirs-sys", ] [[package]] @@ -781,7 +781,7 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys 0.4.1", + "dirs-sys", ] [[package]] @@ -794,17 +794,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "dirs-sys" version = "0.4.1" @@ -3519,7 +3508,7 @@ dependencies = [ "cfg-expr", "heck 0.5.0", "pkg-config", - "toml 0.8.12", + "toml", "version-compare", ] @@ -3732,15 +3721,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - [[package]] name = "toml" version = "0.8.12" diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 9e4960a..627981b 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -13,7 +13,7 @@ bitflags = "1.3.2" clap = { version = "4.0.15", features = ["color", "derive", "std", "cargo", "string", "unicode"] } color-eyre = "0.6.2" colors-transform = "0.2.11" -confy = "0.5.1" +confy = "0.6.1" druid = { version = "0.8", features = ["im", "serde", "image", "png", "jpeg", "bmp", "webp", "svg"] } druid-widget-nursery = "0.1" dtmt-shared = { path = "../../lib/dtmt-shared", version = "*" } diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 688066f..4a98244 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" clap = { version = "4.0.15", features = ["color", "derive", "std", "cargo", "unicode"] } cli-table = { version = "0.4.7", default-features = false, features = ["derive"] } color-eyre = "0.6.2" -confy = "0.5.1" +confy = "0.6.1" csv-async = { version = "1.2.4", features = ["tokio", "serde"] } dtmt-shared = { path = "../../lib/dtmt-shared", version = "*" } futures = "0.3.25" From ecd235be058233677a767663c60d0db9615380a6 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 19:14:07 +0200 Subject: [PATCH 06/19] Update ansi-parser Patched to update heapless while waiting for the merge request. --- Cargo.lock | 49 ++++++++----------------------------------------- Cargo.toml | 1 + 2 files changed, 9 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f3772a..7411d5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,8 +40,7 @@ dependencies = [ [[package]] name = "ansi-parser" version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad5bd94a775101bd68c2de2bb28ca2eccd69f395ae3aec4ac4f6da3c1cd2c6a" +source = "git+https://gitlab.com/lschwiderski/ansi-parser.git?branch=issue/outdated-heapless#af1951d16951a37101139763593be44103e3a477" dependencies = [ "heapless", "nom", @@ -123,18 +122,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "as-slice" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45403b49e3954a4b8428a0ac21a4b7afadccf92bfd96273f1a58cd4812496ae0" -dependencies = [ - "generic-array 0.12.4", - "generic-array 0.13.3", - "generic-array 0.14.7", - "stable_deref_trait", -] - [[package]] name = "associative-cache" version = "1.0.1" @@ -279,7 +266,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] @@ -704,7 +691,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array", "typenum", ] @@ -1389,24 +1376,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "generic-array" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" -dependencies = [ - "typenum", -] - -[[package]] -name = "generic-array" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f797e67af32588215eaaab8327027ee8e71b9dd0b2b26996aedf20c030fce309" -dependencies = [ - "typenum", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -1617,9 +1586,9 @@ dependencies = [ [[package]] name = "hash32" -version = "0.1.1" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4041af86e63ac4298ce40e5cca669066e75b6f1aa3390fe2561ffa5e1d9f4cc" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" dependencies = [ "byteorder", ] @@ -1632,12 +1601,10 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" [[package]] name = "heapless" -version = "0.6.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634bd4d29cbf24424d0a4bfcbf80c6960129dc24424752a7d1d1390607023422" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" dependencies = [ - "as-slice", - "generic-array 0.14.7", "hash32", "stable_deref_trait", ] @@ -1855,7 +1822,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" dependencies = [ - "generic-array 0.14.7", + "generic-array", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d44a5a6..54cf161 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ exclude = ["lib/color-eyre"] [patch.crates-io] color-eyre = { path = "lib/color-eyre" } +ansi-parser = { git = "https://gitlab.com/lschwiderski/ansi-parser.git", branch = "issue/outdated-heapless" } [profile.dev.package.backtrace] opt-level = 3 From 647cb1b8bdaf75e6404b623ec2839953475e23f4 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 19:16:34 +0200 Subject: [PATCH 07/19] Update fastrand --- Cargo.lock | 13 ++----------- lib/sdk/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7411d5e..660e7cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1055,15 +1055,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.1.0" @@ -3108,7 +3099,7 @@ dependencies = [ "byteorder", "color-eyre", "csv-async", - "fastrand 1.9.0", + "fastrand", "futures", "futures-util", "glob", @@ -3492,7 +3483,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", - "fastrand 2.1.0", + "fastrand", "rustix", "windows-sys 0.52.0", ] diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 63f789b..6745ff4 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -8,7 +8,7 @@ bitflags = "1.3.2" byteorder = "1.4.3" color-eyre = "0.6.2" csv-async = { version = "1.2.4", features = ["tokio", "serde"] } -fastrand = "1.8.0" +fastrand = "2.1.0" futures = "0.3.25" futures-util = "0.3.24" glob = "0.3.0" From ae30499a4901d9ef5ce0ad182d6ea81ed9519b24 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 19:19:54 +0200 Subject: [PATCH 08/19] Remove unused dependency --- Cargo.lock | 14 +------------- crates/dtmt/Cargo.toml | 1 - lib/sdk/Cargo.toml | 1 - 3 files changed, 1 insertion(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 660e7cc..336f957 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -399,7 +399,7 @@ checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", - "libloading 0.8.3", + "libloading", ] [[package]] @@ -949,7 +949,6 @@ dependencies = [ "futures", "futures-util", "glob", - "libloading 0.7.4", "luajit2-sys", "nanorand", "notify", @@ -2009,16 +2008,6 @@ version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - [[package]] name = "libloading" version = "0.8.3" @@ -3103,7 +3092,6 @@ dependencies = [ "futures", "futures-util", "glob", - "libloading 0.7.4", "luajit2-sys", "nanorand", "oodle", diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 4a98244..60cd70c 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -13,7 +13,6 @@ dtmt-shared = { path = "../../lib/dtmt-shared", version = "*" } futures = "0.3.25" futures-util = "0.3.24" glob = "0.3.0" -libloading = "0.7.4" nanorand = "0.7.0" oodle = { path = "../../lib/oodle", version = "*" } pin-project-lite = "0.2.9" diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index 6745ff4..f060b4b 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -12,7 +12,6 @@ fastrand = "2.1.0" futures = "0.3.25" futures-util = "0.3.24" glob = "0.3.0" -libloading = "0.7.4" nanorand = "0.7.0" pin-project-lite = "0.2.9" serde = { version = "1.0.147", features = ["derive"] } From 0c4c078b100b71f2226f42bbc9398dae2593c328 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 19:24:57 +0200 Subject: [PATCH 09/19] Update dependencies --- Cargo.lock | 62 +++++++++++++++++++++++++++--------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 336f957..88c8664 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -136,7 +136,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -283,9 +283,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce" [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" [[package]] name = "byteorder" @@ -435,7 +435,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -812,7 +812,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -1274,7 +1274,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -2363,7 +2363,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -2539,7 +2539,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -2663,7 +2663,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -3151,22 +3151,22 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.201" +version = "1.0.202" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780f1cebed1629e4753a1a38a3c72d30b97ec044f0aef68cb26650a3c5cf363c" +checksum = "226b61a0d411b2ba5ff6d7f73a476ac4f8bb900373459cd00fab8512828ba395" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.201" +version = "1.0.202" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e405930b9796f1c00bee880d03fc7e0bb4b9a11afc776885ffe84320da2865" +checksum = "6048858004bcff69094cd972ed40a32500f153bd3be9f716b2eed2e8217c4838" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -3191,9 +3191,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3622f419d1296904700073ea6cc23ad690adbd66f13ea683df73298736f0c1" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -3409,9 +3409,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.62" +version = "2.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f660c3bfcefb88c538776b6685a0c472e3128b51e74d48793dc2a488196e8eb" +checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" dependencies = [ "proc-macro2", "quote", @@ -3502,7 +3502,7 @@ checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -3630,7 +3630,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -3669,21 +3669,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9dd1545e8208b4a5af1aa9bbd0b4cf7e9ea08fabc5d0a5c67fcaafa17433aa3" +checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.12", + "toml_edit 0.22.13", ] [[package]] name = "toml_datetime" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] @@ -3701,9 +3701,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.12" +version = "0.22.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3328d4f68a705b2a4498da1d580585d39a6510f98318a2cec3018a7ec61ddef" +checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" dependencies = [ "indexmap", "serde", @@ -3760,7 +3760,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] @@ -4137,7 +4137,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", "wasm-bindgen-shared", ] @@ -4171,7 +4171,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4204,7 +4204,7 @@ checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.62", + "syn 2.0.63", ] [[package]] From 4bc5777a4b921a28bbc1d5dc675503144f04e9c3 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 20:04:47 +0200 Subject: [PATCH 10/19] Update notify --- Cargo.lock | 75 +++--------------------------------------- crates/dtmt/Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88c8664..8e0f2f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2236,20 +2236,21 @@ dependencies = [ [[package]] name = "notify" -version = "5.2.0" +version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729f63e1ca555a43fe3efa4f3efdf4801c479da85b432242a7b726f353c88486" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "crossbeam-channel", "filetime", "fsevent-sys", "inotify", "kqueue", "libc", + "log", "mio", "walkdir", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -4266,15 +4267,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", -] - [[package]] name = "windows-sys" version = "0.48.0" @@ -4293,21 +4285,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows-targets" version = "0.48.5" @@ -4339,12 +4316,6 @@ dependencies = [ "windows_x86_64_msvc 0.52.5", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4357,12 +4328,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4375,12 +4340,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4399,12 +4358,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4417,12 +4370,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4435,12 +4382,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4453,12 +4394,6 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index 60cd70c..baa21f8 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -30,7 +30,7 @@ zip = "0.6.3" path-clean = "1.0.1" path-slash = "0.2.1" async-recursion = "1.0.2" -notify = "5.1.0" +notify = "6.1.1" luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } shlex = { version = "1.2.0", optional = true } From 3546bc8faa243605015768f3bdf25621ac40397a Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 21:57:45 +0200 Subject: [PATCH 11/19] Update bindgen --- Cargo.lock | 36 +++++++++++++++++++++++++----------- lib/luajit2-sys | 2 +- lib/oodle/Cargo.toml | 2 +- lib/oodle/build.rs | 4 +--- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e0f2f1..a4be1c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -213,23 +213,24 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.64.0" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "cexpr", "clang-sys", + "itertools", "lazy_static", "lazycell", "log", - "peeking_take_while", + "prettyplease", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 1.0.109", + "syn 2.0.63", "which", ] @@ -1890,6 +1891,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2497,12 +2507,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "percent-encoding" version = "2.3.1" @@ -2714,6 +2718,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.63", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" diff --git a/lib/luajit2-sys b/lib/luajit2-sys index 1912016..5d1a075 160000 --- a/lib/luajit2-sys +++ b/lib/luajit2-sys @@ -1 +1 @@ -Subproject commit 19120166f9fc7838b98c71fc348791abc820e323 +Subproject commit 5d1a075742395f767c79d9c0d7466c6fb442f106 diff --git a/lib/oodle/Cargo.toml b/lib/oodle/Cargo.toml index 3283592..6fc5039 100644 --- a/lib/oodle/Cargo.toml +++ b/lib/oodle/Cargo.toml @@ -10,4 +10,4 @@ color-eyre = "0.6.2" tracing = "0.1.37" [build-dependencies] -bindgen = "0.64.0" +bindgen = "0.69.4" diff --git a/lib/oodle/build.rs b/lib/oodle/build.rs index e33c7d5..1a1d4e9 100644 --- a/lib/oodle/build.rs +++ b/lib/oodle/build.rs @@ -1,5 +1,3 @@ -extern crate bindgen; - use std::env; use std::path::PathBuf; @@ -33,7 +31,7 @@ fn main() { .blocklist_file("stdlib.h") // Tell cargo to invalidate the built crate whenever any of the // included header files changed. - .parse_callbacks(Box::new(bindgen::CargoCallbacks)) + .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) // Finish the builder and generate the bindings. .generate() // Unwrap the Result and panic on failure. From b8ac80562ad6514af17220764f25cb8e8ebb0e97 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 22:46:21 +0200 Subject: [PATCH 12/19] Update zip --- Cargo.lock | 135 ++++++++------------------------- Cargo.toml | 3 + crates/dtmm/Cargo.toml | 2 +- crates/dtmt/Cargo.toml | 2 +- crates/dtmt/src/cmd/package.rs | 28 ++----- 5 files changed, 43 insertions(+), 127 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4be1c4..bd1f426 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -110,6 +99,15 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -196,12 +194,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64ct" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" - [[package]] name = "bincode" version = "1.3.3" @@ -382,16 +374,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", -] - [[package]] name = "clang-sys" version = "1.7.0" @@ -580,12 +562,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "core-foundation" version = "0.9.4" @@ -737,6 +713,17 @@ dependencies = [ "serde", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.63", +] + [[package]] name = "diff" version = "0.1.13" @@ -751,7 +738,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -1618,15 +1604,6 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest", -] - [[package]] name = "home" version = "0.5.9" @@ -1807,15 +1784,6 @@ dependencies = [ "libc", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "instant" version = "0.1.12" @@ -2466,17 +2434,6 @@ dependencies = [ "system-deps", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core", - "subtle", -] - [[package]] name = "path-clean" version = "1.0.1" @@ -2495,18 +2452,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" -dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -3225,17 +3170,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha2" version = "0.10.8" @@ -3386,12 +3320,6 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "subtle" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" - [[package]] name = "svgfilters" version = "0.4.0" @@ -4493,40 +4421,37 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zip" -version = "0.6.6" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "f1f4a27345eb6f7aa7bd015ba7eb4175fa4e1b462a29874b779e0bbcf96c6ac7" dependencies = [ - "aes", - "byteorder", + "arbitrary", "bzip2", - "constant_time_eq", "crc32fast", "crossbeam-utils", + "displaydoc", "flate2", - "hmac", - "pbkdf2", - "sha1", + "indexmap", + "thiserror", "time", "zstd", ] [[package]] name = "zstd" -version = "0.11.2+zstd.1.5.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ - "libc", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index 54cf161..39d8f38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,9 @@ members = [ ] exclude = ["lib/color-eyre"] +[workspace.dependencies] +zip = { version = "1.3.0", default-features = false, features = ["deflate", "bzip2", "zstd", "time"] } + [patch.crates-io] color-eyre = { path = "lib/color-eyre" } ansi-parser = { git = "https://gitlab.com/lschwiderski/ansi-parser.git", branch = "issue/outdated-heapless" } diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index 627981b..ef862f3 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -38,4 +38,4 @@ tracing = "0.1.37" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } usvg = "0.25.0" -zip = "0.6.4" +zip = { workspace = true } diff --git a/crates/dtmt/Cargo.toml b/crates/dtmt/Cargo.toml index baa21f8..d836a50 100644 --- a/crates/dtmt/Cargo.toml +++ b/crates/dtmt/Cargo.toml @@ -26,7 +26,7 @@ tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "m tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } tracing = { version = "0.1.37", features = ["async-await"] } -zip = "0.6.3" +zip = { workspace = true } path-clean = "1.0.1" path-slash = "0.2.1" async-recursion = "1.0.2" diff --git a/crates/dtmt/src/cmd/package.rs b/crates/dtmt/src/cmd/package.rs index e922b6e..5ded885 100644 --- a/crates/dtmt/src/cmd/package.rs +++ b/crates/dtmt/src/cmd/package.rs @@ -1,6 +1,5 @@ use std::io::{Cursor, Write}; use std::path::{Path, PathBuf}; -use std::sync::Arc; use clap::{value_parser, Arg, ArgMatches, Command}; use color_eyre::eyre::{Context, Result}; @@ -8,9 +7,9 @@ use color_eyre::Help; use dtmt_shared::ModConfig; use path_slash::{PathBufExt, PathExt}; use tokio::fs; -use tokio::sync::Mutex; use tokio_stream::wrappers::ReadDirStream; use tokio_stream::StreamExt; +use zip::write::SimpleFileOptions; use zip::ZipWriter; use crate::cmd::build::read_project_config; @@ -51,11 +50,7 @@ pub(crate) fn command_definition() -> Command { } #[async_recursion::async_recursion] -async fn process_directory( - zip: Arc>>, - path: P1, - prefix: P2, -) -> Result<()> +async fn process_directory(zip: &mut ZipWriter, path: P1, prefix: P2) -> Result<()> where P1: AsRef + std::marker::Send, P2: AsRef + std::marker::Send, @@ -64,9 +59,7 @@ where let path = path.as_ref(); let prefix = prefix.as_ref(); - zip.lock() - .await - .add_directory(prefix.to_slash_lossy(), Default::default())?; + zip.add_directory(prefix.to_slash_lossy(), SimpleFileOptions::default())?; let read_dir = fs::read_dir(&path) .await @@ -87,12 +80,11 @@ where .await .wrap_err_with(|| format!("Failed to read '{}'", in_path.display()))?; { - let mut zip = zip.lock().await; - zip.start_file(out_path.to_slash_lossy(), Default::default())?; + zip.start_file(out_path.to_slash_lossy(), SimpleFileOptions::default())?; zip.write_all(&data)?; } } else if t.is_dir() { - process_directory(zip.clone(), in_path, out_path).await?; + process_directory(zip, in_path, out_path).await?; } } @@ -107,16 +99,12 @@ where let path = path.as_ref(); let dest = dest.as_ref(); - let data = Cursor::new(Vec::new()); - let zip = ZipWriter::new(data); - let zip = Arc::new(Mutex::new(zip)); + let mut zip = ZipWriter::new(Cursor::new(Vec::with_capacity(1024))); - process_directory(zip.clone(), path, PathBuf::from(&cfg.id)) + process_directory(&mut zip, path, PathBuf::from(&cfg.id)) .await .wrap_err("Failed to add directory to archive")?; - let mut zip = zip.lock().await; - { let name = PathBuf::from(&cfg.id).join("dtmt.cfg"); let path = cfg.dir.join("dtmt.cfg"); @@ -125,7 +113,7 @@ where .await .wrap_err_with(|| format!("Failed to read mod config at {}", path.display()))?; - zip.start_file(name.to_slash_lossy(), Default::default())?; + zip.start_file(name.to_slash_lossy(), SimpleFileOptions::default())?; zip.write_all(&data)?; } From 189c3199a0aec79e6ca2cba6ee9bd3ad455e0e32 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 15 May 2024 22:53:39 +0200 Subject: [PATCH 13/19] Update bitflags --- Cargo.lock | 4 ++-- crates/dtmm/Cargo.toml | 2 +- lib/sdk/Cargo.toml | 2 +- lib/sdk/src/bundle/file.rs | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd1f426..d103e9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -890,7 +890,7 @@ dependencies = [ "ansi-parser", "async-recursion", "bincode", - "bitflags 1.3.2", + "bitflags 2.5.0", "clap", "color-eyre", "colors-transform", @@ -3044,7 +3044,7 @@ name = "sdk" version = "0.3.0" dependencies = [ "async-recursion", - "bitflags 1.3.2", + "bitflags 2.5.0", "byteorder", "color-eyre", "csv-async", diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index ef862f3..c1184ac 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" ansi-parser = "0.9.0" async-recursion = "1.0.5" bincode = "1.3.3" -bitflags = "1.3.2" +bitflags = "2.5.0" clap = { version = "4.0.15", features = ["color", "derive", "std", "cargo", "string", "unicode"] } color-eyre = "0.6.2" colors-transform = "0.2.11" diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index f060b4b..b164d47 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -4,7 +4,7 @@ version = "0.3.0" edition = "2021" [dependencies] -bitflags = "1.3.2" +bitflags = "2.5.0" byteorder = "1.4.3" color-eyre = "0.6.2" csv-async = { version = "1.2.4", features = ["tokio", "serde"] } diff --git a/lib/sdk/src/bundle/file.rs b/lib/sdk/src/bundle/file.rs index ab59884..4d6c56e 100644 --- a/lib/sdk/src/bundle/file.rs +++ b/lib/sdk/src/bundle/file.rs @@ -501,7 +501,7 @@ impl BundleFileVariant { } bitflags! { - #[derive(Default)] + #[derive(Default, Clone, Copy, Debug)] pub struct Properties: u32 { const DATA = 0b100; } From 9577d704235da55e96ad901df74ecc9a2eb5963d Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 16 May 2024 10:58:03 +0200 Subject: [PATCH 14/19] Add missing build tools to CI image --- .ci/image/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/image/Dockerfile b/.ci/image/Dockerfile index 7ab9c0c..f115929 100644 --- a/.ci/image/Dockerfile +++ b/.ci/image/Dockerfile @@ -37,6 +37,7 @@ RUN set -eux; \ apt-get update; \ apt-get install --no-install-recommends -y \ build-essential \ + cmake \ curl \ git \ gpg \ From ef4c2a1d94b316cc52640358cb9521e0c0dd8dd2 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 16 May 2024 11:06:05 +0200 Subject: [PATCH 15/19] Update interprocess --- Cargo.lock | 33 +++++++++++++++++++-------------- crates/dtmm/Cargo.toml | 4 ++-- crates/dtmm/src/main.rs | 26 ++++++++++++++++++-------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d103e9c..7cbb4b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1798,15 +1798,14 @@ dependencies = [ [[package]] name = "interprocess" -version = "1.2.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f2533f3be42fffe3b5e63b71aeca416c1c3bc33e4e27be018521e76b1f38fb" +checksum = "7b4d0250d41da118226e55b3d50ca3f0d9e0a0f6829b92f543ac0054aeea1572" dependencies = [ - "cfg-if", "libc", - "rustc_version", - "to_method", - "winapi", + "recvmsg", + "widestring", + "windows-sys 0.52.0", ] [[package]] @@ -2093,9 +2092,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minijinja" -version = "1.0.21" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e877d961d4f96ce13615862322df7c0b6d169d40cab71a7ef3f9b9e594451e" +checksum = "7165d0e94806d52ad5295e4b54a95176d831814840bc067298ca647e1c956338" dependencies = [ "serde", ] @@ -2765,6 +2764,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b42e27ef78c35d3998403c1d26f3efd9e135d3e5121b0a4845cc5cc27547f4f" +[[package]] +name = "recvmsg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" + [[package]] name = "redox_syscall" version = "0.4.1" @@ -3540,12 +3545,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "to_method" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c4ceeeca15c8384bbc3e011dbd8fccb7f068a440b752b7d9b32ceb0ca0e2e8" - [[package]] name = "tokio" version = "1.37.0" @@ -4178,6 +4177,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "widestring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" + [[package]] name = "winapi" version = "0.3.9" diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index c1184ac..f57d0c8 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -18,10 +18,10 @@ druid = { version = "0.8", features = ["im", "serde", "image", "png", "jpeg", "b druid-widget-nursery = "0.1" dtmt-shared = { path = "../../lib/dtmt-shared", version = "*" } futures = "0.3.25" -interprocess = { version = "1.2.1", default-features = false } +interprocess = "2.1.0" lazy_static = "1.4.0" luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } -minijinja = "1.0.10" +minijinja = { version = "2.0.1", default-features = false } nexusmods = { path = "../../lib/nexusmods", version = "*" } oodle = { path = "../../lib/oodle", version = "*" } open = "5.0.1" diff --git a/crates/dtmm/src/main.rs b/crates/dtmm/src/main.rs index aa223f0..6c0bb48 100644 --- a/crates/dtmm/src/main.rs +++ b/crates/dtmm/src/main.rs @@ -11,7 +11,7 @@ use clap::{command, value_parser, Arg}; use color_eyre::eyre::{self, Context}; use color_eyre::{Report, Result, Section}; use druid::AppLauncher; -use interprocess::local_socket::{LocalSocketListener, LocalSocketStream}; +use interprocess::local_socket::{prelude::*, GenericNamespaced, ListenerOptions}; use tokio::sync::RwLock; use crate::controller::worker::work_thread; @@ -29,9 +29,9 @@ mod util { } mod ui; -// As explained in https://docs.rs/interprocess/latest/interprocess/local_socket/enum.NameTypeSupport.html +// As explained in https://docs.rs/interprocess/2.1.0/interprocess/local_socket/struct.Name.html // namespaces are supported on both platforms we care about: Windows and Linux. -const IPC_ADDRESS: &str = "@dtmm.sock"; +const IPC_ADDRESS: &str = "dtmm.sock"; #[tracing::instrument] fn notify_nxm_download( @@ -42,9 +42,13 @@ fn notify_nxm_download( tracing::debug!("Received Uri '{}', sending to main process.", uri.as_ref()); - let mut stream = LocalSocketStream::connect(IPC_ADDRESS) - .wrap_err_with(|| format!("Failed to connect to '{}'", IPC_ADDRESS)) - .suggestion("Make sure the main window is open.")?; + let mut stream = LocalSocketStream::connect( + IPC_ADDRESS + .to_ns_name::() + .expect("Invalid socket name"), + ) + .wrap_err_with(|| format!("Failed to connect to '{}'", IPC_ADDRESS)) + .suggestion("Make sure the main window is open.")?; tracing::debug!("Connected to main process at '{}'", IPC_ADDRESS); @@ -130,8 +134,14 @@ fn main() -> Result<()> { let _guard = span.enter(); let event_sink = event_sink.clone(); - let server = - LocalSocketListener::bind(IPC_ADDRESS).wrap_err("Failed to create IPC listener")?; + let server = ListenerOptions::new() + .name( + IPC_ADDRESS + .to_ns_name::() + .expect("Invalid socket name"), + ) + .create_sync() + .wrap_err("Failed to create IPC listener")?; tracing::debug!("IPC server listening on '{}'", IPC_ADDRESS); From 96a7eeb1e0cf1f1af484febb8278a9bb679c0586 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Fri, 17 May 2024 14:51:17 +0200 Subject: [PATCH 16/19] Implement faster hex string parsing --- Cargo.toml | 6 + Justfile | 8 + lib/sdk/src/lib.rs | 2 + lib/sdk/src/murmur/mod.rs | 377 +----------------------------------- lib/sdk/src/murmur/types.rs | 365 ++++++++++++++++++++++++++++++++++ lib/sdk/src/murmur/util.rs | 132 +++++++++++++ 6 files changed, 518 insertions(+), 372 deletions(-) create mode 100644 lib/sdk/src/murmur/types.rs create mode 100644 lib/sdk/src/murmur/util.rs diff --git a/Cargo.toml b/Cargo.toml index 39d8f38..8cbe52c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,3 +31,9 @@ strip = "debuginfo" [profile.release-lto] inherits = "release" lto = true + +[profile.perf] +inherits = "release" +strip = false +lto = true +debug = "line-tables-only" diff --git a/Justfile b/Justfile index f9b37bc..dbecc22 100644 --- a/Justfile +++ b/Justfile @@ -1,5 +1,13 @@ +set positional-arguments + fly_target := "main" +build-perf-dtmt: + cargo build --profile perf --bin dtmt + +perf-dtmt *args='': build-perf-dtmt + perf record --call-graph dwarf ./target/perf/dtmt "$@" + ci-build: ci-build-msvc ci-build-linux ci-build-msvc: diff --git a/lib/sdk/src/lib.rs b/lib/sdk/src/lib.rs index 37a4d67..a24b3bd 100644 --- a/lib/sdk/src/lib.rs +++ b/lib/sdk/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(test)] + mod binary; mod bundle; mod context; diff --git a/lib/sdk/src/murmur/mod.rs b/lib/sdk/src/murmur/mod.rs index a2a9ef3..87a8473 100644 --- a/lib/sdk/src/murmur/mod.rs +++ b/lib/sdk/src/murmur/mod.rs @@ -1,15 +1,16 @@ use std::fmt; use color_eyre::eyre::Context; -use color_eyre::Report; +use color_eyre::{Report, Result}; use serde::de::Visitor; -use serde::{Deserialize, Serialize}; -use serde::{Deserializer, Serializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; mod dictionary; // Currently unused // mod murmurhash32; mod murmurhash64; +mod types; +mod util; pub const SEED: u32 = 0; @@ -18,372 +19,4 @@ pub use murmurhash64::hash; pub use murmurhash64::hash32; pub use murmurhash64::hash_inverse as inverse; -fn _swap_bytes_u32(value: u32) -> u32 { - u32::from_le_bytes(value.to_be_bytes()) -} - -fn _swap_bytes_u64(value: u64) -> u64 { - u64::from_le_bytes(value.to_be_bytes()) -} - -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] -pub struct Murmur64(u64); - -impl Murmur64 { - pub fn hash(s: B) -> Self - where - B: AsRef<[u8]>, - { - hash(s.as_ref(), SEED as u64).into() - } -} - -impl From for Murmur64 { - fn from(value: u64) -> Self { - Self(value) - } -} - -impl From for u64 { - fn from(value: Murmur64) -> Self { - value.0 - } -} - -impl TryFrom<&str> for Murmur64 { - type Error = Report; - - fn try_from(value: &str) -> Result { - u64::from_str_radix(value, 16) - .map(Self) - .wrap_err_with(|| format!("Failed to convert value to Murmur64: {value}")) - } -} - -impl fmt::UpperHex for Murmur64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::UpperHex::fmt(&self.0, f) - } -} - -impl fmt::LowerHex for Murmur64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::LowerHex::fmt(&self.0, f) - } -} - -impl fmt::Display for Murmur64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::UpperHex::fmt(&self.0, f) - } -} - -impl<'de> Visitor<'de> for Murmur64 { - type Value = Self; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str( - "an usigned 64 bit integer \ - or a string in hexadecimal format encoding such an integer", - ) - } - - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - let bytes = value.to_le_bytes(); - Ok(Self::from(u64::from_le_bytes(bytes))) - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - Ok(Self::from(value)) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - match Murmur64::try_from(value) { - Ok(hash) => Ok(hash), - Err(err) => Err(E::custom(format!( - "failed to convert '{value}' to Murmur64: {err}" - ))), - } - } -} - -impl<'de> Deserialize<'de> for Murmur64 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(Self(0)) - } -} - -impl Serialize for Murmur64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&format!("{self:016X}")) - } -} - -#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] -pub struct Murmur32(u32); - -impl Murmur32 { - pub fn hash(s: B) -> Self - where - B: AsRef<[u8]>, - { - hash32(s.as_ref(), SEED).into() - } -} - -impl From for Murmur32 { - fn from(value: u32) -> Self { - Self(value) - } -} - -impl From for u32 { - fn from(value: Murmur32) -> Self { - value.0 - } -} - -impl TryFrom<&str> for Murmur32 { - type Error = Report; - - fn try_from(value: &str) -> Result { - u32::from_str_radix(value, 16) - .map(Self) - .wrap_err_with(|| format!("Failed to convert value to Murmur32: {value}")) - } -} - -impl fmt::UpperHex for Murmur32 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::UpperHex::fmt(&self.0, f) - } -} - -impl fmt::Display for Murmur32 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::UpperHex::fmt(&self.0, f) - } -} - -impl Serialize for Murmur32 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&format!("{self:08X}")) - } -} - -impl<'de> Visitor<'de> for Murmur32 { - type Value = Self; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str( - "an usigned 32 bit integer \ - or a string in hexadecimal format encoding such an integer", - ) - } - - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - let bytes = value.to_le_bytes(); - self.visit_u32(u64::from_le_bytes(bytes) as u32) - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - self.visit_u32(value as u32) - } - - fn visit_u32(self, value: u32) -> Result - where - E: serde::de::Error, - { - Ok(Self::from(value)) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - match Murmur32::try_from(value) { - Ok(hash) => Ok(hash), - Err(err) => Err(E::custom(format!( - "failed to convert '{value}' to Murmur32: {err}" - ))), - } - } -} - -impl<'de> Deserialize<'de> for Murmur32 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(Self(0)) - } -} - -// This type encodes the fact that when reading in a bundle, we don't always have a dictionary -// entry for every hash in there. So we do want to have the real string available when needed, -// but at the same time retain the original hash information for when we don't. -// This is especially important when wanting to write back the read bundle, as the hashes need to -// stay the same. -// The previous system of always turning hashes into strings worked well for the purpose of -// displaying hashes, but would have made it very hard to turn a stringyfied hash back into -// an actual hash. -#[derive(Clone, Debug, Eq)] -pub enum IdString64 { - Hash(Murmur64), - String(String), -} - -impl IdString64 { - pub fn to_murmur64(&self) -> Murmur64 { - match self { - Self::Hash(hash) => *hash, - Self::String(s) => Murmur64::hash(s.as_bytes()), - } - } - - pub fn display(&self) -> IdString64Display { - let s = match self { - IdString64::Hash(hash) => hash.to_string(), - IdString64::String(s) => s.clone(), - }; - - IdString64Display(s) - } - - pub fn is_string(&self) -> bool { - match self { - IdString64::Hash(_) => false, - IdString64::String(_) => true, - } - } - - pub fn is_hash(&self) -> bool { - match self { - IdString64::Hash(_) => true, - IdString64::String(_) => false, - } - } -} - -impl> From for IdString64 { - fn from(value: S) -> Self { - Self::String(value.into()) - } -} - -impl From for IdString64 { - fn from(value: Murmur64) -> Self { - Self::Hash(value) - } -} - -impl From for Murmur64 { - fn from(value: IdString64) -> Self { - value.to_murmur64() - } -} - -impl PartialEq for IdString64 { - fn eq(&self, other: &Self) -> bool { - self.to_murmur64() == other.to_murmur64() - } -} - -impl std::hash::Hash for IdString64 { - fn hash(&self, state: &mut H) { - state.write_u64(self.to_murmur64().into()); - } -} - -impl serde::Serialize for IdString64 { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_u64(self.to_murmur64().into()) - } -} - -struct IdString64Visitor; - -impl<'de> serde::de::Visitor<'de> for IdString64Visitor { - type Value = IdString64; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("an u64 or a string") - } - - fn visit_u64(self, value: u64) -> Result - where - E: serde::de::Error, - { - Ok(IdString64::Hash(value.into())) - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - Ok(IdString64::String(v.to_string())) - } - - fn visit_string(self, v: String) -> Result - where - E: serde::de::Error, - { - Ok(IdString64::String(v)) - } -} - -impl<'de> serde::Deserialize<'de> for IdString64 { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_u64(IdString64Visitor) - } -} - -pub struct IdString64Display(String); - -impl std::fmt::Display for IdString64Display { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::fmt::UpperHex for IdString64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - std::fmt::UpperHex::fmt(&self.to_murmur64(), f) - } -} - -impl std::fmt::LowerHex for IdString64 { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - std::fmt::LowerHex::fmt(&self.to_murmur64(), f) - } -} +pub use types::*; diff --git a/lib/sdk/src/murmur/types.rs b/lib/sdk/src/murmur/types.rs new file mode 100644 index 0000000..1146494 --- /dev/null +++ b/lib/sdk/src/murmur/types.rs @@ -0,0 +1,365 @@ +use self::util::{parse_hex32, parse_hex64}; + +use super::*; + +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +pub struct Murmur64(u64); + +impl Murmur64 { + pub fn hash(s: B) -> Self + where + B: AsRef<[u8]>, + { + hash(s.as_ref(), SEED as u64).into() + } +} + +impl From for Murmur64 { + fn from(value: u64) -> Self { + Self(value) + } +} + +impl From for u64 { + fn from(value: Murmur64) -> Self { + value.0 + } +} + +impl TryFrom<&str> for Murmur64 { + type Error = Report; + + fn try_from(value: &str) -> Result { + parse_hex64(value) + .map(Self) + .wrap_err_with(|| format!("Failed to convert value to Murmur64: {value}")) + } +} + +impl fmt::UpperHex for Murmur64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl fmt::LowerHex for Murmur64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::LowerHex::fmt(&self.0, f) + } +} + +impl fmt::Display for Murmur64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl<'de> Visitor<'de> for Murmur64 { + type Value = Self; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str( + "an usigned 64 bit integer \ + or a string in hexadecimal format encoding such an integer", + ) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + let bytes = value.to_le_bytes(); + Ok(Self::from(u64::from_le_bytes(bytes))) + } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + Ok(Self::from(value)) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + match Murmur64::try_from(value) { + Ok(hash) => Ok(hash), + Err(err) => Err(E::custom(format!( + "failed to convert '{value}' to Murmur64: {err}" + ))), + } + } +} + +impl<'de> Deserialize<'de> for Murmur64 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(Self(0)) + } +} + +impl Serialize for Murmur64 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&format!("{self:016X}")) + } +} + +#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] +pub struct Murmur32(u32); + +impl Murmur32 { + pub fn hash(s: B) -> Self + where + B: AsRef<[u8]>, + { + hash32(s.as_ref(), SEED).into() + } +} + +impl From for Murmur32 { + fn from(value: u32) -> Self { + Self(value) + } +} + +impl From for u32 { + fn from(value: Murmur32) -> Self { + value.0 + } +} + +impl TryFrom<&str> for Murmur32 { + type Error = Report; + + fn try_from(value: &str) -> Result { + parse_hex32(value) + .map(Self) + .wrap_err_with(|| format!("Failed to convert value to Murmur32: {value}")) + } +} + +impl fmt::UpperHex for Murmur32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl fmt::Display for Murmur32 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::UpperHex::fmt(&self.0, f) + } +} + +impl Serialize for Murmur32 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&format!("{self:08X}")) + } +} + +impl<'de> Visitor<'de> for Murmur32 { + type Value = Self; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str( + "an usigned 32 bit integer \ + or a string in hexadecimal format encoding such an integer", + ) + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + let bytes = value.to_le_bytes(); + self.visit_u32(u64::from_le_bytes(bytes) as u32) + } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + self.visit_u32(value as u32) + } + + fn visit_u32(self, value: u32) -> Result + where + E: serde::de::Error, + { + Ok(Self::from(value)) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + match Murmur32::try_from(value) { + Ok(hash) => Ok(hash), + Err(err) => Err(E::custom(format!( + "failed to convert '{value}' to Murmur32: {err}" + ))), + } + } +} + +impl<'de> Deserialize<'de> for Murmur32 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(Self(0)) + } +} + +// This type encodes the fact that when reading in a bundle, we don't always have a dictionary +// entry for every hash in there. So we do want to have the real string available when needed, +// but at the same time retain the original hash information for when we don't. +// This is especially important when wanting to write back the read bundle, as the hashes need to +// stay the same. +// The previous system of always turning hashes into strings worked well for the purpose of +// displaying hashes, but would have made it very hard to turn a stringyfied hash back into +// an actual hash. +#[derive(Clone, Debug, Eq)] +pub enum IdString64 { + Hash(Murmur64), + String(String), +} + +impl IdString64 { + pub fn to_murmur64(&self) -> Murmur64 { + match self { + Self::Hash(hash) => *hash, + Self::String(s) => Murmur64::hash(s.as_bytes()), + } + } + + pub fn display(&self) -> IdString64Display { + let s = match self { + IdString64::Hash(hash) => hash.to_string(), + IdString64::String(s) => s.clone(), + }; + + IdString64Display(s) + } + + pub fn is_string(&self) -> bool { + match self { + IdString64::Hash(_) => false, + IdString64::String(_) => true, + } + } + + pub fn is_hash(&self) -> bool { + match self { + IdString64::Hash(_) => true, + IdString64::String(_) => false, + } + } +} + +impl> From for IdString64 { + fn from(value: S) -> Self { + Self::String(value.into()) + } +} + +impl From for IdString64 { + fn from(value: Murmur64) -> Self { + Self::Hash(value) + } +} + +impl From for Murmur64 { + fn from(value: IdString64) -> Self { + value.to_murmur64() + } +} + +impl PartialEq for IdString64 { + fn eq(&self, other: &Self) -> bool { + self.to_murmur64() == other.to_murmur64() + } +} + +impl std::hash::Hash for IdString64 { + fn hash(&self, state: &mut H) { + state.write_u64(self.to_murmur64().into()); + } +} + +impl serde::Serialize for IdString64 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_u64(self.to_murmur64().into()) + } +} + +struct IdString64Visitor; + +impl<'de> serde::de::Visitor<'de> for IdString64Visitor { + type Value = IdString64; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an u64 or a string") + } + + fn visit_u64(self, value: u64) -> Result + where + E: serde::de::Error, + { + Ok(IdString64::Hash(value.into())) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + Ok(IdString64::String(v.to_string())) + } + + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + Ok(IdString64::String(v)) + } +} + +impl<'de> serde::Deserialize<'de> for IdString64 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_u64(IdString64Visitor) + } +} + +pub struct IdString64Display(String); + +impl std::fmt::Display for IdString64Display { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl std::fmt::UpperHex for IdString64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + std::fmt::UpperHex::fmt(&self.to_murmur64(), f) + } +} + +impl std::fmt::LowerHex for IdString64 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + std::fmt::LowerHex::fmt(&self.to_murmur64(), f) + } +} diff --git a/lib/sdk/src/murmur/util.rs b/lib/sdk/src/murmur/util.rs new file mode 100644 index 0000000..134c4e7 --- /dev/null +++ b/lib/sdk/src/murmur/util.rs @@ -0,0 +1,132 @@ +use color_eyre::eyre::bail; +use color_eyre::Result; + +// Generates tables similar to these: +// https://github.com/zbjornson/fast-hex/blob/a3487bca95127634a61bfeae8f8bfc8f0e5baa3f/src/hex.cc#L20-L89 +// `upper` determines upper vs. lower bits (first character is `upper`). +const fn generate_byte_map(upper: bool) -> [u8; 256] { + let mut out = [0u8; 256]; + let factor = if upper { 16 } else { 1 }; + + let mut i = 0; + + while i < 256 { + match i { + 0x30..=0x39 => out[i] = factor * (i as u8 - 0x30), + 0x41..=0x46 => out[i] = factor * (9 + i as u8 - 0x40), + 0x61..=0x66 => out[i] = factor * (9 + i as u8 - 0x60), + _ => out[i] = u8::MAX, + } + i += 1; + } + + out +} + +const BYTE_MAP_UPPER: [u8; 256] = generate_byte_map(true); +const BYTE_MAP_LOWER: [u8; 256] = generate_byte_map(false); + +macro_rules! make_parse_hex { + ($name:ident, $ty:ty, $len:expr) => { + #[inline] + pub fn $name(s: impl AsRef) -> Result<$ty> { + // For the string to be valid hex characters, it needs to be ASCII. + // So we can simply treat it as a byte stream. + let s = s.as_ref().as_bytes(); + + if s.len() != $len { + bail!( + "String length doesn't match. Expected {}, got {}", + $len, + s.len() + ); + } + + let n = $len / 2; + let mut out: $ty = 0; + let mut i = 0; + + while i < n { + let j = i * 2; + + let c1 = BYTE_MAP_UPPER[s[j] as usize]; + if c1 == u8::MAX { + bail!("Invalid character '{:?}' ({})", char::from(c1), c1); + } + + let c2 = BYTE_MAP_LOWER[s[j + 1] as usize]; + if c2 == u8::MAX { + bail!("Invalid character '{:?}' ({})", char::from(c2), c2); + } + + out |= ((c1 + c2) as $ty) << (n - i - 1) * 8; + + i += 1; + } + + Ok(out) + } + }; +} + +make_parse_hex!(parse_hex64, u64, 16); +make_parse_hex!(parse_hex32, u32, 8); + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn parse_32() { + let hash = "A14E8DFA"; + assert_eq!(parse_hex32(hash).unwrap(), 0xA14E8DFA); + } + + #[test] + fn parse_64() { + let hash = "A14E8DFA2CD117E2"; + assert_eq!(parse_hex64(hash).unwrap(), 0xA14E8DFA2CD117E2); + } + + #[test] + fn std_from_radix_32() { + let hash = "A14E8DFA"; + assert_eq!(u32::from_str_radix(hash, 16).unwrap(), 0xA14E8DFA); + } + + #[test] + fn std_from_radix_64() { + let hash = "A14E8DFA2CD117E2"; + assert_eq!(u64::from_str_radix(hash, 16).unwrap(), 0xA14E8DFA2CD117E2); + } +} + +#[cfg(test)] +mod bench { + use super::{parse_hex32, parse_hex64}; + + extern crate test; + + const HASH32: &str = "A14E8DFA"; + const HASH64: &str = "A14E8DFA2CD117E2"; + + #[bench] + fn custom_32(b: &mut test::Bencher) { + b.iter(|| test::black_box(parse_hex32(test::black_box(HASH32)))) + } + + #[bench] + fn std_32(b: &mut test::Bencher) { + b.iter(|| test::black_box(u32::from_str_radix(test::black_box(HASH32), 16))) + } + + #[bench] + fn custom_64(b: &mut test::Bencher) { + b.iter(|| test::black_box(parse_hex64(test::black_box(HASH64)))) + } + + #[bench] + fn std_64(b: &mut test::Bencher) { + b.iter(|| test::black_box(u64::from_str_radix(test::black_box(HASH64), 16))) + } +} From b40375122877d62c6715ad7ff24b9177a97c8287 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 8 Jul 2024 16:55:32 +0200 Subject: [PATCH 17/19] Update ansi-parser --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cbb4b5..2df8581 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,8 +28,8 @@ dependencies = [ [[package]] name = "ansi-parser" -version = "0.9.0" -source = "git+https://gitlab.com/lschwiderski/ansi-parser.git?branch=issue/outdated-heapless#af1951d16951a37101139763593be44103e3a477" +version = "0.9.1" +source = "git+https://gitlab.com/lschwiderski/ansi-parser.git?branch=issue/outdated-heapless#bfcd2689677fa93ce72c55833e891af36c65b2aa" dependencies = [ "heapless", "nom", diff --git a/Cargo.toml b/Cargo.toml index 8cbe52c..8c0e7c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ zip = { version = "1.3.0", default-features = false, features = ["deflate", "bzi [patch.crates-io] color-eyre = { path = "lib/color-eyre" } -ansi-parser = { git = "https://gitlab.com/lschwiderski/ansi-parser.git", branch = "issue/outdated-heapless" } +ansi-parser = { git = "https://gitlab.com/lschwiderski/ansi-parser.git", branch = "issue/outdated-heapless", version = "0.9.1" } [profile.dev.package.backtrace] opt-level = 3 From 94b64b461917ad17c2137a01ad0ce2b09700f152 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 10 Jul 2024 18:40:52 +0200 Subject: [PATCH 18/19] Update zip --- Cargo.lock | 42 ++++++++++++++++++++++++++++++++---------- Cargo.toml | 2 +- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2df8581..faad8f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -640,9 +640,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -658,9 +658,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -2011,6 +2011,12 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -3435,18 +3441,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579e9083ca58dd9dcf91a9923bb9054071b9ebbd800b342194c9feb0ee89fc18" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2470041c06ec3ac1ab38d0356a6119054dedaea53e12fbefc0de730a1c08524" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", @@ -4426,9 +4432,9 @@ checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" [[package]] name = "zip" -version = "1.3.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f4a27345eb6f7aa7bd015ba7eb4175fa4e1b462a29874b779e0bbcf96c6ac7" +checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" dependencies = [ "arbitrary", "bzip2", @@ -4437,11 +4443,27 @@ dependencies = [ "displaydoc", "flate2", "indexmap", + "memchr", "thiserror", "time", + "zopfli", "zstd", ] +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.1" diff --git a/Cargo.toml b/Cargo.toml index 8c0e7c5..f38ac22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ exclude = ["lib/color-eyre"] [workspace.dependencies] -zip = { version = "1.3.0", default-features = false, features = ["deflate", "bzip2", "zstd", "time"] } +zip = { version = "2.1.3", default-features = false, features = ["deflate", "bzip2", "zstd", "time"] } [patch.crates-io] color-eyre = { path = "lib/color-eyre" } From 0f14834e2d162fe6b734407ce6359cbc49127e91 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Wed, 10 Jul 2024 18:41:38 +0200 Subject: [PATCH 19/19] Remove string_template Use minijinja for all templates --- Cargo.lock | 1 - crates/dtmm/Cargo.toml | 1 - .../assets/{mod_main.lua => mod_main.lua.j2} | 4 +++- crates/dtmm/src/controller/deploy.rs | 24 ++++++++++++------- 4 files changed, 18 insertions(+), 12 deletions(-) rename crates/dtmm/assets/{mod_main.lua => mod_main.lua.j2} (98%) diff --git a/Cargo.lock b/Cargo.lock index faad8f6..55cdf0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,7 +910,6 @@ dependencies = [ "sdk", "serde", "serde_sjson", - "string_template", "strip-ansi-escapes", "time", "tokio", diff --git a/crates/dtmm/Cargo.toml b/crates/dtmm/Cargo.toml index f57d0c8..661eb17 100644 --- a/crates/dtmm/Cargo.toml +++ b/crates/dtmm/Cargo.toml @@ -29,7 +29,6 @@ path-slash = "0.2.1" sdk = { path = "../../lib/sdk", version = "*" } serde = { version = "1.0.152", features = ["derive", "rc"] } serde_sjson = { path = "../../lib/serde_sjson", version = "*" } -string_template = "0.2.1" strip-ansi-escapes = "0.2.0" time = { version = "0.3.20", features = ["serde", "serde-well-known", "local-offset"] } tokio = { version = "1.23.0", features = ["rt", "fs", "tracing", "sync"] } diff --git a/crates/dtmm/assets/mod_main.lua b/crates/dtmm/assets/mod_main.lua.j2 similarity index 98% rename from crates/dtmm/assets/mod_main.lua rename to crates/dtmm/assets/mod_main.lua.j2 index e4006f6..c1131be 100644 --- a/crates/dtmm/assets/mod_main.lua +++ b/crates/dtmm/assets/mod_main.lua.j2 @@ -17,7 +17,7 @@ local require_store = {} -- This token is treated as a string template and filled by DTMM during deployment. -- This allows hiding unsafe I/O functions behind a setting. --- It's also a valid table definition, thereby degrading gracefully when not replaced. +-- When not replaced, it's also a valid table definition, thereby degrading gracefully. local is_io_enabled = {{ is_io_enabled }} -- luacheck: ignore 113 local lua_libs = { debug = debug, @@ -207,3 +207,5 @@ function init() Main:init() end + +-- vim: ft=lua diff --git a/crates/dtmm/src/controller/deploy.rs b/crates/dtmm/src/controller/deploy.rs index 4d5f831..a59a1fe 100644 --- a/crates/dtmm/src/controller/deploy.rs +++ b/crates/dtmm/src/controller/deploy.rs @@ -1,4 +1,3 @@ -use std::collections::HashMap; use std::io::{Cursor, ErrorKind}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -16,7 +15,6 @@ use sdk::{ Bundle, BundleDatabase, BundleFile, BundleFileType, BundleFileVariant, FromBinary, ToBinary, }; use serde::{Deserialize, Serialize}; -use string_template::Template; use time::OffsetDateTime; use tokio::fs::{self, DirEntry}; use tokio::io::AsyncWriteExt; @@ -572,12 +570,17 @@ async fn patch_boot_bundle(state: Arc) -> Result> { let span = tracing::debug_span!("Importing mod main script"); let _enter = span.enter(); - let is_io_enabled = format!("{}", state.is_io_enabled); - let mut data = HashMap::new(); - data.insert("is_io_enabled", is_io_enabled.as_str()); + let mut env = Environment::new(); + env.add_template("mod_main.lua", include_str!("../../assets/mod_main.lua.j2")) + .wrap_err("Failed to compile template for `mod_main.lua`")?; + let tmpl = env + .get_template("mod_main.lua") + .wrap_err("Failed to get template `mod_main.lua`")?; + + let lua = tmpl + .render(minijinja::context!(is_io_enabled => if state.is_io_enabled { "true" } else {"false"})) + .wrap_err("Failed to render template `mod_main.lua`")?; - let tmpl = include_str!("../../assets/mod_main.lua"); - let lua = Template::new(tmpl).render(&data); tracing::trace!("Main script rendered:\n===========\n{}\n=============", lua); let file = lua::compile(MOD_BOOT_SCRIPT, lua).wrap_err("Failed to compile mod main Lua file")?; @@ -707,7 +710,7 @@ pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { }, async { let path = state.game_dir.join(DEPLOYMENT_DATA_PATH); - match read_sjson_file::<_, DeploymentData>(path).await { + match read_sjson_file::<_, DeploymentData>(&path).await { Ok(data) => Ok(Some(data)), Err(err) => { if let Some(err) = err.downcast_ref::() @@ -715,7 +718,10 @@ pub(crate) async fn deploy_mods(state: ActionState) -> Result<()> { { Ok(None) } else { - Err(err).wrap_err("Failed to read deployment data") + Err(err).wrap_err(format!( + "Failed to read deployment data from: {}", + path.display() + )) } } }