chore: Update tree-sitter-rust

Not much development is happening in that plugin.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tree-sitter-rust"]
+	path = tree-sitter-rust
+	url = git@github.com:sclu1034/tree-sitter-rust.git

Cargo.lock

@@ -26,6 +26,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "anes"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
+
 [[package]]
 name = "anstream"
 version = "0.2.6"
@@ -66,6 +72,17 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -93,6 +110,18 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bumpalo"
+version = "3.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535"
+
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -105,6 +134,45 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "ciborium"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f"
+dependencies = [
+ "ciborium-io",
+ "ciborium-ll",
+ "serde",
+]
+
+[[package]]
+name = "ciborium-io"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369"
+
+[[package]]
+name = "ciborium-ll"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b"
+dependencies = [
+ "ciborium-io",
+ "half",
+]
+
+[[package]]
+name = "clap"
+version = "3.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+dependencies = [
+ "bitflags",
+ "clap_lex 0.2.4",
+ "indexmap",
+ "textwrap",
+]
+
 [[package]]
 name = "clap"
 version = "4.2.1"
@@ -125,7 +193,7 @@ dependencies = [
  "anstream",
  "anstyle",
  "bitflags",
- "clap_lex",
+ "clap_lex 0.4.1",
  "once_cell",
  "strsim",
  "unicase",
@@ -144,6 +212,15 @@ dependencies = [
  "syn 2.0.14",
 ]
 
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
 [[package]]
 name = "clap_lex"
 version = "0.4.1"
@@ -192,6 +269,42 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb"
+dependencies = [
+ "anes",
+ "atty",
+ "cast",
+ "ciborium",
+ "clap 3.2.23",
+ "criterion-plot",
+ "itertools",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
+dependencies = [
+ "cast",
+ "itertools",
+]
+
 [[package]]
 name = "crossbeam"
 version = "0.8.2"
@@ -259,6 +372,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "either"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
+
 [[package]]
 name = "errno"
 version = "0.3.1"
@@ -296,6 +415,12 @@ version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4"
 
+[[package]]
+name = "half"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+
 [[package]]
 name = "hashbrown"
 version = "0.12.3"
@@ -308,6 +433,24 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.1"
@@ -336,7 +479,7 @@ version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.1",
  "libc",
  "windows-sys 0.48.0",
 ]
@@ -347,18 +490,43 @@ version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.1",
  "io-lifetimes",
  "rustix",
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
+
+[[package]]
+name = "js-sys"
+version = "0.3.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "kak-highlight"
 version = "1.0.0"
 dependencies = [
- "clap",
+ "clap 4.2.1",
  "color-eyre",
+ "criterion",
  "crossbeam",
  "serde",
  "signal-hook",
@@ -441,6 +609,25 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
+dependencies = [
+ "hermit-abi 0.2.6",
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.30.3"
@@ -456,6 +643,18 @@ version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
+[[package]]
+name = "oorandom"
+version = "11.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+
+[[package]]
+name = "os_str_bytes"
+version = "6.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267"
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -474,6 +673,34 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116"
 
+[[package]]
+name = "plotters"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2538b639e642295546c50fcd545198c9d64ee2a38620a628724a3b266d5fbf97"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a81d2759aae1dae668f783c308bc5c8ebd191ff4184aaa1b37f65a6ae5a56f"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.56"
@@ -492,6 +719,28 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "num_cpus",
+]
+
 [[package]]
 name = "regex"
 version = "1.7.3"
@@ -538,6 +787,21 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
+
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -564,6 +828,17 @@ dependencies = [
  "syn 2.0.14",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.96"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "0.6.1"
@@ -635,6 +910,12 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
+
 [[package]]
 name = "thiserror"
 version = "1.0.40"
@@ -665,6 +946,16 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "toml"
 version = "0.7.3"
@@ -795,8 +1086,6 @@ dependencies = [
 [[package]]
 name = "tree-sitter-rust"
 version = "0.20.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "797842733e252dc11ae5d403a18060bf337b822fc2ae5ddfaa6ff4d9cc20bda6"
 dependencies = [
  "cc",
  "tree-sitter",
@@ -841,6 +1130,80 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
+[[package]]
+name = "walkdir"
+version = "2.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
+dependencies = [
+ "same-file",
+ "winapi-util",
+]
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d"
+
+[[package]]
+name = "web-sys"
+version = "0.3.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -857,6 +1220,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

Cargo.toml

@@ -1,6 +1,5 @@
 [package]
 name = "kak-highlight"
-author = "Lucas Schwiderski"
 description = "Tree-sitter-based highlighting for Kakoune"
 version = "1.0.0"
 edition = "2021"
@@ -19,4 +18,19 @@ tracing-error = "0.2.0"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 tree-sitter = "0.20.10"
 tree-sitter-highlight = "0.20.1"
-tree-sitter-rust = "0.20.3"
+tree-sitter-rust = { path = "./tree-sitter-rust", version = "*" }
+
+[dev-dependencies]
+criterion = "0.4.0"
+
+[lib]
+bench = false
+
+[[bin]]
+name = "kak-highlight"
+path = "src/main.rs"
+bench = false
+
+[[bench]]
+name = "worker"
+harness = false

benches/worker.rs

@@ -0,0 +1,66 @@
+use std::collections::HashMap;
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use tree_sitter_highlight::{HighlightConfiguration, Highlighter};
+
+use kak_highlight::daemon::worker::{highlight_content, make_highlighter_config};
+
+fn make_tokens() -> HashMap<String, String> {
+    [
+        "constructor",
+        "function",
+        "function.macro",
+        "function.method",
+        "keyword",
+        "module",
+        "operator",
+        "punctuation",
+        "string",
+        "type",
+        "variable",
+        "variable.parameter",
+    ]
+    .into_iter()
+    .fold(HashMap::new(), |mut map, val| {
+        map.insert(String::from(val), String::from(val));
+        map
+    })
+}
+
+fn run_benchmark(c: &mut Criterion, name: &str, content: &str) {
+    let timestamp = 0;
+    let tokens = make_tokens();
+    let names: Vec<_> = tokens.keys().collect();
+
+    let mut highlighter = Highlighter::new();
+    let highlight_config = make_highlighter_config(&names).unwrap();
+
+    c.bench_function(name, |b| {
+        b.iter(|| {
+            let _ = highlight_content(
+                &mut highlighter,
+                &highlight_config,
+                &tokens,
+                content.as_bytes(),
+                timestamp,
+            );
+        })
+    });
+}
+
+fn bench_main_rs(c: &mut Criterion) {
+    let content = r#"fn main() {
+    println!("Hello, world!");
+}
+"#;
+
+    run_benchmark(c, "main.rs", content);
+}
+
+fn bench_ast_rs(c: &mut Criterion) {
+    let content = include_str!("../tree-sitter-rust/examples/ast.rs");
+    run_benchmark(c, "ast.rs", content);
+}
+
+criterion_group!(benches, bench_main_rs, bench_ast_rs);
+criterion_main!(benches);

rc/highlight.kak

@@ -1,5 +1,4 @@
 declare-option str kak_highlight_cmd "kak-highlight"
-set-option global kak_highlight_cmd "kak-highlight -vv"
 
 declare-option str kak_highlight_log "/tmp/kak-highlight.log"
 declare-option -hidden range-specs kak_highlight_ranges
@@ -26,29 +25,44 @@ content = """
     }
 }
 
-define-command kak-highlight-enable -docstring %{
+define-command kak-highlight-enable-window -docstring %{
-    kak-highlight-enable
+    kak-highlight-enable-window
-    Start a daemon for the current session
+    Enable the highlighter for this window
 } %{
-    nop %sh{
-        (eval "${kak_opt_kak_highlight_cmd} --log '${kak_opt_kak_highlight_log}' daemon '${kak_session}'") >/dev/null 2>&1 </dev/null &
-    }
-
-    hook global KakEnd .* %{
-        nop %sh{
-            pkill -f "${kak_opt_kak_highlight_cmd} .* daemon ${kak_session}"
-        }
-    }
-}
-
-hook global WinSetOption filetype=rust %{
     add-highlighter window/kak_highlight_ranges ranges kak_highlight_ranges
 
     hook window -group kak-highlight NormalIdle .* kak-highlight
     hook window -group kak-highlight InsertIdle .* kak-highlight
     hook window -group kak-highlight BufReload .* kak-highlight
 
-    hook -once -always window WinSetOption filetype=.* %{
+    kak-highlight
+}
+
+define-command kak-highlight-disable-window -docstring %{
+    kak-highlight-disable-window
+    Disable the highlighter for this window
+} %{
+    remove-hooks window kak-highlight
     remove-highlighter window/kak_highlight_ranges
+}
+
+define-command kak-highlight-start -docstring %{
+    kak-highlight-start
+    Start the daemon
+} %{
+    nop %sh{
+        (eval "${kak_opt_kak_highlight_cmd} --log '${kak_opt_kak_highlight_log}' daemon '${kak_session}'") >/dev/null 2>&1 </dev/null &
     }
 }
+
+define-command kak-highlight-stop -docstring %{
+    kak-highlight-stop
+    Stop the daemon
+} %{
+    nop %sh{
+        pkill -f "${kak_opt_kak_highlight_cmd} .* daemon ${kak_session}"
+    }
+}
+
+hook global KakBegin .* kak-highlight-start
+hook global KakEnd .* kak-highlight-stop

src/daemon/mod.rs

@@ -17,7 +17,7 @@ use crate::daemon::worker::TaskScheduler;
 use self::listener::Listener;
 
 mod listener;
-mod worker;
+pub mod worker;
 
 #[tracing::instrument]
 pub fn handle(runtime_dir: PathBuf, config_path: Option<PathBuf>, session: String) -> Result<()> {

src/daemon/worker.rs

@@ -13,7 +13,7 @@ use color_eyre::Result;
 use crossbeam::deque::{Injector, Stealer, Worker};
 use tree_sitter_highlight::{HighlightConfiguration, HighlightEvent, Highlighter};
 
-use crate::kakoune::{self, editor_quote};
+use crate::kakoune::editor_quote;
 use crate::Request;
 
 type Task = Result<UnixStream>;
@@ -34,6 +34,22 @@ pub struct TaskContext {
     tokens: Arc<HashMap<String, String>>,
 }
 
+pub fn make_highlighter_config<S: AsRef<str>>(
+    names: impl AsRef<[S]>,
+) -> Result<HighlightConfiguration> {
+    let mut config = HighlightConfiguration::new(
+        tree_sitter_rust::language(),
+        tree_sitter_rust::HIGHLIGHT_QUERY,
+        tree_sitter_rust::INJECTIONS_QUERY,
+        tree_sitter_rust::LOCALS_QUERY,
+    )
+    .wrap_err("Invalid highlighter config")?;
+
+    config.configure(names.as_ref());
+
+    Ok(config)
+}
+
 impl TaskScheduler {
     pub fn new(workers: u8, tokens: HashMap<String, String>) -> Result<Self> {
         let terminate = Arc::new(AtomicBool::new(false));
@@ -43,18 +59,10 @@ impl TaskScheduler {
         let stealers: Vec<_> = workers.iter().map(|w| w.stealer()).collect();
         let stealers = Arc::new(stealers);
 
-        let mut highlight_config = HighlightConfiguration::new(
-            tree_sitter_rust::language(),
-            tree_sitter_rust::HIGHLIGHT_QUERY,
-            "",
-            "",
-        )
-        .wrap_err("Invalid highlighter config")?;
-
         let names: Vec<_> = tokens.keys().collect();
         tracing::debug!("Highlighter tokens: {:?}", names);
-        highlight_config.configure(&names);
 
+        let highlight_config = make_highlighter_config(names)?;
         let highlight_config = Arc::new(highlight_config);
         let tokens = Arc::new(tokens);
 
@@ -151,7 +159,13 @@ fn handle_connection(ctx: &mut TaskContext, task: Task) -> Result<()> {
     tracing::info!("Received request");
     tracing::debug!(?req);
 
-    let response = process_request(ctx, &req)
+    let response = highlight_content(
+        &mut ctx.highlighter,
+        &ctx.highlight_config,
+        &ctx.tokens,
+        req.content.as_bytes(),
+        req.timestamp,
+    )
     .unwrap_or_else(|err| format!("fail {}", editor_quote(format!("{}", err))));
 
     let mut child = Command::new("kak")
@@ -181,24 +195,69 @@ fn handle_connection(ctx: &mut TaskContext, task: Task) -> Result<()> {
     Ok(())
 }
 
-#[tracing::instrument(skip(ctx, req), fields(
+// Tree-sitter operates on byte offsets, while Kakoune's range specs use
-    session = req.session,
+// (row, column) tuples. Calculating those on the fly is very expensive, as it requires
-    client = req.client,
+// iterating the whole content to find line breaks.
-    content_len = req.content.len(),
+// So instead, generate a map once where any byte offset can be looked up later.
-    timestamp = req.timestamp,
+fn build_range_map(content: &[u8]) -> Vec<(usize, usize)> {
-))]
+    let mut map = Vec::with_capacity(content.len());
-fn process_request(ctx: &mut TaskContext, req: &Request) -> Result<String> {
-    let names: Vec<_> = ctx.tokens.keys().collect();
 
-    let highlights = ctx
+    // Kakoune's line indices are 1-based
-        .highlighter
+    let mut row = 1;
-        .highlight(&ctx.highlight_config, req.content.as_bytes(), None, |_| {
+    let mut column = 1;
+
+    for byte in content.iter() {
+        map.push((row, column));
+
+        if *byte == b'\n' {
+            row += 1;
+            column = 1;
+        } else {
+            column += 1;
+        }
+    }
+
+    map
+}
+
+#[tracing::instrument(skip(highlighter, highlight_config, tokens), fields(
+    content_len = content.len(),
+))]
+pub fn highlight_content(
+    highlighter: &mut Highlighter,
+    highlight_config: &HighlightConfiguration,
+    tokens: &HashMap<String, String>,
+    content: &[u8],
+    timestamp: u64,
+) -> Result<String> {
+    // By building vectors for both keys and values with the same order, we can
+    // use the highlighter's returned index to retrieve the face directly, rather
+    // than having to perform a second lookup in the hash map.
+    let mut names = Vec::with_capacity(tokens.len());
+    let mut faces = Vec::with_capacity(tokens.len());
+    for (name, face) in tokens.iter() {
+        names.push(name);
+        faces.push(face);
+    }
+
+    let highlights = highlighter
+        .highlight(highlight_config, content, None, |lang| {
+            if lang == "rust" {
+                Some(highlight_config)
+            } else {
                 None
+            }
         })
         .wrap_err("Failed to highlight content")?;
 
-    let mut stack = VecDeque::new();
+    let mut stack = VecDeque::with_capacity(64);
-    let mut range_spec = String::new();
+    let range_map = build_range_map(content);
+    // We're going to do lots of small pushes to this String, which would trigger
+    // lots of incremental re-allocations. To avoid that, start with a capacity that should
+    // hoefully be somewhat close to, but bigger than the final size.
+    let mut response = String::with_capacity(content.len() * 2);
+    response.push_str("set-option buffer kak_highlight_ranges ");
+    response.push_str(&timestamp.to_string());
 
     for res in highlights {
         match res? {
@@ -208,22 +267,23 @@ fn process_request(ctx: &mut TaskContext, req: &Request) -> Result<String> {
                     // as `end` here.
                     let end = end.saturating_sub(1);
 
-                    let range =
+                    let (start_row, start_column) = range_map[start];
-                        kakoune::range_from_byte_offsets(req.content.as_bytes(), start, end);
+                    let (end_row, end_column) = range_map[end];
+                    let face: &String = faces[*index];
 
-                    tracing::trace!(start, end, ?range, index);
+                    response.push(' ');
 
-                    let spec = format!(
+                    tracing::trace!(index, start_offset = start, end_offset = end, face);
-                        "{}.{},{}.{}|{}",
-                        range.start_point.row,
-                        range.start_point.column,
-                        range.end_point.row,
-                        range.end_point.column,
-                        ctx.tokens[names[*index]]
-                    );
 
-                    range_spec.push(' ');
+                    response.push_str(&start_row.to_string());
-                    range_spec.push_str(&spec);
+                    response.push('.');
+                    response.push_str(&start_column.to_string());
+                    response.push(',');
+                    response.push_str(&end_row.to_string());
+                    response.push('.');
+                    response.push_str(&end_column.to_string());
+                    response.push('|');
+                    response.push_str(face);
                 }
             }
             HighlightEvent::HighlightStart(index) => {
@@ -237,10 +297,97 @@ fn process_request(ctx: &mut TaskContext, req: &Request) -> Result<String> {
         }
     }
 
-    let response = format!(
-        "set-option buffer kak_highlight_ranges {}{range_spec}",
-        req.timestamp
-    );
-
     Ok(response)
 }
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use tree_sitter_highlight::Highlighter;
+
+    use crate::daemon::worker::make_highlighter_config;
+
+    use super::highlight_content;
+
+    fn make_tokens() -> HashMap<String, String> {
+        [
+            "constructor",
+            "function",
+            "function.macro",
+            "function.method",
+            "keyword",
+            "module",
+            "operator",
+            "punctuation",
+            "string",
+            "type",
+            "variable",
+            "variable.parameter",
+        ]
+        .into_iter()
+        .fold(HashMap::new(), |mut map, val| {
+            map.insert(String::from(val), String::from(val));
+            map
+        })
+    }
+
+    fn run_test(content: impl AsRef<str>, specs: &[&str]) {
+        let timestamp = 0;
+        let tokens = make_tokens();
+        let names: Vec<_> = tokens.keys().collect();
+
+        let mut highlighter = Highlighter::new();
+        let highlight_config = make_highlighter_config(&names).unwrap();
+
+        let response = highlight_content(
+            &mut highlighter,
+            &highlight_config,
+            &tokens,
+            content.as_ref().as_bytes(),
+            timestamp,
+        )
+        .unwrap();
+
+        assert_eq!(
+            response,
+            format!(
+                "set-option buffer kak_highlight_ranges {timestamp} {}",
+                specs.join(" ")
+            )
+            .trim()
+        );
+    }
+
+    #[test]
+    fn empty_content() {
+        let content = String::new();
+        let specs = [];
+        run_test(content, &specs);
+    }
+
+    #[test]
+    fn main_rs() {
+        let content = r#"fn main() {
+    println!("Hello, world!");
+}
+"#;
+
+        let specs = vec![
+            "1.1,1.2|keyword",
+            "1.4,1.7|function",
+            "1.8,1.8|punctuation",
+            "1.9,1.9|punctuation",
+            "1.11,1.11|punctuation",
+            "2.5,2.11|function.macro",
+            "2.12,2.12|function.macro",
+            "2.13,2.13|punctuation",
+            "2.14,2.28|string",
+            "2.29,2.29|punctuation",
+            "2.30,2.30|punctuation",
+            "3.1,3.1|punctuation",
+        ];
+
+        run_test(content, &specs);
+    }
+}

src/kakoune.rs

@@ -1,47 +1,4 @@
-use tree_sitter::{Point, Range};
-
 pub fn editor_quote(s: impl AsRef<str>) -> String {
     // TODO
     format!("'{}'", s.as_ref())
 }
-
-pub fn range_from_byte_offsets(content: impl AsRef<[u8]>, start: usize, end: usize) -> Range {
-    // Kakoune's line indices are 1-based
-    let mut start_row = 1;
-    let mut start_column = 1;
-    let mut end_row = 1;
-    let mut end_column = 1;
-
-    for (i, byte) in content.as_ref().iter().enumerate() {
-        if i < start {
-            if *byte == b'\n' {
-                start_row += 1;
-                start_column = 1;
-            } else {
-                start_column += 1;
-            }
-        }
-
-        if i < end {
-            if *byte == b'\n' {
-                end_row += 1;
-                end_column = 1;
-            } else {
-                end_column += 1;
-            }
-        }
-    }
-
-    Range {
-        start_byte: start,
-        end_byte: end,
-        start_point: Point {
-            row: start_row,
-            column: start_column,
-        },
-        end_point: Point {
-            row: end_row,
-            column: end_column,
-        },
-    }
-}

src/lib.rs

@@ -0,0 +1,19 @@
+use serde::Deserialize;
+
+mod config;
+mod kakoune;
+
+pub mod client;
+pub mod daemon;
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct Request {
+    /// The buffer content
+    content: String,
+    /// The Kakoune timestamp
+    timestamp: u64,
+    /// The Kakoune session
+    session: String,
+    /// The Kakoune client
+    client: String,
+}

src/main.rs

@@ -5,16 +5,12 @@ use std::path::PathBuf;
 use clap::{Parser, Subcommand};
 use color_eyre::eyre::Context;
 use color_eyre::Result;
-use serde::Deserialize;
 use tracing::metadata::LevelFilter;
 use tracing_error::ErrorLayer;
 use tracing_subscriber::fmt;
 use tracing_subscriber::prelude::*;
 
-mod client;
+use kak_highlight::{client, daemon};
-mod config;
-mod daemon;
-mod kakoune;
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
@@ -44,18 +40,6 @@ enum Command {
     },
 }
 
-#[derive(Clone, Debug, Deserialize)]
-struct Request {
-    /// The buffer content
-    content: String,
-    /// The Kakoune timestamp
-    timestamp: u64,
-    /// The Kakoune session
-    session: String,
-    /// The Kakoune client
-    client: String,
-}
-
 #[tracing::instrument]
 fn main() -> Result<()> {
     let cli = Cli::parse();
@@ -90,8 +74,14 @@ fn main() -> Result<()> {
         .unwrap_or_else(|_| "/var".into())
         .join("kak-highlight");
 
-    match cli.command {
+    let res = match cli.command {
         Command::Daemon { session } => daemon::handle(runtime_dir, cli.config, session),
         Command::Request => client::handle(runtime_dir),
+    };
+
+    if let Err(err) = &res {
+        tracing::error!("{:?}", err)
     }
+
+    res
 }

tree-sitter-rust

@@ -0,0 +1 @@
+Subproject commit 15452ac6becd16a622ec6ea70de283063926a0bd