feat(sdk): Link LuaJIT for bytecode compilation

This removes the LuaJIT binary as a runtime dependency and decreases
the complexity of the compilation, by not needing to juggle a bunch of
temp files anymore.

However, it was a bit of a pain to get everything set up in the end.

Closes #4.

Cargo.lock

@@ -855,6 +855,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "futures"
 version = "0.3.25"
@@ -1358,6 +1364,17 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "luajit2-sys"
+version = "0.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33bb7acccd5a0224645ba06eba391af5f7194ff1762c2545860b43afcfd41af2"
+dependencies = [
+ "cc",
+ "fs_extra",
+ "libc",
+]
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"
@@ -1966,6 +1983,7 @@ dependencies = [
  "futures-util",
  "glob",
  "libloading",
+ "luajit2-sys",
  "nanorand",
  "oodle-sys",
  "pin-project-lite",

lib/sdk/Cargo.toml

@@ -22,3 +22,4 @@ tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "m
 tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] }
 tracing = { version = "0.1.37", features = ["async-await"] }
 tracing-error = "0.2.0"
+luajit2-sys = "0.0.2"

lib/sdk/src/bundle/file.rs

@@ -1,3 +1,4 @@
+use std::ffi::CString;
 use std::io::{Cursor, Read, Seek, Write};
 use std::path::Path;
 
@@ -660,7 +661,11 @@ impl BundleFile {
         S: AsRef<str>,
     {
         match file_type {
-            BundleFileType::Lua => lua::compile(name, sjson).await,
+            BundleFileType::Lua => {
+                let sjson =
+                    CString::new(sjson.as_ref()).wrap_err("failed to build CString from SJSON")?;
+                lua::compile(name, sjson)
+            }
             BundleFileType::Unknown(_) => {
                 eyre::bail!("Unknown file type. Cannot compile from SJSON");
             }

lib/sdk/src/filetype/lua.rs

@@ -1,13 +1,12 @@
-use std::io::{Cursor, Write};
+use std::ffi::CStr;
+use std::io::Cursor;
 
-use color_eyre::{eyre::Context, Result};
+use color_eyre::eyre;
-use tokio::{fs, process::Command};
+use color_eyre::Result;
+use luajit2_sys as lua;
 
-use crate::{
+use crate::bundle::file::{BundleFileVariant, UserFile};
-    binary::sync::WriteExt,
+use crate::{BundleFile, BundleFileType};
-    bundle::file::{BundleFileVariant, UserFile},
-    BundleFile, BundleFileType,
-};
 
 #[tracing::instrument(skip_all, fields(buf_len = data.as_ref().len()))]
 pub(crate) async fn decompile<T>(_ctx: &crate::Context, data: T) -> Result<Vec<UserFile>>
@@ -19,72 +18,81 @@ where
 }
 
 #[tracing::instrument(skip_all)]
-pub(crate) async fn compile<S>(name: String, code: S) -> Result<BundleFile>
+pub fn compile<S>(name: String, code: S) -> Result<BundleFile>
 where
-    S: AsRef<str>,
+    S: AsRef<CStr>,
 {
-    let in_file_path = {
+    let bytecode = unsafe {
-        let mut path = std::env::temp_dir();
+        let state = lua::luaL_newstate();
-        let name: String = std::iter::repeat_with(fastrand::alphanumeric)
+        lua::luaL_openlibs(state);
-            .take(10)
-            .collect();
-        path.push(name + "-dtmt.lua");
 
-        path
+        lua::lua_pushstring(state, code.as_ref().as_ptr() as _);
-    };
+        lua::lua_setglobal(state, b"code\0".as_ptr() as _);
 
-    let out_file_path = {
+        lua::lua_pushstring(state, name.as_ptr() as _);
-        let mut path = std::env::temp_dir();
+        lua::lua_setglobal(state, b"name\0".as_ptr() as _);
 
-        let name: String = std::iter::repeat_with(fastrand::alphanumeric)
+        let run = b"return string.dump(loadstring(code, \"@\" .. name), false)\0";
-            .take(10)
+        match lua::luaL_loadstring(state, run.as_ptr() as _) as u32 {
-            .collect();
+            lua::LUA_OK => {}
-        path.push(name + "-dtmt.luab");
+            lua::LUA_ERRSYNTAX => {
+                let err = lua::lua_tostring(state, -1);
+                let err = CStr::from_ptr(err).to_string_lossy().to_string();
 
-        path
+                lua::lua_close(state);
-    };
 
-    fs::write(&in_file_path, code.as_ref().as_bytes())
+                eyre::bail!("Invalid syntax: {}", err);
-        .await
+            }
-        .wrap_err_with(|| format!("failed to write file {}", in_file_path.display()))?;
+            lua::LUA_ERRMEM => {
+                lua::lua_close(state);
+                eyre::bail!("Failed to allocate sufficient memory to compile LuaJIT bytecode")
+            }
+            _ => unreachable!(),
+        }
 
-    // TODO: Make executable name configurable
+        match lua::lua_pcall(state, 0, 1, 0) as u32 {
-    Command::new("luajit")
+            lua::LUA_OK => {
-        .arg("-bg")
+                // The binary data is pretty much guaranteed to contain NUL bytes,
-        .arg("-F")
+                // so we can't rely on `lua_tostring` and `CStr` here. Instead we have to
-        .arg(name.clone() + ".lua")
+                // explicitely query the string length and build our vector from that.
-        .arg("-o")
+                // However, on the bright side, we don't have to go through any string types anymore,
-        .arg("Windows")
+                // and can instead treat it as raw bytes immediately.
-        .arg(&in_file_path)
+                let mut len = 0;
-        .arg(&out_file_path)
+                let data = lua::lua_tolstring(state, -1, &mut len) as *const u8;
-        .status()
+                let data = std::slice::from_raw_parts(data, len).to_vec();
-        .await
-        .wrap_err("failed to compile to LuaJIT byte code")?;
 
-    let mut data = Cursor::new(Vec::new());
+                lua::lua_close(state);
-
-    let bytecode = {
-        let mut data = fs::read(&out_file_path)
-            .await
-            .wrap_err_with(|| format!("failed to read file {}", out_file_path.display()))?;
-
-        // Add Fatshark's custom magic bytes
-        data[1] = 0x46;
-        data[2] = 0x53;
-        data[3] = 0x82;
 
                 data
+            }
+            lua::LUA_ERRRUN => {
+                let err = lua::lua_tostring(state, -1);
+                let err = CStr::from_ptr(err).to_string_lossy().to_string();
+
+                lua::lua_close(state);
+
+                eyre::bail!("Failed to compile LuaJIT bytecode: {}", err);
+            }
+            lua::LUA_ERRMEM => {
+                lua::lua_close(state);
+                eyre::bail!("Failed to allocate sufficient memory to compile LuaJIT bytecode")
+            }
+            // We don't use an error handler function, so this should be unreachable
+            lua::LUA_ERRERR => unreachable!(),
+            _ => unreachable!(),
+        }
     };
 
-    data.write_u32(bytecode.len() as u32)?;
+    // TODO: How are these twelve bytes generated?
-    // I believe this is supposed to be a uleb128, but it seems to be always 0x2 in binary.
+    let mut data = vec![0x0; 12];
-    data.write_u64(0x2)?;
+    // Set Fatshark's custom LuaJIT magic bytes
-    data.write_all(&bytecode)?;
+    data.extend_from_slice(&[0x1b, 0x46, 0x53, 0x82]);
+    data.extend_from_slice(&bytecode[4..]);
 
     let mut file = BundleFile::new(name, BundleFileType::Lua);
     let mut variant = BundleFileVariant::new();
 
-    variant.set_data(data.into_inner());
+    variant.set_data(data);
     file.add_variant(variant);
 
     Ok(file)