crates/dtmm/assets/init.lua

@@ -0,0 +1,70 @@
+local StateGame = require("scripts/game_states/state_game")
+local StateSplash = require("scripts/game_states/game/state_splash")
+local GameStateMachine = require("scripts/foundation/utilities/game_state_machine")
+
+local function hook(obj, fn_name, cb)
+    local orig = obj[fn_name]
+
+    obj[fn_name] = function(...)
+        return cb(orig, ...)
+    end
+end
+
+function init(mod_data, boot_gui)
+    local ModLoader = require("scripts/mods/mod_loader")
+    local mod_loader = ModLoader:new(mod_data, boot_gui)
+
+    -- The mod loader needs to remain active during game play, to
+    -- enable reloads
+    hook(StateGame, "update", function(func, dt, ...)
+        mod_loader:update(dt)
+        return func(dt, ...)
+    end)
+
+    -- Skip splash view
+    hook(StateSplash, "on_enter", function(func, self, ...)
+        local result = func(self, ...)
+
+        self._should_skip = true
+        self._continue = true
+
+        return result
+    end)
+
+    -- Trigger state change events
+    hook(GameStateMachine, "_change_state", function(func, self, ...)
+        local old_state = self._state
+        local old_state_name = old_state and self:current_state_name()
+
+        if old_state_name then
+            mod_loader:on_game_state_changed("exit", old_state_name, old_state)
+        end
+
+        local result = func(self, ...)
+
+        local new_state = self._state
+        local new_state_name = new_state and self:current_state_name()
+
+        if new_state_name then
+            mod_loader:on_game_state_changed("enter", new_state_name, new_state)
+        end
+
+        return result
+    end)
+
+    -- Trigger ending state change event
+    hook(GameStateMachine, "destroy", function(func, self, ...)
+        local old_state = self._state
+        local old_state_name = old_state and self:current_state_name()
+
+        if old_state_name then
+            mod_loader:on_game_state_changed("exit", old_state_name)
+        end
+
+        return func(self, ...)
+    end)
+
+    return mod_loader
+end
+
+return init

crates/dtmm/assets/mod_loader.lua

@@ -0,0 +1,405 @@
+-- Copyright on this file is owned by Fatshark.
+-- It is extracted, used and modified with permission only for
+-- the purpose of loading mods within Warhammer 40,000: Darktide.
+local ModLoader = class("ModLoader")
+
+local table_unpack = table.unpack or unpack
+local table_pack = table.pack or pack
+
+local ScriptGui = require("scripts/foundation/utilities/script_gui")
+
+local FONT_MATERIAL = "content/ui/fonts/arial"
+
+local LOG_LEVELS = {
+    spew = 4,
+    info = 3,
+    warning = 2,
+    error = 1
+}
+local DEFAULT_SETTINGS = {
+    log_level = LOG_LEVELS.error,
+    developer_mode = false
+}
+
+local Keyboard = Keyboard
+local BUTTON_INDEX_R = Keyboard.button_index("r")
+local BUTTON_INDEX_LEFT_SHIFT = Keyboard.button_index("left shift")
+local BUTTON_INDEX_LEFT_CTRL = Keyboard.button_index("left ctrl")
+
+ModLoader.init = function(self, mod_data, boot_gui)
+    table.dump(mod_data, nil, 5, function(...) Log.info("ModLoader", ...) end)
+
+    self._mod_data = mod_data
+    self._gui = boot_gui
+
+    self._settings = Application.user_setting("mod_settings") or DEFAULT_SETTINGS
+
+    self._mods = {}
+    self._num_mods = nil
+    self._chat_print_buffer = {}
+    self._reload_data = {}
+    self._ui_time = 0
+
+    self._state = "scanning"
+end
+
+ModLoader.developer_mode_enabled = function(self)
+    return self._settings.developer_mode
+end
+
+ModLoader.set_developer_mode = function(self, enabled)
+    self._settings.developer_mode = enabled
+end
+
+ModLoader._draw_state_to_gui = function(self, gui, dt)
+    local state = self._state
+    local t = self._ui_time + dt
+    self._ui_time = t
+    local status_str = "Loading mods"
+
+    if state == "scanning" then
+        status_str = "Scanning for mods"
+    elseif state == "loading" or state == "initializing" then
+        local mod = self._mods[self._mod_load_index]
+        status_str = string.format("Loading mod %q", mod.name)
+    end
+
+    local msg = status_str .. string.rep(".", (2 * t) % 4)
+    ScriptGui.text(gui, msg, FONT_MATERIAL, 25, Vector3(20, 30, 1), Color.white())
+end
+
+ModLoader.remove_gui = function(self)
+    self._gui = nil
+end
+
+ModLoader.mod_data = function(self, id)
+    -- Since this primarily exists for DMF,
+    -- we can optimize the search for its use case of looking for the
+    -- mod currently being loaded
+    local mod_data = self._mods[self._mod_load_index]
+
+    if mod_data.id ~= id then
+        mod_data = nil
+
+        for _, v in ipairs(self._mods) do
+            if v.id == id then
+                mod_data = v
+            end
+        end
+    end
+
+    return mod_data
+end
+
+ModLoader._check_reload = function()
+    return Keyboard.pressed(BUTTON_INDEX_R) and
+        Keyboard.button(BUTTON_INDEX_LEFT_SHIFT) +
+        Keyboard.button(BUTTON_INDEX_LEFT_CTRL) == 2
+end
+
+ModLoader.update = function(self, dt)
+    local chat_print_buffer = self._chat_print_buffer
+    local num_delayed_prints = #chat_print_buffer
+
+    if num_delayed_prints > 0 and Managers.chat then
+        for i = 1, num_delayed_prints, 1 do
+            -- TODO: Use new chat system
+            -- Managers.chat:add_local_system_message(1, chat_print_buffer[i], true)
+
+            chat_print_buffer[i] = nil
+        end
+    end
+
+    local old_state = self._state
+
+    if self._settings.developer_mode and self:_check_reload() then
+        self._reload_requested = true
+    end
+
+    if self._reload_requested and old_state == "done" then
+        self:_reload_mods()
+    end
+
+    if old_state == "done" then
+        self:_run_callbacks("update", dt)
+    elseif old_state == "scanning" then
+        Log.info("ModLoader", "Scanning for mods")
+        self:_build_mod_table()
+
+        self._state = self:_load_mod(1)
+        self._ui_time = 0
+    elseif old_state == "loading" then
+        local handle = self._loading_resource_handle
+
+        if ResourcePackage.has_loaded(handle) then
+            ResourcePackage.flush(handle)
+
+            local mod = self._mods[self._mod_load_index]
+            local next_index = mod.package_index + 1
+            local mod_data = mod.data
+
+            if next_index <= #mod_data.packages then
+                self:_load_package(mod, next_index)
+            else
+                self._state = "initializing"
+            end
+        end
+    elseif old_state == "initializing" then
+        local mod = self._mods[self._mod_load_index]
+        local mod_data = mod.data
+
+        Log.info("ModLoader", "Initializing mod %q", mod.name)
+
+        mod.state = "running"
+        local ok, object = xpcall(mod_data.run, function(err)
+            if type(err) == "string" then
+                return err .. "\n" .. Script.callstack()
+            else
+                return err
+            end
+        end)
+
+        if not ok then
+            if object.error then
+                object = string.format(
+                    "%s\n<<Lua Stack>>\n%s\n<</Lua Stack>>\n<<Lua Locals>>\n%s\n<</Lua Locals>>\n<<Lua Self>>\n%s\n<</Lua Self>>",
+                    object.error, object.traceback, object.locals, object.self)
+            end
+
+            Log.error("ModLoader", "Failed 'run' for %q: %s", mod.name, object)
+        end
+
+        mod.object = object or {}
+
+        self:_run_callback(mod, "init", self._reload_data[mod.id])
+
+        Log.info("ModLoader", "Finished loading %q", mod.name)
+
+        self._state = self:_load_mod(self._mod_load_index + 1)
+    end
+
+    local gui = self._gui
+    if gui then
+        self:_draw_state_to_gui(gui, dt)
+    end
+
+    if old_state ~= self._state then
+        Log.info("ModLoader", "%s -> %s", old_state, self._state)
+    end
+end
+
+ModLoader.all_mods_loaded = function(self)
+    return self._state == "done"
+end
+
+ModLoader.destroy = function(self)
+    self:_run_callbacks("on_destroy")
+    self:unload_all_mods()
+end
+
+ModLoader._run_callbacks = function(self, callback_name, ...)
+    for i = 1, self._num_mods, 1 do
+        local mod = self._mods[i]
+
+        if mod and not mod.callbacks_disabled then
+            self:_run_callback(mod, callback_name, ...)
+        end
+    end
+end
+
+ModLoader._run_callback = function(self, mod, callback_name, ...)
+    local object = mod.object
+    local cb = object[callback_name]
+
+    if not cb then
+        return
+    end
+
+    local args = table_pack(...)
+
+    local success, val = xpcall(
+        function() return cb(object, table_unpack(args)) end,
+        function(err)
+            if type(err) == "string" then
+                return err .. "\n" .. Script.callstack()
+            else
+                return err
+            end
+        end
+    )
+
+    if success then
+        return val
+    else
+        Log.error("ModLoader", "Failed to run callback %q for mod %q with id %q. Disabling callbacks until reload.",
+            callback_name, mod.name, mod.id)
+        if val.error then
+            Log.error("ModLoader",
+                "Error: %s\n<<Lua Stack>>\n%s<</Lua Stack>>\n<<Lua Locals>>\n%s<</Lua Locals>>\n<<Lua Self>>\n%s<</Lua Self>>",
+                val.error, val.traceback, val.locals, val.self)
+        else
+            Log.error("ModLoader", "Error: %s", val or "[unknown error]")
+        end
+
+        mod.callbacks_disabled = true
+    end
+end
+
+ModLoader._start_scan = function(self)
+    Log.info("ModLoader", "Starting mod scan")
+    self._state = "scanning"
+end
+
+ModLoader._build_mod_table = function(self)
+    fassert(table.is_empty(self._mods), "Trying to add mods to non-empty mod table")
+
+    for i, mod_data in ipairs(self._mod_data) do
+        Log.info("ModLoader", "mods[%d] = id=%q | name=%q | bundled=%s", i, mod_data.id, mod_data.name,
+            tostring(mod_data.bundled))
+
+        self._mods[i] = {
+            id = mod_data.id,
+            state = "not_loaded",
+            callbacks_disabled = false,
+            name = mod_data.name,
+            loaded_packages = {},
+            packages = mod_data.packages,
+            data = mod_data,
+            bundled = mod_data.bundled or false,
+        }
+    end
+
+    self._num_mods = #self._mods
+
+    Log.info("ModLoader", "Found %i mods", self._num_mods)
+end
+
+ModLoader._load_mod = function(self, index)
+    self._ui_time = 0
+    local mods = self._mods
+    local mod = mods[index]
+
+    if not mod then
+        table.clear(self._reload_data)
+
+        return "done"
+    end
+
+    Log.info("ModLoader", "Loading mod %q", mod.id)
+
+    mod.state = "loading"
+
+    Crashify.print_property(string.format("Mod:%s:%s", mod.id, mod.name), true)
+
+    self._mod_load_index = index
+
+    if mod.bundled and mod.packages[1] then
+        self:_load_package(mod, 1)
+        return "loading"
+    else
+        return "initializing"
+    end
+end
+
+ModLoader._load_package = function(self, mod, index)
+    mod.package_index = index
+    local package_name = mod.packages[index]
+
+    if not package_name then
+        return
+    end
+
+    Log.info("ModLoader", "Loading package %q", package_name)
+
+    local resource_handle = Application.resource_package(package_name)
+    self._loading_resource_handle = resource_handle
+
+    ResourcePackage.load(resource_handle)
+
+    table.insert(mod.loaded_packages, resource_handle)
+end
+
+ModLoader.unload_all_mods = function(self)
+    if self._state ~= "done" then
+        Log.error("ModLoader", "Mods can't be unloaded, mod state is not \"done\". current: %q", self._state)
+
+        return
+    end
+
+    Log.info("ModLoader", "Unload all mod packages")
+
+    for i = self._num_mods, 1, -1 do
+        local mod = self._mods[i]
+
+        if mod then
+            self:unload_mod(i)
+        end
+
+        self._mods[i] = nil
+    end
+
+    self._num_mods = nil
+    self._state = "unloaded"
+end
+
+ModLoader.unload_mod = function(self, index)
+    local mod = self._mods[index]
+
+    if mod then
+        Log.info("ModLoader", "Unloading %q.", mod.name)
+
+        for _, handle in ipairs(mod.loaded_packages) do
+            ResourcePackage.unload(handle)
+            Application.release_resource_package(handle)
+        end
+
+        mod.state = "not_loaded"
+    else
+        Log.error("ModLoader", "Mod index %i can't be unloaded, has not been loaded", index)
+    end
+end
+
+ModLoader._reload_mods = function(self)
+    Log.info("ModLoader", "reloading mods")
+
+    for i = 1, self._num_mods, 1 do
+        local mod = self._mods[i]
+
+        if mod and mod.state == "running" then
+            Log.info("ModLoader", "reloading %s", mod.name)
+
+            self._reload_data[mod.id] = self:_run_callback(mod, "on_reload")
+        else
+            Log.info("ModLoader", "not reloading mod, state: %s", mod.state)
+        end
+    end
+
+    self:unload_all_mods()
+    self:_start_scan()
+
+    self._reload_requested = false
+end
+
+ModLoader.on_game_state_changed = function(self, status, state_name, state_object)
+    if self._state == "done" then
+        self:_run_callbacks("on_game_state_changed", status, state_name, state_object)
+    else
+        Log.warning("ModLoader", "Ignored on_game_state_changed call due to being in state %q", self._state)
+    end
+end
+
+ModLoader.print = function(self, level, str, ...)
+    local f = Log[level]
+    if f then
+        f("ModLoader", str, ...)
+    else
+        local message = string.format("[ModLoader][" .. level .. "] " .. str, ...)
+        local log_level = LOG_LEVELS[level] or 99
+
+        if log_level <= 2 then
+            print(message)
+        end
+    end
+end
+
+return ModLoader

crates/dtmm/assets/mod_main.lua.j2

@@ -105,10 +105,16 @@ end
 require("scripts/main")
 log("mod_main", "'scripts/main' loaded")
 
--- Inject our state into the game. The state needs to run after `StateGame._init_managers`,
+-- We need to inject two states into two different state machines:
--- since some parts of DMF, and presumably other mods, depend on some of those managers to exist.
+-- First, we inject one into the `"Main"` state machine at a specific location, so that we're
+-- still early in the process, but right after `StateRequireScripts` where most game files
+-- are already available to `require` and hook.
+-- This is where the `ModLoader` is created initially.
+-- Then, we inject into the very first position of the `"Game"` state machine. This runs right
+-- after `StateGame._init_managers`, at which point all the parts needed for DMF and other mods
+-- have been initialized.
+-- This is where `ModLoader` will finally start loading mods.
 local function patch_mod_loading_state()
-    local StateBootSubStateBase = require("scripts/game_states/boot/state_boot_sub_state_base")
     local StateBootLoadDML = class("StateBootLoadDML", "StateBootSubStateBase")
     local StateGameLoadMods = class("StateGameLoadMods")
 
@@ -121,19 +127,21 @@ local function patch_mod_loading_state()
 
         self._package_manager = package_manager
         self._package_handles = {
-            ["packages/mods"] = package_manager:load("packages/mods", "StateBootDML", nil),
+            ["packages/mods"] = package_manager:load("packages/mods", "StateBootLoadDML", nil),
-            ["packages/dml"] = package_manager:load("packages/dml", "StateBootDML", nil),
         }
     end
 
-    StateBootLoadDML._state_update = function(self, dt)
+    StateBootLoadDML._state_update = function(self, _)
         local package_manager = self._package_manager
 
         if package_manager:update() then
-            local DML = require("scripts/mods/dml/init")
             local mod_data = require("scripts/mods/mod_data")
-            local mod_loader = DML.create_loader(mod_data)
+
+            local create_mod_loader = require("scripts/mods/init")
+            local mod_loader = create_mod_loader(mod_data)
+
             Managers.mod = mod_loader
+
             log("StateBootLoadDML", "DML loaded, exiting")
             return true, false
         end
@@ -148,9 +156,7 @@ local function patch_mod_loading_state()
         self._next_state_params = params
     end
 
-    function StateGameLoadMods:update(main_dt)
+    function StateGameLoadMods:update(_)
-        local state = self._loading_state
-
         -- We're relying on the fact that DML internally makes sure
         -- that `Managers.mod:update()` is being called appropriately.
         -- The implementation as of this writing is to hook `StateGame.update`.
@@ -188,8 +194,6 @@ local function patch_mod_loading_state()
             GameStateMachine_init(self, parent, StateGameLoadMods, params, creation_context, state_change_callbacks, name)
             -- And since we're done now, we can revert the function to its original
             GameStateMachine.init = GameStateMachine_init
-
-            return
         else
             -- In all other cases, simply call the original
             GameStateMachine_init(self, parent, start_state, params, creation_context, state_change_callbacks, name)

crates/dtmm/src/controller/app.rs

@@ -161,27 +161,21 @@ where
 }
 
 pub(crate) fn check_mod_order(state: &ActionState) -> Result<()> {
-    {
-        let first = state.mods.get(0);
-        if first.is_none() || !(first.unwrap().id == "dml" && first.unwrap().enabled) {
-            // TODO: Add a suggestion where to get it, once that's published
-            eyre::bail!("'Darktide Mod Loader' needs to be installed, enabled and at the top of the load order");
-        }
-    }
-
     if tracing::enabled!(tracing::Level::DEBUG) {
-        let order = state.mods.iter().filter(|i| i.enabled).enumerate().fold(
+        let order = state
-            String::new(),
+            .mods
-            |mut s, (i, info)| {
+            .iter()
+            .enumerate()
+            .filter(|(_, i)| i.enabled)
+            .fold(String::new(), |mut s, (i, info)| {
                 s.push_str(&format!("{}: {} - {}\n", i, info.id, info.name));
                 s
-            },
+            });
-        );
 
         tracing::debug!("Mod order:\n{}", order);
     }
 
-    for (i, mod_info) in state.mods.iter().filter(|i| i.enabled).enumerate() {
+    for (i, mod_info) in state.mods.iter().enumerate().filter(|(_, i)| i.enabled) {
         for dep in &mod_info.depends {
             let dep_info = state.mods.iter().enumerate().find(|(_, m)| m.id == dep.id);
 

crates/dtmm/src/controller/deploy.rs

@@ -26,7 +26,6 @@ use crate::state::{ActionState, PackageInfo};
 
 pub const MOD_BUNDLE_NAME: &str = "packages/mods";
 pub const BOOT_BUNDLE_NAME: &str = "packages/boot";
-pub const DML_BUNDLE_NAME: &str = "packages/dml";
 pub const BUNDLE_DATABASE_NAME: &str = "bundle_database.data";
 pub const MOD_BOOT_SCRIPT: &str = "scripts/mod_main";
 pub const MOD_DATA_SCRIPT: &str = "scripts/mods/mod_data";
@@ -225,11 +224,7 @@ async fn copy_mod_folders(state: Arc<ActionState>) -> Result<Vec<String>> {
 
     let mut tasks = Vec::new();
 
-    for mod_info in state
+    for mod_info in state.mods.iter().filter(|m| m.enabled && !m.bundled) {
-        .mods
-        .iter()
-        .filter(|m| m.id != "dml" && m.enabled && !m.bundled)
-    {
         let span = tracing::trace_span!("copying legacy mod", name = mod_info.name);
         let _enter = span.enter();
 
@@ -283,7 +278,7 @@ fn build_mod_data_lua(state: Arc<ActionState>) -> Result<String> {
         .mods
         .iter()
         .filter_map(|m| {
-            if m.id == "dml" || !m.enabled {
+            if !m.enabled {
                 return None;
             }
 
@@ -325,31 +320,29 @@ async fn build_bundles(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
 
     let mut bundles = Vec::new();
 
-    {
+    let mut add_lua_asset = |name, data: &str| {
-        tracing::trace!("Building mod data script");
+        let span = tracing::info_span!("Compiling Lua", name, data_len = data.len());
-
-        let span = tracing::debug_span!("Building mod data script");
         let _enter = span.enter();
 
-        let lua = build_mod_data_lua(state.clone()).wrap_err("Failed to build Lua mod data")?;
+        let file = lua::compile(name, data).wrap_err("Failed to compile Lua")?;
-
-        tracing::trace!("Compiling mod data script");
-
-        let file =
-            lua::compile(MOD_DATA_SCRIPT, lua).wrap_err("Failed to compile mod data Lua file")?;
-
-        tracing::trace!("Compile mod data script");
 
         mod_bundle.add_file(file);
-    }
+
+        Ok::<_, Report>(())
+    };
+
+    build_mod_data_lua(state.clone())
+        .wrap_err("Failed to build 'mod_data.lua'")
+        .and_then(|data| add_lua_asset(MOD_DATA_SCRIPT, &data))?;
+    add_lua_asset("scripts/mods/init", include_str!("../../assets/init.lua"))?;
+    add_lua_asset(
+        "scripts/mods/mod_loader",
+        include_str!("../../assets/mod_loader.lua"),
+    )?;
 
     tracing::trace!("Preparing tasks to deploy bundle files");
 
-    for mod_info in state
+    for mod_info in state.mods.iter().filter(|m| m.enabled && m.bundled) {
-        .mods
-        .iter()
-        .filter(|m| m.id != "dml" && m.enabled && m.bundled)
-    {
         let span = tracing::trace_span!("building mod packages", name = mod_info.name);
         let _enter = span.enter();
 
@@ -497,75 +490,6 @@ async fn patch_boot_bundle(state: Arc<ActionState>) -> Result<Vec<Bundle>> {
         boot_bundle.add_file(f);
     }
 
-    {
-        tracing::trace!("Handling DML packages and bundle");
-        let span = tracing::trace_span!("handle DML");
-        let _enter = span.enter();
-
-        let mut variant = BundleFileVariant::new();
-
-        let mod_info = state
-            .mods
-            .iter()
-            .find(|m| m.id == "dml")
-            .ok_or_else(|| eyre::eyre!("DML not found in mod list"))?;
-        let pkg_info = mod_info
-            .packages
-            .get(0)
-            .ok_or_else(|| eyre::eyre!("invalid mod package for DML"))
-            .with_suggestion(|| "Re-download and import the newest version.".to_string())?;
-        let bundle_name = format!("{:016x}", Murmur64::hash(&pkg_info.name));
-        let src = state.mod_dir.join(&mod_info.id).join(&bundle_name);
-
-        {
-            let bin = fs::read(&src)
-                .await
-                .wrap_err_with(|| format!("Failed to read bundle file '{}'", src.display()))?;
-            let name = Bundle::get_name_from_path(&state.ctx, &src);
-
-            let dml_bundle = Bundle::from_binary(&state.ctx, name, bin)
-                .wrap_err_with(|| format!("Failed to parse bundle '{}'", src.display()))?;
-
-            bundles.push(dml_bundle);
-        };
-
-        {
-            let dest = bundle_dir.join(&bundle_name);
-            let pkg_name = pkg_info.name.clone();
-            let mod_name = mod_info.name.clone();
-
-            tracing::debug!(
-                "Copying bundle {} for mod {}: {} -> {}",
-                pkg_name,
-                mod_name,
-                src.display(),
-                dest.display()
-            );
-            // We attempt to remove any previous file, so that the hard link can be created.
-            // We can reasonably ignore errors here, as a 'NotFound' is actually fine, the copy
-            // may be possible despite an error here, or the error will be reported by it anyways.
-            // TODO: There is a chance that we delete an actual game bundle, but with 64bit
-            // hashes, it's low enough for now, and the setup required to detect
-            // "game bundle vs mod bundle" is non-trivial.
-            let _ = fs::remove_file(&dest).await;
-            fs::copy(&src, &dest).await.wrap_err_with(|| {
-                format!(
-                    "Failed to copy bundle {pkg_name} for mod {mod_name}. Src: {}, dest: {}",
-                    src.display(),
-                    dest.display()
-                )
-            })?;
-        }
-
-        let pkg = make_package(pkg_info).wrap_err("Failed to create package file for dml")?;
-        variant.set_data(pkg.to_binary()?);
-
-        let mut f = BundleFile::new(DML_BUNDLE_NAME.to_string(), BundleFileType::Package);
-        f.add_variant(variant);
-
-        boot_bundle.add_file(f);
-    }
-
     {
         let span = tracing::debug_span!("Importing mod main script");
         let _enter = span.enter();