.ci/Dockerfile.linux

@@ -1,7 +1,35 @@
 FROM dtmt-ci-base-linux
 
+# Create dummy crates and copy their Cargo.toml, so that dependencies can be cached
+RUN set -e; \
+    cargo new --bin crates/dtmt; \
+    cargo new --bin crates/dtmm; \
+    cargo new --lib lib/dtmt-shared; \
+    cargo new --lib lib/nexusmods; \
+    cargo new --lib lib/sdk; \
+    cargo new --lib lib/serde_sjson; \
+    cargo new --lib lib/ansi-parser
+
+COPY Cargo.toml Cargo.lock /src/dtmt/
+COPY crates/dtmt/Cargo.toml /src/dtmt/crates/dtmt/
+COPY crates/dtmm/Cargo.toml /src/dtmt/crates/dtmm/
+COPY lib/dtmt-shared/Cargo.toml /src/dtmt/lib/dtmt-shared/
+COPY lib/nexusmods/Cargo.toml /src/dtmt/lib/nexusmods/
+COPY lib/sdk/Cargo.toml /src/dtmt/lib/sdk/
+COPY lib/serde_sjson/Cargo.toml /src/dtmt/lib/serde_sjson/
+COPY lib/ansi-parser/Cargo.toml /src/dtmt/lib/ansi-parser/
+
+# Crates with build scripts cannot be split that way, but they shouldn't change too often
+COPY lib/luajit2-sys /src/dtmt/lib/luajit2-sys
+COPY lib/oodle /src/dtmt/lib/oodle
+# color-eyre needs to be copied, too, then, as it's used by `oodle`
+COPY lib/color-eyre /src/dtmt/lib/color-eyre
+COPY --from=dtmt-ci-base-linux /src/*.lib /src/dtmt/lib/oodle/
+
+RUN cargo build --release --locked
+RUN rm -r crates lib
+
 COPY . /src/dtmt
 COPY --from=dtmt-ci-base-linux /src/*.lib /src/*.so /src/dtmt/lib/oodle/
-RUN --mount=type=cache,id=cargo-registry,target=/cargo/registry \
+
-    --mount=type=cache,id=cargo-target,target=/src/dtmt/target \
+RUN cargo build --release --locked
-    cargo build --release --locked

.ci/image/Dockerfile.linux

@@ -0,0 +1,25 @@
+FROM rust:slim-bullseye
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+        curl \
+        git \
+        gpg \
+        jq \
+     	libatk1.0-dev \
+        libclang-13-dev \
+     	libglib2.0-dev \
+        libgtk-3-dev \
+        libpango1.0-dev \
+        libssl-dev \
+        libzstd-dev \
+        pkg-config; \
+    apt-get remove -y --auto-remove; \
+    rm -rf /var/lib/apt/lists/*; \
+    rustup default nightly
+
+WORKDIR /src/dtmt
+
+COPY *.so *.a /src/

.ci/image/Dockerfile.msvc

@@ -1,68 +1,13 @@
 # https://jake-shadle.github.io/xwin/
-FROM debian:bullseye-slim as xwin
+FROM dtmt-ci-base-linux
-
-ARG XWIN_VERSION=0.5.0
-ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
-ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
-
-RUN set -eux; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y \
-        tar \
-        ; \
-    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
-    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/bin --strip-components=1 $XWIN_PREFIX/xwin; \
-    apt-get remove -y --auto-remove; \
-    rm -rf \
-        /var/lib/apt/lists/* \
-        /root/$XWIN_PREFIX.tar.gz;
-
-RUN set -eux; \
-    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
-    xwin \
-        --log-level debug \
-        --cache-dir /root/.xwin-cache \
-        --manifest-version 16 \
-        --accept-license \
-        splat \
-        --output /xwin; \
-    # Even though this build step only exists temporary, to copy the
-    # final data out of, it still generates a cache entry on the Docker host.
-    # And to keep that to a minimum, we still delete the stuff we don't need.
-    rm -rf /root/.xwin-cache;
-
-FROM rust:slim-bullseye as linux
-
-RUN set -eux; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y \
-        build-essential \
-        curl \
-        git \
-        gpg \
-        jq \
-        libatk1.0-dev \
-        libclang-13-dev \
-        libglib2.0-dev \
-        libgtk-3-dev \
-        libpango1.0-dev \
-        libssl-dev \
-        libzstd-dev \
-        pkg-config; \
-    apt-get remove -y --auto-remove; \
-    rm -rf /var/lib/apt/lists/*; \
-    rustup default nightly
-
-WORKDIR /src/dtmt
-
-COPY lib/oodle/*.so lib/oodle/*.a /src/
-
-FROM linux as msvc
 
 ENV KEYRINGS /usr/local/share/keyrings
+ARG XWIN_VERSION=0.2.11
+ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
 
 ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key
 ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key
+ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
 
 RUN set -eux; \
     mkdir -p $KEYRINGS; \
@@ -81,7 +26,7 @@ RUN set -eux; \
         llvm-13 \
         lld-13 \
         winehq-staging \
-        ; \
+        tar; \
     # ensure that clang/clang++ are callable directly
     ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \
     # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages
@@ -99,15 +44,19 @@ RUN set -eux; \
     update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \
     rustup target add x86_64-pc-windows-msvc; \
     rustup component add rust-src; \
+    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
+    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \
+    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
+    xwin --accept-license splat --output /xwin; \
     # Remove unneeded files to reduce image size
     apt-get remove -y --auto-remove; \
     rm -rf \
+        .xwin-cache \
+        /usr/local/cargo/bin/xwin \
+        /root/$XWIN_PREFIX.tar.gz \
         /var/lib/apt/lists/* \
         /root/*.key;
 
-COPY lib/oodle/*.lib /src
-COPY --from=xwin /xwin /xwin
-
 # Note that we're using the full target triple for each variable instead of the
 # simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for
 # build dependencies that need to compile and execute in the host environment
@@ -134,3 +83,7 @@ ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \
 # Run wineboot just to setup the default WINEPREFIX so we don't do it every
 # container run
 RUN wine wineboot --init
+
+WORKDIR /src/dtmt
+
+COPY *.lib /src

.ci/pipelines/base-pipeline.yml

@@ -0,0 +1,43 @@
+---
+
+# The base pipeline that runs continuously, checks for branches and
+# creates a new pipeline instance for each of them.
+
+resource_types:
+- name: gitea-pr
+  type: registry-image
+  source:
+    repository: registry.local:5000/gitea-pr
+
+resources:
+- name: repo-pr
+  type: gitea-pr
+  source:
+    access_token: ((gitea_api_key))
+    owner: ((owner))
+    repo: ((repo))
+    url: https://git.sclu1034.dev
+- name: repo
+  type: git
+  source:
+    uri: https://git.sclu1034.dev/bitsquid_dt/dtmt
+
+jobs:
+  - name: set-pipelines
+    plan:
+    - in_parallel:
+      - get: repo-pr
+        trigger: true
+      - get: repo
+    - load_var: prs
+      file: repo-pr/prs.json
+    - across:
+      - var: pr
+        values: ((.:prs))
+      set_pipeline: dtmt-pr
+      file: repo/.ci/pipelines/pr.yml
+      vars:
+        pr: ((.:pr))
+        gitea_api_key: ((gitea_api_key))
+      instance_vars:
+        number: ((.:pr.number))

.ci/pipelines/base.yml

@@ -1,206 +0,0 @@
-# yaml-language-server: $schema=https://raw.githubusercontent.com/cappyzawa/concourse-pipeline-jsonschema/master/concourse_jsonschema.json#/definitions/Config
----
-
-# The actual CI pipeline that is run per branch
-resource_types:
-- name: gitea-package
-  type: registry-image
-  source:
-    repository: registry.local:5000/gitea-package
-
-- name: gitea-status
-  type: registry-image
-  source:
-    repository: registry.local:5000/gitea-status
-
-- name: gitea-pr
-  type: registry-image
-  source:
-    repository: registry.local:5000/gitea-pr
-
-
-resources:
-- name: repo
-  type: git
-  source:
-    uri: http://forgejo:3000/bitsquid_dt/dtmt
-    branch: master
-
-- name: repo-pr
-  type: gitea-pr
-  source:
-    access_token: ((gitea_api_key))
-    owner: ((owner))
-    repo: ((repo))
-    url: https://git.sclu1034.dev
-
-- name: gitea-package
-  type: gitea-package
-  source:
-    access_token: ((gitea_api_key))
-    url: http://forgejo:3000
-    owner: bitsquid_dt
-    type: generic
-    name: dtmt
-
-
-- name: status-build-msvc
-  type: gitea-status
-  source:
-    access_token: ((gitea_api_key))
-    url: http://forgejo:3000
-    owner: bitsquid_dt
-    repo: dtmt
-    context: build/msvc
-    description: "Build for the target platform: msvc"
-
-- name: status-build-linux
-  type: gitea-status
-  source:
-    access_token: ((gitea_api_key))
-    url: http://forgejo:3000
-    owner: bitsquid_dt
-    repo: dtmt
-    context: build/linux
-    description: "Build for the target platform: linux"
-
-
-jobs:
-- name: set-pipelines
-  plan:
-  - in_parallel:
-    - get: repo-pr
-      trigger: true
-
-    - get: repo
-
-  - load_var: prs
-    file: repo-pr/prs.json
-
-  - across:
-    - var: pr
-      values: ((.:prs))
-    set_pipeline: dtmt-pr
-    file: repo/.ci/pipelines/pr.yml
-    vars:
-      pr: ((.:pr))
-      gitea_api_key: ((gitea_api_key))
-    instance_vars:
-      number: ((.:pr.number))
-
-
-- name: build-msvc
-  on_success:
-    put: state-success
-    resource: status-build-msvc
-    no_get: true
-    params:
-      state: success
-      sha: ((.:git_sha))
-
-  on_failure:
-    put: state-failure
-    resource: status-build-msvc
-    no_get: true
-    params:
-      state: failure
-      sha: ((.:git_sha))
-
-  plan:
-  - get: repo
-    trigger: true
-
-  - load_var: git_sha
-    file: repo/.git/ref
-
-  - put: state-pending
-    resource: status-build-msvc
-    no_get: true
-    params:
-      state: pending
-      sha: ((.:git_sha))
-
-  - task: build
-    file: repo/.ci/tasks/build.yml
-    vars:
-      pr: ""
-      target: msvc
-      gitea_url: http://forgejo:3000
-      gitea_api_key: ((gitea_api_key))
-
-  - load_var: version_number
-    reveal: true
-    file: artifact/version
-
-  - put: package
-    resource: gitea-package
-    no_get: true
-    inputs:
-      - artifact
-    params:
-      version: ((.:version_number))
-      fail_fast: true
-      override: true
-      globs:
-        - artifact/dtmt
-        - artifact/dtmm
-        - artifact/*.exe
-        - artifact/*.sha256
-
-- name: build-linux
-  on_success:
-    put: state-success
-    resource: status-build-linux
-    no_get: true
-    params:
-      state: success
-      sha: ((.:git_sha))
-
-  on_failure:
-    put: state-failure
-    resource: status-build-linux
-    no_get: true
-    params:
-      state: failure
-      sha: ((.:git_sha))
-
-  plan:
-  - get: repo
-    trigger: true
-
-  - load_var: git_sha
-    file: repo/.git/ref
-
-  - put: state-pending
-    resource: status-build-linux
-    no_get: true
-    params:
-      state: pending
-      sha: ((.:git_sha))
-
-  - task: build
-    file: repo/.ci/tasks/build.yml
-    vars:
-      pr: ""
-      target: linux
-      gitea_url: http://forgejo:3000
-      gitea_api_key: ((gitea_api_key))
-
-  - load_var: version_number
-    reveal: true
-    file: artifact/version
-
-  - put: package
-    resource: gitea-package
-    no_get: true
-    inputs:
-      - artifact
-    params:
-      version: ((.:version_number))
-      fail_fast: true
-      override: true
-      globs:
-        - artifact/dtmt
-        - artifact/dtmm
-        - artifact/*.exe
-        - artifact/*.sha256

.ci/tasks/build.sh

@@ -19,9 +19,7 @@ install_artifact() {
 
 cd "repo"
 
-PR=${PR:-}
+if [ -n "${PR:-}" ]; then
-
-if [ -n "$PR" ]; then
     title "PR: $(echo "$PR" | jq '.number') - $(echo "$PR" | jq '.title')"
     ref="pr-$(echo "$PR" | jq '.number')-$(git rev-parse --short "$(cat .git/ref || echo "HEAD")" 2>/dev/null || echo 'manual')"
 else

CHANGELOG.adoc

@@ -16,7 +16,6 @@
 - dtmt: add utility to migrate mod projects
 - dtmm: reset dtkit-patch installations
 - sdk: implement decompiling Lua files
-- dtmm: fetch cover image for Nexus mods
 - dtmm: fetch file version for Nexus mods
 
 === Fixed

Justfile

@@ -16,21 +16,22 @@ build-image-msvc:
 build-image-linux:
     docker build -f .ci/Dockerfile.linux .
 
-ci-image:
+ci-image: ci-image-msvc ci-image-linux
-    # The MSVC image depends on the Linux image. So by building that first,
+
-    # we actually build both, and cache them, so that "building" the
+ci-image-msvc: ci-image-linux
-    # Linux image afterwards merely needs to pull the cache.
+    docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image
-    docker build --target msvc -t dtmt-ci-base-msvc -f .ci/image/Dockerfile .
-    docker build --target linux -t dtmt-ci-base-linux -f .ci/image/Dockerfile .
     docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc
-    docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
     docker push registry.sclu1034.dev/dtmt-ci-base-msvc
+
+ci-image-linux:
+    docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image
+    docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
     docker push registry.sclu1034.dev/dtmt-ci-base-linux
 
 set-base-pipeline:
     fly -t {{fly_target}} set-pipeline \
-        --pipeline dtmt \
+        --pipeline dtmt-prs \
-        --config .ci/pipelines/base.yml \
+        --config .ci/pipelines/base-pipeline.yml \
         -v gitea_api_key=${GITEA_API_KEY} \
         -v owner=bitsquid_dt \
         -v repo=dtmt

crates/dtmm/src/controller/import.rs

@@ -27,16 +27,6 @@ fn find_archive_file<R: Read + Seek>(
     path
 }
 
-fn image_data_to_buffer(data: impl AsRef<[u8]>) -> Result<ImageBuf> {
-    // Druid somehow doesn't return an error compatible with eyre, here.
-    // So we have to wrap through `Display` manually.
-    ImageBuf::from_data(data.as_ref()).map_err(|err| {
-        Report::msg(err.to_string())
-            .wrap_err("Invalid image data")
-            .suggestion("Supported formats are: PNG, JPEG, Bitmap and WebP")
-    })
-}
-
 // Runs the content of a `.mod` file to extract what data we can get
 // from legacy mods.
 // 1. Create a global function `new_mod` that stores
@@ -448,8 +438,6 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
         None
     };
 
-    tracing::trace!(?nexus);
-
     let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
 
     if tracing::enabled!(tracing::Level::DEBUG) {
@@ -464,13 +452,7 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
     let (mut mod_cfg, root) =
         extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?;
     tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id);
-
+    tracing::debug!(root, ?mod_cfg);
-    let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
-    let dest = mod_dir.join(&mod_cfg.id);
-    tracing::trace!("Creating mods directory {}", dest.display());
-    fs::create_dir_all(&dest)
-        .await
-        .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
 
     let image = if let Some(path) = &mod_cfg.image {
         let name = archive
@@ -486,40 +468,33 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
         f.read_to_end(&mut buf)
             .wrap_err("Failed to read file index from archive")?;
 
-        let img = image_data_to_buffer(buf)?;
+        // Druid somehow doesn't return an error compatible with eyre, here.
-        Some(img)
+        // So we have to wrap through `Display` manually.
-    } else if let Some((nexus, _)) = &nexus {
+        let img = match ImageBuf::from_data(&buf) {
-        let api = NexusApi::new(state.nexus_api_key.to_string())?;
+            Ok(img) => img,
-        let url = nexus.picture_url.as_ref();
-        let data = api
-            .picture(url)
-            .await
-            .wrap_err_with(|| format!("Failed to download Nexus image from '{}'", url))?;
-
-        let img = image_data_to_buffer(&data)?;
-
-        let name = "image.bin";
-        let path = dest.join(name);
-        match fs::write(&path, &data).await {
-            Ok(_) => {
-                mod_cfg.image = Some(name.into());
-                Some(img)
-            }
             Err(err) => {
-                let err = Report::new(err).wrap_err(format!(
+                let err = Report::msg(err.to_string())
-                    "Failed to write Nexus picture to file '{}'",
+                    .wrap_err("Invalid image data")
-                    path.display()
+                    .note("Supported formats are: PNG, JPEG, Bitmap and WebP")
-                ));
+                    .suggestion("Contact the mod author to fix this");
-                tracing::error!("{:?}", err);
+                return Err(err);
-                None
             }
-        }
+        };
+
+        Some(img)
     } else {
         None
     };
 
     tracing::trace!(?image);
-    tracing::debug!(root, ?mod_cfg);
+
+    let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
+    let dest = mod_dir.join(&mod_cfg.id);
+
+    tracing::trace!("Creating mods directory {}", dest.display());
+    fs::create_dir_all(&dest)
+        .await
+        .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
 
     let packages = if mod_cfg.bundled {
         extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")?

crates/dtmm/src/state/data.rs

@@ -78,7 +78,6 @@ pub(crate) struct NexusInfo {
     pub author: String,
     pub summary: Arc<String>,
     pub description: Arc<String>,
-    pub picture_url: Arc<String>,
 }
 
 impl From<NexusMod> for NexusInfo {
@@ -90,7 +89,6 @@ impl From<NexusMod> for NexusInfo {
             author: value.author,
             summary: Arc::new(value.summary),
             description: Arc::new(value.description),
-            picture_url: Arc::new(value.picture_url.into()),
         }
     }
 }

lib/nexusmods/src/lib.rs

@@ -4,7 +4,7 @@ use std::convert::Infallible;
 use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::header::{HeaderMap, HeaderValue, InvalidHeaderValue};
-use reqwest::{Client, IntoUrl, RequestBuilder, Url};
+use reqwest::{Client, RequestBuilder, Url};
 use serde::Deserialize;
 use thiserror::Error;
 
@@ -126,16 +126,6 @@ impl Api {
         Ok(file.version)
     }
 
-    #[tracing::instrument(skip(self))]
-    pub async fn picture(&self, url: impl IntoUrl + std::fmt::Debug) -> Result<Vec<u8>> {
-        let res = self.client.get(url).send().await?.error_for_status()?;
-
-        res.bytes()
-            .await
-            .map(|bytes| bytes.to_vec())
-            .map_err(From::from)
-    }
-
     pub fn parse_file_name<S: AsRef<str>>(
         name: S,
     ) -> Option<(String, u64, String, OffsetDateTime)> {