Merge branch 'master' into feat/nexus-file-version

* master:
  dtmm: Fix writing Nexus image to disk
  dtmm: Fix importing from `.mod` file
  ci: Combine Dockerfiles into multi-stage build
  ci: Rework MSVC image building
  ci: Fix base pipeline
  ci: Improve caching setup for image building
  ci: Create build artifacts for commits on master
  ci: Fix undefined variable
  Add changelog entry
  dtmm: Fetch mod image from Nexus

.ci/Dockerfile.linux

@@ -1,35 +1,7 @@
 FROM dtmt-ci-base-linux
 
-# Create dummy crates and copy their Cargo.toml, so that dependencies can be cached
-RUN set -e; \
-    cargo new --bin crates/dtmt; \
-    cargo new --bin crates/dtmm; \
-    cargo new --lib lib/dtmt-shared; \
-    cargo new --lib lib/nexusmods; \
-    cargo new --lib lib/sdk; \
-    cargo new --lib lib/serde_sjson; \
-    cargo new --lib lib/ansi-parser
-
-COPY Cargo.toml Cargo.lock /src/dtmt/
-COPY crates/dtmt/Cargo.toml /src/dtmt/crates/dtmt/
-COPY crates/dtmm/Cargo.toml /src/dtmt/crates/dtmm/
-COPY lib/dtmt-shared/Cargo.toml /src/dtmt/lib/dtmt-shared/
-COPY lib/nexusmods/Cargo.toml /src/dtmt/lib/nexusmods/
-COPY lib/sdk/Cargo.toml /src/dtmt/lib/sdk/
-COPY lib/serde_sjson/Cargo.toml /src/dtmt/lib/serde_sjson/
-COPY lib/ansi-parser/Cargo.toml /src/dtmt/lib/ansi-parser/
-
-# Crates with build scripts cannot be split that way, but they shouldn't change too often
-COPY lib/luajit2-sys /src/dtmt/lib/luajit2-sys
-COPY lib/oodle /src/dtmt/lib/oodle
-# color-eyre needs to be copied, too, then, as it's used by `oodle`
-COPY lib/color-eyre /src/dtmt/lib/color-eyre
-COPY --from=dtmt-ci-base-linux /src/*.lib /src/dtmt/lib/oodle/
-
-RUN cargo build --release --locked
-RUN rm -r crates lib
-
 COPY . /src/dtmt
 COPY --from=dtmt-ci-base-linux /src/*.lib /src/*.so /src/dtmt/lib/oodle/
-
+RUN --mount=type=cache,id=cargo-registry,target=/cargo/registry \
-RUN cargo build --release --locked
+    --mount=type=cache,id=cargo-target,target=/src/dtmt/target \
+    cargo build --release --locked

.ci/image/Dockerfile

@@ -1,13 +1,68 @@
 # https://jake-shadle.github.io/xwin/
-FROM dtmt-ci-base-linux
+FROM debian:bullseye-slim as xwin
+
+ARG XWIN_VERSION=0.5.0
+ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
+ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        tar \
+        ; \
+    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
+    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/bin --strip-components=1 $XWIN_PREFIX/xwin; \
+    apt-get remove -y --auto-remove; \
+    rm -rf \
+        /var/lib/apt/lists/* \
+        /root/$XWIN_PREFIX.tar.gz;
+
+RUN set -eux; \
+    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
+    xwin \
+        --log-level debug \
+        --cache-dir /root/.xwin-cache \
+        --manifest-version 16 \
+        --accept-license \
+        splat \
+        --output /xwin; \
+    # Even though this build step only exists temporary, to copy the
+    # final data out of, it still generates a cache entry on the Docker host.
+    # And to keep that to a minimum, we still delete the stuff we don't need.
+    rm -rf /root/.xwin-cache;
+
+FROM rust:slim-bullseye as linux
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+        curl \
+        git \
+        gpg \
+        jq \
+        libatk1.0-dev \
+        libclang-13-dev \
+        libglib2.0-dev \
+        libgtk-3-dev \
+        libpango1.0-dev \
+        libssl-dev \
+        libzstd-dev \
+        pkg-config; \
+    apt-get remove -y --auto-remove; \
+    rm -rf /var/lib/apt/lists/*; \
+    rustup default nightly
+
+WORKDIR /src/dtmt
+
+COPY lib/oodle/*.so lib/oodle/*.a /src/
+
+FROM linux as msvc
 
 ENV KEYRINGS /usr/local/share/keyrings
-ARG XWIN_VERSION=0.2.11
-ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
 
 ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key
 ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key
-ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
 
 RUN set -eux; \
     mkdir -p $KEYRINGS; \
@@ -26,7 +81,7 @@ RUN set -eux; \
         llvm-13 \
         lld-13 \
         winehq-staging \
-        tar; \
+        ; \
     # ensure that clang/clang++ are callable directly
     ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \
     # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages
@@ -44,19 +99,15 @@ RUN set -eux; \
     update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \
     rustup target add x86_64-pc-windows-msvc; \
     rustup component add rust-src; \
-    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
-    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \
-    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
-    xwin --accept-license splat --output /xwin; \
     # Remove unneeded files to reduce image size
     apt-get remove -y --auto-remove; \
     rm -rf \
-        .xwin-cache \
-        /usr/local/cargo/bin/xwin \
-        /root/$XWIN_PREFIX.tar.gz \
         /var/lib/apt/lists/* \
         /root/*.key;
 
+COPY lib/oodle/*.lib /src
+COPY --from=xwin /xwin /xwin
+
 # Note that we're using the full target triple for each variable instead of the
 # simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for
 # build dependencies that need to compile and execute in the host environment
@@ -83,7 +134,3 @@ ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \
 # Run wineboot just to setup the default WINEPREFIX so we don't do it every
 # container run
 RUN wine wineboot --init
-
-WORKDIR /src/dtmt
-
-COPY *.lib /src

.ci/image/Dockerfile.linux

@@ -1,25 +0,0 @@
-FROM rust:slim-bullseye
-
-RUN set -eux; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y \
-        build-essential \
-        curl \
-        git \
-        gpg \
-        jq \
-     	libatk1.0-dev \
-        libclang-13-dev \
-     	libglib2.0-dev \
-        libgtk-3-dev \
-        libpango1.0-dev \
-        libssl-dev \
-        libzstd-dev \
-        pkg-config; \
-    apt-get remove -y --auto-remove; \
-    rm -rf /var/lib/apt/lists/*; \
-    rustup default nightly
-
-WORKDIR /src/dtmt
-
-COPY *.so *.a /src/

.ci/pipelines/base-pipeline.yml

@@ -1,43 +0,0 @@
----
-
-# The base pipeline that runs continuously, checks for branches and
-# creates a new pipeline instance for each of them.
-
-resource_types:
-- name: gitea-pr
-  type: registry-image
-  source:
-    repository: registry.local:5000/gitea-pr
-
-resources:
-- name: repo-pr
-  type: gitea-pr
-  source:
-    access_token: ((gitea_api_key))
-    owner: ((owner))
-    repo: ((repo))
-    url: https://git.sclu1034.dev
-- name: repo
-  type: git
-  source:
-    uri: https://git.sclu1034.dev/bitsquid_dt/dtmt
-
-jobs:
-  - name: set-pipelines
-    plan:
-    - in_parallel:
-      - get: repo-pr
-        trigger: true
-      - get: repo
-    - load_var: prs
-      file: repo-pr/prs.json
-    - across:
-      - var: pr
-        values: ((.:prs))
-      set_pipeline: dtmt-pr
-      file: repo/.ci/pipelines/pr.yml
-      vars:
-        pr: ((.:pr))
-        gitea_api_key: ((gitea_api_key))
-      instance_vars:
-        number: ((.:pr.number))

.ci/pipelines/base.yml

@@ -0,0 +1,206 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/cappyzawa/concourse-pipeline-jsonschema/master/concourse_jsonschema.json#/definitions/Config
+---
+
+# The actual CI pipeline that is run per branch
+resource_types:
+- name: gitea-package
+  type: registry-image
+  source:
+    repository: registry.local:5000/gitea-package
+
+- name: gitea-status
+  type: registry-image
+  source:
+    repository: registry.local:5000/gitea-status
+
+- name: gitea-pr
+  type: registry-image
+  source:
+    repository: registry.local:5000/gitea-pr
+
+
+resources:
+- name: repo
+  type: git
+  source:
+    uri: http://forgejo:3000/bitsquid_dt/dtmt
+    branch: master
+
+- name: repo-pr
+  type: gitea-pr
+  source:
+    access_token: ((gitea_api_key))
+    owner: ((owner))
+    repo: ((repo))
+    url: https://git.sclu1034.dev
+
+- name: gitea-package
+  type: gitea-package
+  source:
+    access_token: ((gitea_api_key))
+    url: http://forgejo:3000
+    owner: bitsquid_dt
+    type: generic
+    name: dtmt
+
+
+- name: status-build-msvc
+  type: gitea-status
+  source:
+    access_token: ((gitea_api_key))
+    url: http://forgejo:3000
+    owner: bitsquid_dt
+    repo: dtmt
+    context: build/msvc
+    description: "Build for the target platform: msvc"
+
+- name: status-build-linux
+  type: gitea-status
+  source:
+    access_token: ((gitea_api_key))
+    url: http://forgejo:3000
+    owner: bitsquid_dt
+    repo: dtmt
+    context: build/linux
+    description: "Build for the target platform: linux"
+
+
+jobs:
+- name: set-pipelines
+  plan:
+  - in_parallel:
+    - get: repo-pr
+      trigger: true
+
+    - get: repo
+
+  - load_var: prs
+    file: repo-pr/prs.json
+
+  - across:
+    - var: pr
+      values: ((.:prs))
+    set_pipeline: dtmt-pr
+    file: repo/.ci/pipelines/pr.yml
+    vars:
+      pr: ((.:pr))
+      gitea_api_key: ((gitea_api_key))
+    instance_vars:
+      number: ((.:pr.number))
+
+
+- name: build-msvc
+  on_success:
+    put: state-success
+    resource: status-build-msvc
+    no_get: true
+    params:
+      state: success
+      sha: ((.:git_sha))
+
+  on_failure:
+    put: state-failure
+    resource: status-build-msvc
+    no_get: true
+    params:
+      state: failure
+      sha: ((.:git_sha))
+
+  plan:
+  - get: repo
+    trigger: true
+
+  - load_var: git_sha
+    file: repo/.git/ref
+
+  - put: state-pending
+    resource: status-build-msvc
+    no_get: true
+    params:
+      state: pending
+      sha: ((.:git_sha))
+
+  - task: build
+    file: repo/.ci/tasks/build.yml
+    vars:
+      pr: ""
+      target: msvc
+      gitea_url: http://forgejo:3000
+      gitea_api_key: ((gitea_api_key))
+
+  - load_var: version_number
+    reveal: true
+    file: artifact/version
+
+  - put: package
+    resource: gitea-package
+    no_get: true
+    inputs:
+      - artifact
+    params:
+      version: ((.:version_number))
+      fail_fast: true
+      override: true
+      globs:
+        - artifact/dtmt
+        - artifact/dtmm
+        - artifact/*.exe
+        - artifact/*.sha256
+
+- name: build-linux
+  on_success:
+    put: state-success
+    resource: status-build-linux
+    no_get: true
+    params:
+      state: success
+      sha: ((.:git_sha))
+
+  on_failure:
+    put: state-failure
+    resource: status-build-linux
+    no_get: true
+    params:
+      state: failure
+      sha: ((.:git_sha))
+
+  plan:
+  - get: repo
+    trigger: true
+
+  - load_var: git_sha
+    file: repo/.git/ref
+
+  - put: state-pending
+    resource: status-build-linux
+    no_get: true
+    params:
+      state: pending
+      sha: ((.:git_sha))
+
+  - task: build
+    file: repo/.ci/tasks/build.yml
+    vars:
+      pr: ""
+      target: linux
+      gitea_url: http://forgejo:3000
+      gitea_api_key: ((gitea_api_key))
+
+  - load_var: version_number
+    reveal: true
+    file: artifact/version
+
+  - put: package
+    resource: gitea-package
+    no_get: true
+    inputs:
+      - artifact
+    params:
+      version: ((.:version_number))
+      fail_fast: true
+      override: true
+      globs:
+        - artifact/dtmt
+        - artifact/dtmm
+        - artifact/*.exe
+        - artifact/*.sha256

.ci/tasks/build.sh

@@ -19,7 +19,9 @@ install_artifact() {
 
 cd "repo"
 
-if [ -n "${PR:-}" ]; then
+PR=${PR:-}
+
+if [ -n "$PR" ]; then
     title "PR: $(echo "$PR" | jq '.number') - $(echo "$PR" | jq '.title')"
     ref="pr-$(echo "$PR" | jq '.number')-$(git rev-parse --short "$(cat .git/ref || echo "HEAD")" 2>/dev/null || echo 'manual')"
 else

CHANGELOG.adoc

@@ -16,6 +16,7 @@
 - dtmt: add utility to migrate mod projects
 - dtmm: reset dtkit-patch installations
 - sdk: implement decompiling Lua files
+- dtmm: fetch cover image for Nexus mods
 - dtmm: fetch file version for Nexus mods
 
 === Fixed

Justfile

@@ -16,22 +16,21 @@ build-image-msvc:
 build-image-linux:
     docker build -f .ci/Dockerfile.linux .
 
-ci-image: ci-image-msvc ci-image-linux
+ci-image:
-
+    # The MSVC image depends on the Linux image. So by building that first,
-ci-image-msvc: ci-image-linux
+    # we actually build both, and cache them, so that "building" the
-    docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image
+    # Linux image afterwards merely needs to pull the cache.
+    docker build --target msvc -t dtmt-ci-base-msvc -f .ci/image/Dockerfile .
+    docker build --target linux -t dtmt-ci-base-linux -f .ci/image/Dockerfile .
     docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc
-    docker push registry.sclu1034.dev/dtmt-ci-base-msvc
-
-ci-image-linux:
-    docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image
     docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
+    docker push registry.sclu1034.dev/dtmt-ci-base-msvc
     docker push registry.sclu1034.dev/dtmt-ci-base-linux
 
 set-base-pipeline:
     fly -t {{fly_target}} set-pipeline \
-        --pipeline dtmt-prs \
+        --pipeline dtmt \
-        --config .ci/pipelines/base-pipeline.yml \
+        --config .ci/pipelines/base.yml \
         -v gitea_api_key=${GITEA_API_KEY} \
         -v owner=bitsquid_dt \
         -v repo=dtmt

crates/dtmm/src/controller/import.rs

@@ -27,6 +27,16 @@ fn find_archive_file<R: Read + Seek>(
     path
 }
 
+fn image_data_to_buffer(data: impl AsRef<[u8]>) -> Result<ImageBuf> {
+    // Druid somehow doesn't return an error compatible with eyre, here.
+    // So we have to wrap through `Display` manually.
+    ImageBuf::from_data(data.as_ref()).map_err(|err| {
+        Report::msg(err.to_string())
+            .wrap_err("Invalid image data")
+            .suggestion("Supported formats are: PNG, JPEG, Bitmap and WebP")
+    })
+}
+
 // Runs the content of a `.mod` file to extract what data we can get
 // from legacy mods.
 // 1. Create a global function `new_mod` that stores
@@ -438,6 +448,8 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
         None
     };
 
+    tracing::trace!(?nexus);
+
     let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
 
     if tracing::enabled!(tracing::Level::DEBUG) {
@@ -452,7 +464,13 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
     let (mut mod_cfg, root) =
         extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?;
     tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id);
-    tracing::debug!(root, ?mod_cfg);
+
+    let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
+    let dest = mod_dir.join(&mod_cfg.id);
+    tracing::trace!("Creating mods directory {}", dest.display());
+    fs::create_dir_all(&dest)
+        .await
+        .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
 
     let image = if let Some(path) = &mod_cfg.image {
         let name = archive
@@ -468,33 +486,40 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
         f.read_to_end(&mut buf)
             .wrap_err("Failed to read file index from archive")?;
 
-        // Druid somehow doesn't return an error compatible with eyre, here.
+        let img = image_data_to_buffer(buf)?;
-        // So we have to wrap through `Display` manually.
-        let img = match ImageBuf::from_data(&buf) {
-            Ok(img) => img,
-            Err(err) => {
-                let err = Report::msg(err.to_string())
-                    .wrap_err("Invalid image data")
-                    .note("Supported formats are: PNG, JPEG, Bitmap and WebP")
-                    .suggestion("Contact the mod author to fix this");
-                return Err(err);
-            }
-        };
-
         Some(img)
+    } else if let Some((nexus, _)) = &nexus {
+        let api = NexusApi::new(state.nexus_api_key.to_string())?;
+        let url = nexus.picture_url.as_ref();
+        let data = api
+            .picture(url)
+            .await
+            .wrap_err_with(|| format!("Failed to download Nexus image from '{}'", url))?;
+
+        let img = image_data_to_buffer(&data)?;
+
+        let name = "image.bin";
+        let path = dest.join(name);
+        match fs::write(&path, &data).await {
+            Ok(_) => {
+                mod_cfg.image = Some(name.into());
+                Some(img)
+            }
+            Err(err) => {
+                let err = Report::new(err).wrap_err(format!(
+                    "Failed to write Nexus picture to file '{}'",
+                    path.display()
+                ));
+                tracing::error!("{:?}", err);
+                None
+            }
+        }
     } else {
         None
     };
 
     tracing::trace!(?image);
-
+    tracing::debug!(root, ?mod_cfg);
-    let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
-    let dest = mod_dir.join(&mod_cfg.id);
-
-    tracing::trace!("Creating mods directory {}", dest.display());
-    fs::create_dir_all(&dest)
-        .await
-        .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
 
     let packages = if mod_cfg.bundled {
         extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")?

crates/dtmm/src/state/data.rs

@@ -78,6 +78,7 @@ pub(crate) struct NexusInfo {
     pub author: String,
     pub summary: Arc<String>,
     pub description: Arc<String>,
+    pub picture_url: Arc<String>,
 }
 
 impl From<NexusMod> for NexusInfo {
@@ -89,6 +90,7 @@ impl From<NexusMod> for NexusInfo {
             author: value.author,
             summary: Arc::new(value.summary),
             description: Arc::new(value.description),
+            picture_url: Arc::new(value.picture_url.into()),
         }
     }
 }

lib/nexusmods/src/lib.rs

@@ -4,7 +4,7 @@ use std::convert::Infallible;
 use lazy_static::lazy_static;
 use regex::Regex;
 use reqwest::header::{HeaderMap, HeaderValue, InvalidHeaderValue};
-use reqwest::{Client, RequestBuilder, Url};
+use reqwest::{Client, IntoUrl, RequestBuilder, Url};
 use serde::Deserialize;
 use thiserror::Error;
 
@@ -126,6 +126,16 @@ impl Api {
         Ok(file.version)
     }
 
+    #[tracing::instrument(skip(self))]
+    pub async fn picture(&self, url: impl IntoUrl + std::fmt::Debug) -> Result<Vec<u8>> {
+        let res = self.client.get(url).send().await?.error_for_status()?;
+
+        res.bytes()
+            .await
+            .map(|bytes| bytes.to_vec())
+            .map_err(From::from)
+    }
+
     pub fn parse_file_name<S: AsRef<str>>(
         name: S,
     ) -> Option<(String, u64, String, OffsetDateTime)> {