Merge pull request 'Improve CI' (#153) from feat/ci into master

Reviewed-on: #153

.ci/Dockerfile.linux

@@ -1,35 +1,7 @@
 FROM dtmt-ci-base-linux
 
-# Create dummy crates and copy their Cargo.toml, so that dependencies can be cached
-RUN set -e; \
-    cargo new --bin crates/dtmt; \
-    cargo new --bin crates/dtmm; \
-    cargo new --lib lib/dtmt-shared; \
-    cargo new --lib lib/nexusmods; \
-    cargo new --lib lib/sdk; \
-    cargo new --lib lib/serde_sjson; \
-    cargo new --lib lib/ansi-parser
-
-COPY Cargo.toml Cargo.lock /src/dtmt/
-COPY crates/dtmt/Cargo.toml /src/dtmt/crates/dtmt/
-COPY crates/dtmm/Cargo.toml /src/dtmt/crates/dtmm/
-COPY lib/dtmt-shared/Cargo.toml /src/dtmt/lib/dtmt-shared/
-COPY lib/nexusmods/Cargo.toml /src/dtmt/lib/nexusmods/
-COPY lib/sdk/Cargo.toml /src/dtmt/lib/sdk/
-COPY lib/serde_sjson/Cargo.toml /src/dtmt/lib/serde_sjson/
-COPY lib/ansi-parser/Cargo.toml /src/dtmt/lib/ansi-parser/
-
-# Crates with build scripts cannot be split that way, but they shouldn't change too often
-COPY lib/luajit2-sys /src/dtmt/lib/luajit2-sys
-COPY lib/oodle /src/dtmt/lib/oodle
-# color-eyre needs to be copied, too, then, as it's used by `oodle`
-COPY lib/color-eyre /src/dtmt/lib/color-eyre
-COPY --from=dtmt-ci-base-linux /src/*.lib /src/dtmt/lib/oodle/
-
-RUN cargo build --release --locked
-RUN rm -r crates lib
-
 COPY . /src/dtmt
 COPY --from=dtmt-ci-base-linux /src/*.lib /src/*.so /src/dtmt/lib/oodle/
-
-RUN cargo build --release --locked
+RUN --mount=type=cache,id=cargo-registry,target=/cargo/registry \
+    --mount=type=cache,id=cargo-target,target=/src/dtmt/target \
+    cargo build --release --locked

.ci/image/Dockerfile

@@ -1,13 +1,68 @@
 # https://jake-shadle.github.io/xwin/
-FROM dtmt-ci-base-linux
+FROM debian:bullseye-slim as xwin
+
+ARG XWIN_VERSION=0.5.0
+ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
+ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        tar \
+        ; \
+    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
+    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/bin --strip-components=1 $XWIN_PREFIX/xwin; \
+    apt-get remove -y --auto-remove; \
+    rm -rf \
+        /var/lib/apt/lists/* \
+        /root/$XWIN_PREFIX.tar.gz;
+
+RUN set -eux; \
+    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
+    xwin \
+        --log-level debug \
+        --cache-dir /root/.xwin-cache \
+        --manifest-version 16 \
+        --accept-license \
+        splat \
+        --output /xwin; \
+    # Even though this build step only exists temporary, to copy the
+    # final data out of, it still generates a cache entry on the Docker host.
+    # And to keep that to a minimum, we still delete the stuff we don't need.
+    rm -rf /root/.xwin-cache;
+
+FROM rust:slim-bullseye as linux
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+        curl \
+        git \
+        gpg \
+        jq \
+        libatk1.0-dev \
+        libclang-13-dev \
+        libglib2.0-dev \
+        libgtk-3-dev \
+        libpango1.0-dev \
+        libssl-dev \
+        libzstd-dev \
+        pkg-config; \
+    apt-get remove -y --auto-remove; \
+    rm -rf /var/lib/apt/lists/*; \
+    rustup default nightly
+
+WORKDIR /src/dtmt
+
+COPY lib/oodle/*.so lib/oodle/*.a /src/
+
+FROM linux as msvc
 
 ENV KEYRINGS /usr/local/share/keyrings
-ARG XWIN_VERSION=0.2.11
-ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
 
 ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key
 ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key
-ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
 
 RUN set -eux; \
     mkdir -p $KEYRINGS; \
@@ -26,7 +81,7 @@ RUN set -eux; \
         llvm-13 \
         lld-13 \
         winehq-staging \
-        tar; \
+        ; \
     # ensure that clang/clang++ are callable directly
     ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \
     # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages
@@ -44,19 +99,15 @@ RUN set -eux; \
     update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \
     rustup target add x86_64-pc-windows-msvc; \
     rustup component add rust-src; \
-    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
-    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \
-    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
-    xwin --accept-license splat --output /xwin; \
     # Remove unneeded files to reduce image size
     apt-get remove -y --auto-remove; \
     rm -rf \
-        .xwin-cache \
-        /usr/local/cargo/bin/xwin \
-        /root/$XWIN_PREFIX.tar.gz \
         /var/lib/apt/lists/* \
         /root/*.key;
 
+COPY lib/oodle/*.lib /src
+COPY --from=xwin /xwin /xwin
+
 # Note that we're using the full target triple for each variable instead of the
 # simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for
 # build dependencies that need to compile and execute in the host environment
@@ -83,7 +134,3 @@ ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \
 # Run wineboot just to setup the default WINEPREFIX so we don't do it every
 # container run
 RUN wine wineboot --init
-
-WORKDIR /src/dtmt
-
-COPY *.lib /src

.ci/image/Dockerfile.linux

@@ -1,25 +0,0 @@
-FROM rust:slim-bullseye
-
-RUN set -eux; \
-    apt-get update; \
-    apt-get install --no-install-recommends -y \
-        build-essential \
-        curl \
-        git \
-        gpg \
-        jq \
-     	libatk1.0-dev \
-        libclang-13-dev \
-     	libglib2.0-dev \
-        libgtk-3-dev \
-        libpango1.0-dev \
-        libssl-dev \
-        libzstd-dev \
-        pkg-config; \
-    apt-get remove -y --auto-remove; \
-    rm -rf /var/lib/apt/lists/*; \
-    rustup default nightly
-
-WORKDIR /src/dtmt
-
-COPY *.so *.a /src/

.ci/pipelines/base.yml

@@ -123,6 +123,7 @@ jobs:
   - task: build
     file: repo/.ci/tasks/build.yml
     vars:
+      pr: ""
       target: msvc
       gitea_url: http://forgejo:3000
       gitea_api_key: ((gitea_api_key))
@@ -180,6 +181,7 @@ jobs:
   - task: build
     file: repo/.ci/tasks/build.yml
     vars:
+      pr: ""
       target: linux
       gitea_url: http://forgejo:3000
       gitea_api_key: ((gitea_api_key))

Justfile

@@ -16,16 +16,15 @@ build-image-msvc:
 build-image-linux:
     docker build -f .ci/Dockerfile.linux .
 
-ci-image: ci-image-msvc ci-image-linux
-
-ci-image-msvc: ci-image-linux
-    docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image
+ci-image:
+    # The MSVC image depends on the Linux image. So by building that first,
+    # we actually build both, and cache them, so that "building" the
+    # Linux image afterwards merely needs to pull the cache.
+    docker build --target msvc -t dtmt-ci-base-msvc -f .ci/image/Dockerfile .
+    docker build --target linux -t dtmt-ci-base-linux -f .ci/image/Dockerfile .
     docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc
-    docker push registry.sclu1034.dev/dtmt-ci-base-msvc
-
-ci-image-linux:
-    docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image
     docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
+    docker push registry.sclu1034.dev/dtmt-ci-base-msvc
     docker push registry.sclu1034.dev/dtmt-ci-base-linux
 
 set-base-pipeline: