Merge pull request 'Implement CI' (#83) from feat/cross-compile into master

Reviewed-on: #83

.ci/image/Dockerfile.linux

@@ -0,0 +1,22 @@
+FROM rust:slim-bullseye
+
+RUN set -eux; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+     	libatk1.0-dev \
+        libclang-13-dev \
+     	libglib2.0-dev \
+        libgtk-3-dev \
+        libpango1.0-dev \
+        libssl-dev \
+        libzstd-dev \
+        pkg-config; \
+    apt-get remove -y --auto-remove; \
+    rm -rf /var/lib/apt/lists/*; \
+    rustup default nightly
+
+WORKDIR /src/dtmt
+
+COPY *.so /src
+

.ci/image/Dockerfile.msvc

@@ -0,0 +1,103 @@
+# https://jake-shadle.github.io/xwin/
+FROM rust:slim-bullseye
+
+ENV KEYRINGS /usr/local/share/keyrings
+ARG XWIN_VERSION=0.2.11
+ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
+
+ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key
+ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key
+ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
+
+RUN set -eux; \
+    mkdir -p $KEYRINGS; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+        gpg \
+        curl; \
+    # clang/lld/llvm
+    gpg --dearmor > $KEYRINGS/llvm.gpg < /root/llvm-snapshot.gpg.key; \
+    # wine
+    gpg --dearmor > $KEYRINGS/winehq.gpg < /root/winehq.key; \
+    echo "deb [signed-by=$KEYRINGS/llvm.gpg] http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" > /etc/apt/sources.list.d/llvm.list; \
+    echo "deb [signed-by=$KEYRINGS/winehq.gpg] https://dl.winehq.org/wine-builds/debian/ bullseye main" > /etc/apt/sources.list.d/winehq.list; \
+    dpkg --add-architecture i386; \
+    apt-get update; \
+    apt-get install --no-install-recommends -y \
+        build-essential \
+     	libatk1.0-dev \
+        libclang-13-dev \
+     	libglib2.0-dev \
+        libgtk-3-dev \
+        libpango1.0-dev \
+        libssl-dev \
+        libzstd-dev \
+        pkg-config \
+        gcc-mingw-w64-x86-64 \
+     	# g++-mingw-w64-x86-64 \
+        clang-13 \
+        llvm-13 \
+        lld-13 \
+        winehq-staging \
+        tar; \
+    # ensure that clang/clang++ are callable directly
+    ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \
+    # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages
+    ln -s clang-13 /usr/bin/clang-cl && ln -s llvm-ar-13 /usr/bin/llvm-lib && ln -s lld-link-13 /usr/bin/lld-link; \
+    # Verify the symlinks are correct
+    clang++ -v; \
+    ld.lld -v; \
+    # Doesn't have an actual -v/--version flag, but it still exits with 0
+    llvm-lib -v; \
+    clang-cl -v; \
+    lld-link --version; \
+    # Use clang instead of gcc when compiling and linking binaries targeting the host (eg proc macros, build files)
+    update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100; \
+    update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100; \
+    update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \
+    rustup default nightly; \
+    rustup target add x86_64-pc-windows-msvc; \
+    rustup component add rust-src; \
+    # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
+    tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \
+    # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
+    xwin --accept-license splat --output /xwin; \
+    # Remove unneeded files to reduce image size
+    apt-get remove -y --auto-remove; \
+    rm -rf \
+        .xwin-cache \
+        /usr/local/cargo/bin/xwin \
+        /root/$XWIN_PREFIX.tar.gz \
+        /var/lib/apt/lists/* \
+        /root/*.key;
+
+# Note that we're using the full target triple for each variable instead of the
+# simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for
+# build dependencies that need to compile and execute in the host environment
+ENV CC_x86_64_pc_windows_msvc="clang-cl" \
+    CXX_x86_64_pc_windows_msvc="clang-cl" \
+    AR_x86_64_pc_windows_msvc="llvm-lib" \
+    # wine can be quite spammy with log messages and they're generally uninteresting
+    WINEDEBUG="-all" \
+    # Use wine to run test executables
+    CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUNNER="wine" \
+    # Note that we only disable unused-command-line-argument here since clang-cl
+    # doesn't implement all of the options supported by cl, but the ones it doesn't
+    # are _generally_ not interesting.
+    CL_FLAGS="-Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc/xwin/crt/include /imsvc/xwin/sdk/include/ucrt /imsvc/xwin/sdk/include/um /imsvc/xwin/sdk/include/shared" \
+    # Let cargo know what linker to invoke if you haven't already specified it
+    # in a .cargo/config.toml file
+    CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER="lld-link" \
+    CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS="-Lnative=/xwin/crt/lib/x86_64 -Lnative=/xwin/sdk/lib/um/x86_64 -Lnative=/xwin/sdk/lib/ucrt/x86_64"
+
+# These are separate since docker/podman won't transform environment variables defined in the same ENV block
+ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \
+    CXXFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS"
+
+# Run wineboot just to setup the default WINEPREFIX so we don't do it every
+# container run
+RUN wine wineboot --init
+
+WORKDIR /src/dtmt
+
+COPY *.lib /src

.ci/pipelines/check.yml

@@ -0,0 +1,28 @@
+---
+
+# The actual CI pipeline that is run per branch
+
+resources:
+- name: repo
+  type: git
+  source:
+    uri: https://git.sclu1034.dev/bitsquid_dt/dtmt
+    branch: ((branch))
+
+jobs:
+- name: build-msvc
+  plan:
+  - get: repo
+    trigger: true
+  - task: build
+    file: repo/.ci/tasks/build.yml
+    vars:
+      target: msvc
+- name: build-linux
+  plan:
+  - get: repo
+    trigger: true
+  - task: build
+    file: repo/.ci/tasks/build.yml
+    vars:
+      target: linux

.ci/tasks/build.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -eux
+
+OUTPUT="$1"
+
+case "$TARGET" in
+    msvc)
+        cp /src/*.lib ./lib/oodle/
+        cargo build --color always --locked --release --target x86_64-pc-windows-msvc -Zbuild-std
+
+        if [ -d "$OUTPUT" ]; then
+            install -t "$OUTPUT/" target/x86_64-pc-windows-msvc/release/dtmt.exe
+            install -t "$OUTPUT/" target/x86_64-pc-windows-msvc/release/dtmm.exe
+        fi
+
+        ;;
+    linux)
+        cp /src/*.so ./lib/oodle/
+        cargo build --color always --locked --profile release-lto
+
+        if [ -d "$OUTPUT" ]; then
+            install -t "$OUTPUT/" target/release/dtmt
+            install -t "$OUTPUT/" target/release/dtmm
+        fi
+
+        ;;
+    *)
+        set +x
+        echo "Env var 'TARGET' must either be 'msvc' or 'linux'. Got '$TARGET'." >&2
+        exit 1
+esac

.ci/tasks/build.yml

@@ -0,0 +1,24 @@
+---
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    repository: registry.local:5000/dtmt-ci-base-((target))
+    tag: latest
+
+inputs:
+- name: repo
+
+outputs:
+- name: artifact
+
+params:
+  CI: true
+  TARGET: ((target))
+
+run:
+  path: .ci/tasks/build.sh
+  dir: repo
+  args:
+    - /artifact

.fdignore

@@ -1,3 +1,5 @@
 /target
-liboo2corelinux64.so
-oo2core_8_win64.dll
+*.a
+*.so
+*.dll
+*.lib

.gitignore

@@ -1,6 +1,7 @@
 /target
 /data
 .envrc
+*.a
 *.so
 *.dll
 *.lib

.gitmodules

@@ -1,9 +1,12 @@
 [submodule "lib/serde_sjson"]
 	path = lib/serde_sjson
-	url = git@git.sclu1034.dev:lucas/serde_sjson.git
+	url = https://git.sclu1034.dev/lucas/serde_sjson.git
 [submodule "lib/steamlocate-rs"]
 	path = lib/steamlocate-rs
-	url = git@github.com:sclu1034/steamlocate-rs.git
+	url = https://github.com/sclu1034/steamlocate-rs.git
 [submodule "crates/dtmm/assets/icons"]
 	path = crates/dtmm/assets/icons
 	url = https://github.com/tabler/tabler-icons
+[submodule "lib/luajit2-sys"]
+	path = lib/luajit2-sys
+	url = https://github.com/sclu1034/luajit2-sys.git

Cargo.lock

@@ -1858,9 +1858,8 @@ dependencies = [
 [[package]]
 name = "luajit2-sys"
 version = "0.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33bb7acccd5a0224645ba06eba391af5f7194ff1762c2545860b43afcfd41af2"
 dependencies = [
+ "bindgen",
  "cc",
  "fs_extra",
  "libc",

Cargo.toml

@@ -12,4 +12,10 @@ members = [
 
 [profile.release]
 strip = "debuginfo"
+
+# The MSVC toolchain cannot handle LTO properly. Some symbol related to
+# panic unwind would always be missing.
+# So we use a separate profile for when we can compile with LTO.
+[profile.release-lto]
+inherits = "release"
 lto = true

Justfile

@@ -0,0 +1,11 @@
+ci-image: ci-image-msvc ci-image-linux
+
+ci-image-msvc:
+    docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image
+    docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc
+    docker push registry.sclu1034.dev/dtmt-ci-base-msvc
+
+ci-image-linux:
+    docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image
+    docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
+    docker push registry.sclu1034.dev/dtmt-ci-base-linux

lib/luajit2-sys

@@ -0,0 +1 @@
+Subproject commit 18797c4d2a53834210fd096dd39195ce7f2bce21

lib/sdk/Cargo.toml

@@ -22,6 +22,6 @@ tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "m
 tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] }
 tracing = { version = "0.1.37", features = ["async-await"] }
 tracing-error = "0.2.0"
-luajit2-sys = "0.0.2"
+luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" }
 async-recursion = "1.0.2"
 path-slash = "0.2.1"