From e2a285a739b2f487b397a19b63b1b81662f08b46 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 23 Mar 2023 13:42:03 +0100 Subject: [PATCH 1/5] feat: Add Docker images for CI compilation --- .ci/image/Dockerfile.linux | 22 ++++++++ .ci/image/Dockerfile.msvc | 103 +++++++++++++++++++++++++++++++++++++ .fdignore | 6 ++- .gitignore | 1 + .gitmodules | 3 ++ Cargo.lock | 3 +- Justfile | 11 ++++ lib/luajit2-sys | 1 + lib/sdk/Cargo.toml | 2 +- 9 files changed, 147 insertions(+), 5 deletions(-) create mode 100644 .ci/image/Dockerfile.linux create mode 100644 .ci/image/Dockerfile.msvc create mode 100644 Justfile create mode 160000 lib/luajit2-sys diff --git a/.ci/image/Dockerfile.linux b/.ci/image/Dockerfile.linux new file mode 100644 index 0000000..523d0ea --- /dev/null +++ b/.ci/image/Dockerfile.linux @@ -0,0 +1,22 @@ +FROM rust:slim-bullseye + +RUN set -eux; \ + apt-get update; \ + apt-get install --no-install-recommends -y \ + build-essential \ + libatk1.0-dev \ + libclang-13-dev \ + libglib2.0-dev \ + libgtk-3-dev \ + libpango1.0-dev \ + libssl-dev \ + libzstd-dev \ + pkg-config; \ + apt-get remove -y --auto-remove; \ + rm -rf /var/lib/apt/lists/*; \ + rustup default nightly + +WORKDIR /src/dtmt + +COPY *.so /src + diff --git a/.ci/image/Dockerfile.msvc b/.ci/image/Dockerfile.msvc new file mode 100644 index 0000000..4c09735 --- /dev/null +++ b/.ci/image/Dockerfile.msvc @@ -0,0 +1,103 @@ +# https://jake-shadle.github.io/xwin/ +FROM rust:slim-bullseye + +ENV KEYRINGS /usr/local/share/keyrings +ARG XWIN_VERSION=0.2.11 +ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl" + +ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key +ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key +ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz + +RUN set -eux; \ + mkdir -p $KEYRINGS; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + gpg \ + curl; \ + # clang/lld/llvm + gpg --dearmor > $KEYRINGS/llvm.gpg < /root/llvm-snapshot.gpg.key; \ + # wine + gpg --dearmor > $KEYRINGS/winehq.gpg < /root/winehq.key; \ + echo "deb [signed-by=$KEYRINGS/llvm.gpg] http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-13 main" > /etc/apt/sources.list.d/llvm.list; \ + echo "deb [signed-by=$KEYRINGS/winehq.gpg] https://dl.winehq.org/wine-builds/debian/ bullseye main" > /etc/apt/sources.list.d/winehq.list; \ + dpkg --add-architecture i386; \ + apt-get update; \ + apt-get install --no-install-recommends -y \ + build-essential \ + libatk1.0-dev \ + libclang-13-dev \ + libglib2.0-dev \ + libgtk-3-dev \ + libpango1.0-dev \ + libssl-dev \ + libzstd-dev \ + pkg-config \ + gcc-mingw-w64-x86-64 \ + # g++-mingw-w64-x86-64 \ + clang-13 \ + llvm-13 \ + lld-13 \ + winehq-staging \ + tar; \ + # ensure that clang/clang++ are callable directly + ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \ + # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages + ln -s clang-13 /usr/bin/clang-cl && ln -s llvm-ar-13 /usr/bin/llvm-lib && ln -s lld-link-13 /usr/bin/lld-link; \ + # Verify the symlinks are correct + clang++ -v; \ + ld.lld -v; \ + # Doesn't have an actual -v/--version flag, but it still exits with 0 + llvm-lib -v; \ + clang-cl -v; \ + lld-link --version; \ + # Use clang instead of gcc when compiling and linking binaries targeting the host (eg proc macros, build files) + update-alternatives --install /usr/bin/cc cc /usr/bin/clang 100; \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 100; \ + update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \ + rustup default nightly; \ + rustup target add x86_64-pc-windows-msvc; \ + rustup component add rust-src; \ + # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`. + tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \ + # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively + xwin --accept-license splat --output /xwin; \ + # Remove unneeded files to reduce image size + apt-get remove -y --auto-remove; \ + rm -rf \ + .xwin-cache \ + /usr/local/cargo/bin/xwin \ + /root/$XWIN_PREFIX.tar.gz \ + /var/lib/apt/lists/* \ + /root/*.key; + +# Note that we're using the full target triple for each variable instead of the +# simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for +# build dependencies that need to compile and execute in the host environment +ENV CC_x86_64_pc_windows_msvc="clang-cl" \ + CXX_x86_64_pc_windows_msvc="clang-cl" \ + AR_x86_64_pc_windows_msvc="llvm-lib" \ + # wine can be quite spammy with log messages and they're generally uninteresting + WINEDEBUG="-all" \ + # Use wine to run test executables + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUNNER="wine" \ + # Note that we only disable unused-command-line-argument here since clang-cl + # doesn't implement all of the options supported by cl, but the ones it doesn't + # are _generally_ not interesting. + CL_FLAGS="-Wno-unused-command-line-argument -fuse-ld=lld-link /imsvc/xwin/crt/include /imsvc/xwin/sdk/include/ucrt /imsvc/xwin/sdk/include/um /imsvc/xwin/sdk/include/shared" \ + # Let cargo know what linker to invoke if you haven't already specified it + # in a .cargo/config.toml file + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER="lld-link" \ + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS="-Lnative=/xwin/crt/lib/x86_64 -Lnative=/xwin/sdk/lib/um/x86_64 -Lnative=/xwin/sdk/lib/ucrt/x86_64" + +# These are separate since docker/podman won't transform environment variables defined in the same ENV block +ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \ + CXXFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" + +# Run wineboot just to setup the default WINEPREFIX so we don't do it every +# container run +RUN wine wineboot --init + +WORKDIR /src/dtmt + +COPY *.lib /src diff --git a/.fdignore b/.fdignore index cbb939d..05d32f8 100644 --- a/.fdignore +++ b/.fdignore @@ -1,3 +1,5 @@ /target -liboo2corelinux64.so -oo2core_8_win64.dll +*.a +*.so +*.dll +*.lib diff --git a/.gitignore b/.gitignore index 4a1b934..7f254dd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /target /data .envrc +*.a *.so *.dll *.lib diff --git a/.gitmodules b/.gitmodules index 1c9c3c2..7a31ce5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "crates/dtmm/assets/icons"] path = crates/dtmm/assets/icons url = https://github.com/tabler/tabler-icons +[submodule "lib/luajit2-sys"] + path = lib/luajit2-sys + url = git@github.com:sclu1034/luajit2-sys.git diff --git a/Cargo.lock b/Cargo.lock index 9cb3b5e..98ee3be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1858,9 +1858,8 @@ dependencies = [ [[package]] name = "luajit2-sys" version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33bb7acccd5a0224645ba06eba391af5f7194ff1762c2545860b43afcfd41af2" dependencies = [ + "bindgen", "cc", "fs_extra", "libc", diff --git a/Justfile b/Justfile new file mode 100644 index 0000000..008dce2 --- /dev/null +++ b/Justfile @@ -0,0 +1,11 @@ +ci-image: ci-image-msvc ci-image-linux + +ci-image-msvc: + docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image + docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc + docker push registry.sclu1034.dev/dtmt-ci-base-msvc + +ci-image-linux: + docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image + docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux + docker push registry.sclu1034.dev/dtmt-ci-base-linux diff --git a/lib/luajit2-sys b/lib/luajit2-sys new file mode 160000 index 0000000..9ab05ce --- /dev/null +++ b/lib/luajit2-sys @@ -0,0 +1 @@ +Subproject commit 9ab05ce18cc35b3235b0cc7665d6e860193c9b11 diff --git a/lib/sdk/Cargo.toml b/lib/sdk/Cargo.toml index e834bfc..63f789b 100644 --- a/lib/sdk/Cargo.toml +++ b/lib/sdk/Cargo.toml @@ -22,6 +22,6 @@ tokio = { version = "1.21.2", features = ["rt-multi-thread", "fs", "process", "m tokio-stream = { version = "0.1.11", features = ["fs", "io-util"] } tracing = { version = "0.1.37", features = ["async-await"] } tracing-error = "0.2.0" -luajit2-sys = "0.0.2" +luajit2-sys = { path = "../../lib/luajit2-sys", version = "*" } async-recursion = "1.0.2" path-slash = "0.2.1" -- 2.45.3 From a0166f07ff9ba1a986761682784d0dd84b818f75 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 23 Mar 2023 15:32:33 +0100 Subject: [PATCH 2/5] fix(ci): Fix linking on Linux --- Cargo.toml | 6 ++++++ lib/luajit2-sys | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 56d5fbe..2f47e65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,4 +12,10 @@ members = [ [profile.release] strip = "debuginfo" + +# The MSVC toolchain cannot handle LTO properly. Some symbol related to +# panic unwind would always be missing. +# So we use a separate profile for when we can compile with LTO. +[profile.release-lto] +inherits = "release" lto = true diff --git a/lib/luajit2-sys b/lib/luajit2-sys index 9ab05ce..18797c4 160000 --- a/lib/luajit2-sys +++ b/lib/luajit2-sys @@ -1 +1 @@ -Subproject commit 9ab05ce18cc35b3235b0cc7665d6e860193c9b11 +Subproject commit 18797c4d2a53834210fd096dd39195ce7f2bce21 -- 2.45.3 From 25a11bb79e026c68e8235acd2ed744d574b93001 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 23 Mar 2023 16:49:56 +0100 Subject: [PATCH 3/5] feat(ci): Use HTTPS URLs for submodules This saves setting up SSH access for the CI pipeline. --- .gitmodules | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7a31ce5..fc31e70 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,12 @@ [submodule "lib/serde_sjson"] path = lib/serde_sjson - url = git@git.sclu1034.dev:lucas/serde_sjson.git + url = https://git.sclu1034.dev/lucas/serde_sjson.git [submodule "lib/steamlocate-rs"] path = lib/steamlocate-rs - url = git@github.com:sclu1034/steamlocate-rs.git + url = https://github.com/sclu1034/steamlocate-rs.git [submodule "crates/dtmm/assets/icons"] path = crates/dtmm/assets/icons url = https://github.com/tabler/tabler-icons [submodule "lib/luajit2-sys"] path = lib/luajit2-sys - url = git@github.com:sclu1034/luajit2-sys.git + url = https://github.com/sclu1034/luajit2-sys.git -- 2.45.3 From 8b70eba79e3dd950c1f7d90cd791baffcf4266d2 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 23 Mar 2023 18:01:31 +0100 Subject: [PATCH 4/5] feat(ci): Add tasks to build in CI --- .ci/tasks/build.sh | 32 ++++++++++++++++++++++++++++++++ .ci/tasks/build.yml | 24 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 .ci/tasks/build.sh create mode 100644 .ci/tasks/build.yml diff --git a/.ci/tasks/build.sh b/.ci/tasks/build.sh new file mode 100755 index 0000000..384c0a8 --- /dev/null +++ b/.ci/tasks/build.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +set -eux + +OUTPUT="$1" + +case "$TARGET" in + msvc) + cp /src/*.lib ./lib/oodle/ + cargo build --color always --locked --release --target x86_64-pc-windows-msvc -Zbuild-std + + if [ -d "$OUTPUT" ]; then + install -t "$OUTPUT/" target/x86_64-pc-windows-msvc/release/dtmt.exe + install -t "$OUTPUT/" target/x86_64-pc-windows-msvc/release/dtmm.exe + fi + + ;; + linux) + cp /src/*.so ./lib/oodle/ + cargo build --color always --locked --profile release-lto + + if [ -d "$OUTPUT" ]; then + install -t "$OUTPUT/" target/release/dtmt + install -t "$OUTPUT/" target/release/dtmm + fi + + ;; + *) + set +x + echo "Env var 'TARGET' must either be 'msvc' or 'linux'. Got '$TARGET'." >&2 + exit 1 +esac diff --git a/.ci/tasks/build.yml b/.ci/tasks/build.yml new file mode 100644 index 0000000..935f81b --- /dev/null +++ b/.ci/tasks/build.yml @@ -0,0 +1,24 @@ +--- +platform: linux + +image_resource: + type: registry-image + source: + repository: registry.local:5000/dtmt-ci-base-((target)) + tag: latest + +inputs: +- name: repo + +outputs: +- name: artifact + +params: + CI: true + TARGET: ((target)) + +run: + path: .ci/tasks/build.sh + dir: repo + args: + - /artifact -- 2.45.3 From 4929b3f201ba3aa78e2fe3a9b596bf7ddee0d059 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 23 Mar 2023 19:17:07 +0100 Subject: [PATCH 5/5] feat(ci): Add basic pipeline --- .ci/pipelines/check.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .ci/pipelines/check.yml diff --git a/.ci/pipelines/check.yml b/.ci/pipelines/check.yml new file mode 100644 index 0000000..4d350ac --- /dev/null +++ b/.ci/pipelines/check.yml @@ -0,0 +1,28 @@ +--- + +# The actual CI pipeline that is run per branch + +resources: +- name: repo + type: git + source: + uri: https://git.sclu1034.dev/bitsquid_dt/dtmt + branch: ((branch)) + +jobs: +- name: build-msvc + plan: + - get: repo + trigger: true + - task: build + file: repo/.ci/tasks/build.yml + vars: + target: msvc +- name: build-linux + plan: + - get: repo + trigger: true + - task: build + file: repo/.ci/tasks/build.yml + vars: + target: linux -- 2.45.3