bitsquid_dt/dtmt
bitsquid_dt/dtmt
2
Fork 0
Code Issues 34 Pull requests 11 Projects 1 Releases 3 Packages 1 Wiki Activity
Var Dump: 
dumpVar: only available in dev mode
Mailing List

Merge branch 'master' into feat/nexus-file-version
All checks were successful
lint/clippy Checking for common mistakes and opportunities for code improvement
build/msvc Build for the target platform: msvc
build/linux Build for the target platform: linux

Browse source 
* master:
  dtmm: Fix writing Nexus image to disk
  dtmm: Fix importing from `.mod` file
  ci: Combine Dockerfiles into multi-stage build
  ci: Rework MSVC image building
  ci: Fix base pipeline
  ci: Improve caching setup for image building
  ci: Create build artifacts for commits on master
  ci: Fix undefined variable
  Add changelog entry
  dtmm: Fetch mod image from Nexus
This commit is contained in:
Lucas Schwiderski 2023-11-29 15:42:00 +01:00
commit 5bcc4b8c70
Signed by: lucas
GPG key ID: AA12679AAA6DF4D8
11 changed files with 345 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
.ci/Dockerfile.linux
View file

@ -1,35 +1,7 @@
FROM dtmt-ci-base-linux
# Create dummy crates and copy their Cargo.toml, so that dependencies can be cached
RUN set -e; \
cargo new --bin crates/dtmt; \
cargo new --bin crates/dtmm; \
cargo new --lib lib/dtmt-shared; \
cargo new --lib lib/nexusmods; \
cargo new --lib lib/sdk; \
cargo new --lib lib/serde_sjson; \
cargo new --lib lib/ansi-parser
COPY Cargo.toml Cargo.lock /src/dtmt/
COPY crates/dtmt/Cargo.toml /src/dtmt/crates/dtmt/
COPY crates/dtmm/Cargo.toml /src/dtmt/crates/dtmm/
COPY lib/dtmt-shared/Cargo.toml /src/dtmt/lib/dtmt-shared/
COPY lib/nexusmods/Cargo.toml /src/dtmt/lib/nexusmods/
COPY lib/sdk/Cargo.toml /src/dtmt/lib/sdk/
COPY lib/serde_sjson/Cargo.toml /src/dtmt/lib/serde_sjson/
COPY lib/ansi-parser/Cargo.toml /src/dtmt/lib/ansi-parser/
# Crates with build scripts cannot be split that way, but they shouldn't change too often
COPY lib/luajit2-sys /src/dtmt/lib/luajit2-sys
COPY lib/oodle /src/dtmt/lib/oodle
# color-eyre needs to be copied, too, then, as it's used by `oodle`
COPY lib/color-eyre /src/dtmt/lib/color-eyre
COPY --from=dtmt-ci-base-linux /src/*.lib /src/dtmt/lib/oodle/
RUN cargo build --release --locked
RUN rm -r crates lib
COPY . /src/dtmt
COPY --from=dtmt-ci-base-linux /src/*.lib /src/*.so /src/dtmt/lib/oodle/
RUN cargo build --release --locked
RUN --mount=type=cache,id=cargo-registry,target=/cargo/registry \
--mount=type=cache,id=cargo-target,target=/src/dtmt/target \
cargo build --release --locked

79
.ci/image/Dockerfile.msvc → .ci/image/Dockerfile
View file

@ -1,13 +1,68 @@
# https://jake-shadle.github.io/xwin/
FROM dtmt-ci-base-linux
FROM debian:bullseye-slim as xwin
ARG XWIN_VERSION=0.5.0
ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
RUN set -eux; \
apt-get update; \
apt-get install --no-install-recommends -y \
tar \
; \
# Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/bin --strip-components=1 $XWIN_PREFIX/xwin; \
apt-get remove -y --auto-remove; \
rm -rf \
/var/lib/apt/lists/* \
/root/$XWIN_PREFIX.tar.gz;
RUN set -eux; \
# Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
xwin \
--log-level debug \
--cache-dir /root/.xwin-cache \
--manifest-version 16 \
--accept-license \
splat \
--output /xwin; \
# Even though this build step only exists temporary, to copy the
# final data out of, it still generates a cache entry on the Docker host.
# And to keep that to a minimum, we still delete the stuff we don't need.
rm -rf /root/.xwin-cache;
FROM rust:slim-bullseye as linux
RUN set -eux; \
apt-get update; \
apt-get install --no-install-recommends -y \
build-essential \
curl \
git \
gpg \
jq \
libatk1.0-dev \
libclang-13-dev \
libglib2.0-dev \
libgtk-3-dev \
libpango1.0-dev \
libssl-dev \
libzstd-dev \
pkg-config; \
apt-get remove -y --auto-remove; \
rm -rf /var/lib/apt/lists/*; \
rustup default nightly
WORKDIR /src/dtmt
COPY lib/oodle/*.so lib/oodle/*.a /src/
FROM linux as msvc
ENV KEYRINGS /usr/local/share/keyrings
ARG XWIN_VERSION=0.2.11
ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl"
ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key
ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key
ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz
RUN set -eux; \
mkdir -p $KEYRINGS; \
@ -26,7 +81,7 @@ RUN set -eux; \
llvm-13 \
lld-13 \
winehq-staging \
tar; \
; \
# ensure that clang/clang++ are callable directly
ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \
# We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages
@ -44,19 +99,15 @@ RUN set -eux; \
update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \
rustup target add x86_64-pc-windows-msvc; \
rustup component add rust-src; \
# Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`.
tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \
# Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively
xwin --accept-license splat --output /xwin; \
# Remove unneeded files to reduce image size
apt-get remove -y --auto-remove; \
rm -rf \
.xwin-cache \
/usr/local/cargo/bin/xwin \
/root/$XWIN_PREFIX.tar.gz \
/var/lib/apt/lists/* \
/root/*.key;
COPY lib/oodle/*.lib /src
COPY --from=xwin /xwin /xwin
# Note that we're using the full target triple for each variable instead of the
# simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for
# build dependencies that need to compile and execute in the host environment
@ -83,7 +134,3 @@ ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \
# Run wineboot just to setup the default WINEPREFIX so we don't do it every
# container run
RUN wine wineboot --init
WORKDIR /src/dtmt
COPY *.lib /src

25
.ci/image/Dockerfile.linux
View file

@ -1,25 +0,0 @@
FROM rust:slim-bullseye
RUN set -eux; \
apt-get update; \
apt-get install --no-install-recommends -y \
build-essential \
curl \
git \
gpg \
jq \
libatk1.0-dev \
libclang-13-dev \
libglib2.0-dev \
libgtk-3-dev \
libpango1.0-dev \
libssl-dev \
libzstd-dev \
pkg-config; \
apt-get remove -y --auto-remove; \
rm -rf /var/lib/apt/lists/*; \
rustup default nightly
WORKDIR /src/dtmt
COPY *.so *.a /src/

43
.ci/pipelines/base-pipeline.yml
View file

@ -1,43 +0,0 @@
---
# The base pipeline that runs continuously, checks for branches and
# creates a new pipeline instance for each of them.
resource_types:
- name: gitea-pr
type: registry-image
source:
repository: registry.local:5000/gitea-pr
resources:
- name: repo-pr
type: gitea-pr
source:
access_token: ((gitea_api_key))
owner: ((owner))
repo: ((repo))
url: https://git.sclu1034.dev
- name: repo
type: git
source:
uri: https://git.sclu1034.dev/bitsquid_dt/dtmt
jobs:
- name: set-pipelines
plan:
- in_parallel:
- get: repo-pr
trigger: true
- get: repo
- load_var: prs
file: repo-pr/prs.json
- across:
- var: pr
values: ((.:prs))
set_pipeline: dtmt-pr
file: repo/.ci/pipelines/pr.yml
vars:
pr: ((.:pr))
gitea_api_key: ((gitea_api_key))
instance_vars:
number: ((.:pr.number))

206
.ci/pipelines/base.yml Normal file
View file

@ -0,0 +1,206 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/cappyzawa/concourse-pipeline-jsonschema/master/concourse_jsonschema.json#/definitions/Config
---
# The actual CI pipeline that is run per branch
resource_types:
- name: gitea-package
type: registry-image
source:
repository: registry.local:5000/gitea-package
- name: gitea-status
type: registry-image
source:
repository: registry.local:5000/gitea-status
- name: gitea-pr
type: registry-image
source:
repository: registry.local:5000/gitea-pr
resources:
- name: repo
type: git
source:
uri: http://forgejo:3000/bitsquid_dt/dtmt
branch: master
- name: repo-pr
type: gitea-pr
source:
access_token: ((gitea_api_key))
owner: ((owner))
repo: ((repo))
url: https://git.sclu1034.dev
- name: gitea-package
type: gitea-package
source:
access_token: ((gitea_api_key))
url: http://forgejo:3000
owner: bitsquid_dt
type: generic
name: dtmt
- name: status-build-msvc
type: gitea-status
source:
access_token: ((gitea_api_key))
url: http://forgejo:3000
owner: bitsquid_dt
repo: dtmt
context: build/msvc
description: "Build for the target platform: msvc"
- name: status-build-linux
type: gitea-status
source:
access_token: ((gitea_api_key))
url: http://forgejo:3000
owner: bitsquid_dt
repo: dtmt
context: build/linux
description: "Build for the target platform: linux"
jobs:
- name: set-pipelines
plan:
- in_parallel:
- get: repo-pr
trigger: true
- get: repo
- load_var: prs
file: repo-pr/prs.json
- across:
- var: pr
values: ((.:prs))
set_pipeline: dtmt-pr
file: repo/.ci/pipelines/pr.yml
vars:
pr: ((.:pr))
gitea_api_key: ((gitea_api_key))
instance_vars:
number: ((.:pr.number))
- name: build-msvc
on_success:
put: state-success
resource: status-build-msvc
no_get: true
params:
state: success
sha: ((.:git_sha))
on_failure:
put: state-failure
resource: status-build-msvc
no_get: true
params:
state: failure
sha: ((.:git_sha))
plan:
- get: repo
trigger: true
- load_var: git_sha
file: repo/.git/ref
- put: state-pending
resource: status-build-msvc
no_get: true
params:
state: pending
sha: ((.:git_sha))
- task: build
file: repo/.ci/tasks/build.yml
vars:
pr: ""
target: msvc
gitea_url: http://forgejo:3000
gitea_api_key: ((gitea_api_key))
- load_var: version_number
reveal: true
file: artifact/version
- put: package
resource: gitea-package
no_get: true
inputs:
- artifact
params:
version: ((.:version_number))
fail_fast: true
override: true
globs:
- artifact/dtmt
- artifact/dtmm
- artifact/*.exe
- artifact/*.sha256
- name: build-linux
on_success:
put: state-success
resource: status-build-linux
no_get: true
params:
state: success
sha: ((.:git_sha))
on_failure:
put: state-failure
resource: status-build-linux
no_get: true
params:
state: failure
sha: ((.:git_sha))
plan:
- get: repo
trigger: true
- load_var: git_sha
file: repo/.git/ref
- put: state-pending
resource: status-build-linux
no_get: true
params:
state: pending
sha: ((.:git_sha))
- task: build
file: repo/.ci/tasks/build.yml
vars:
pr: ""
target: linux
gitea_url: http://forgejo:3000
gitea_api_key: ((gitea_api_key))
- load_var: version_number
reveal: true
file: artifact/version
- put: package
resource: gitea-package
no_get: true
inputs:
- artifact
params:
version: ((.:version_number))
fail_fast: true
override: true
globs:
- artifact/dtmt
- artifact/dtmm
- artifact/*.exe
- artifact/*.sha256

4
.ci/tasks/build.sh
View file

@ -19,7 +19,9 @@ install_artifact() {
cd "repo"
if [ -n "${PR:-}" ]; then
PR=${PR:-}
if [ -n "$PR" ]; then
title "PR: $(echo "$PR" | jq '.number') - $(echo "$PR" | jq '.title')"
ref="pr-$(echo "$PR" | jq '.number')-$(git rev-parse --short "$(cat .git/ref || echo "HEAD")" 2>/dev/null || echo 'manual')"
else

1
CHANGELOG.adoc
View file

@ -16,6 +16,7 @@
- dtmt: add utility to migrate mod projects
- dtmm: reset dtkit-patch installations
- sdk: implement decompiling Lua files
- dtmm: fetch cover image for Nexus mods
- dtmm: fetch file version for Nexus mods
=== Fixed

19
Justfile
View file

@ -16,22 +16,21 @@ build-image-msvc:
build-image-linux:
docker build -f .ci/Dockerfile.linux .
ci-image: ci-image-msvc ci-image-linux
ci-image-msvc: ci-image-linux
docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image
ci-image:
# The MSVC image depends on the Linux image. So by building that first,
# we actually build both, and cache them, so that "building" the
# Linux image afterwards merely needs to pull the cache.
docker build --target msvc -t dtmt-ci-base-msvc -f .ci/image/Dockerfile .
docker build --target linux -t dtmt-ci-base-linux -f .ci/image/Dockerfile .
docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc
docker push registry.sclu1034.dev/dtmt-ci-base-msvc
ci-image-linux:
docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image
docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux
docker push registry.sclu1034.dev/dtmt-ci-base-msvc
docker push registry.sclu1034.dev/dtmt-ci-base-linux
set-base-pipeline:
fly -t {{fly_target}} set-pipeline \
--pipeline dtmt-prs \
--config .ci/pipelines/base-pipeline.yml \
--pipeline dtmt \
--config .ci/pipelines/base.yml \
-v gitea_api_key=${GITEA_API_KEY} \
-v owner=bitsquid_dt \
-v repo=dtmt

69
crates/dtmm/src/controller/import.rs
View file

@ -27,6 +27,16 @@ fn find_archive_file<R: Read + Seek>(
path
}
fn image_data_to_buffer(data: impl AsRef<[u8]>) -> Result<ImageBuf> {
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
ImageBuf::from_data(data.as_ref()).map_err(|err| {
Report::msg(err.to_string())
.wrap_err("Invalid image data")
.suggestion("Supported formats are: PNG, JPEG, Bitmap and WebP")
})
}
// Runs the content of a `.mod` file to extract what data we can get
// from legacy mods.
// 1. Create a global function `new_mod` that stores
@ -438,6 +448,8 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
None
};
tracing::trace!(?nexus);
let mut archive = ZipArchive::new(data).wrap_err("Failed to open ZIP archive")?;
if tracing::enabled!(tracing::Level::DEBUG) {
@ -452,7 +464,13 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
let (mut mod_cfg, root) =
extract_mod_config(&mut archive).wrap_err("Failed to extract mod configuration")?;
tracing::info!("Importing mod {} ({})", mod_cfg.name, mod_cfg.id);
tracing::debug!(root, ?mod_cfg);
let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
let dest = mod_dir.join(&mod_cfg.id);
tracing::trace!("Creating mods directory {}", dest.display());
fs::create_dir_all(&dest)
.await
.wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
let image = if let Some(path) = &mod_cfg.image {
let name = archive
@ -468,33 +486,40 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result<Mod
f.read_to_end(&mut buf)
.wrap_err("Failed to read file index from archive")?;
// Druid somehow doesn't return an error compatible with eyre, here.
// So we have to wrap through `Display` manually.
let img = match ImageBuf::from_data(&buf) {
Ok(img) => img,
Err(err) => {
let err = Report::msg(err.to_string())
.wrap_err("Invalid image data")
.note("Supported formats are: PNG, JPEG, Bitmap and WebP")
.suggestion("Contact the mod author to fix this");
return Err(err);
}
};
let img = image_data_to_buffer(buf)?;
Some(img)
} else if let Some((nexus, _)) = &nexus {
let api = NexusApi::new(state.nexus_api_key.to_string())?;
let url = nexus.picture_url.as_ref();
let data = api
.picture(url)
.await
.wrap_err_with(|| format!("Failed to download Nexus image from '{}'", url))?;
let img = image_data_to_buffer(&data)?;
let name = "image.bin";
let path = dest.join(name);
match fs::write(&path, &data).await {
Ok(_) => {
mod_cfg.image = Some(name.into());
Some(img)
}
Err(err) => {
let err = Report::new(err).wrap_err(format!(
"Failed to write Nexus picture to file '{}'",
path.display()
));
tracing::error!("{:?}", err);
None
}
}
} else {
None
};
tracing::trace!(?image);
let mod_dir = state.data_dir.join(state.mod_dir.as_ref());
let dest = mod_dir.join(&mod_cfg.id);
tracing::trace!("Creating mods directory {}", dest.display());
fs::create_dir_all(&dest)
.await
.wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?;
tracing::debug!(root, ?mod_cfg);
let packages = if mod_cfg.bundled {
extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")?

2
crates/dtmm/src/state/data.rs
View file

@ -78,6 +78,7 @@ pub(crate) struct NexusInfo {
pub author: String,
pub summary: Arc<String>,
pub description: Arc<String>,
pub picture_url: Arc<String>,
}
impl From<NexusMod> for NexusInfo {
@ -89,6 +90,7 @@ impl From<NexusMod> for NexusInfo {
author: value.author,
summary: Arc::new(value.summary),
description: Arc::new(value.description),
picture_url: Arc::new(value.picture_url.into()),
}
}
}

12
lib/nexusmods/src/lib.rs
View file

@ -4,7 +4,7 @@ use std::convert::Infallible;
use lazy_static::lazy_static;
use regex::Regex;
use reqwest::header::{HeaderMap, HeaderValue, InvalidHeaderValue};
use reqwest::{Client, RequestBuilder, Url};
use reqwest::{Client, IntoUrl, RequestBuilder, Url};
use serde::Deserialize;
use thiserror::Error;
@ -126,6 +126,16 @@ impl Api {
Ok(file.version)
}
#[tracing::instrument(skip(self))]
pub async fn picture(&self, url: impl IntoUrl + std::fmt::Debug) -> Result<Vec<u8>> {
let res = self.client.get(url).send().await?.error_for_status()?;
res.bytes()
.await
.map(|bytes| bytes.to_vec())
.map_err(From::from)
}
pub fn parse_file_name<S: AsRef<str>>(
name: S,
) -> Option<(String, u64, String, OffsetDateTime)> {