diff --git a/.ci/Dockerfile.linux b/.ci/Dockerfile.linux index b46f7ff..9e93e44 100644 --- a/.ci/Dockerfile.linux +++ b/.ci/Dockerfile.linux @@ -1,7 +1,35 @@ FROM dtmt-ci-base-linux +# Create dummy crates and copy their Cargo.toml, so that dependencies can be cached +RUN set -e; \ + cargo new --bin crates/dtmt; \ + cargo new --bin crates/dtmm; \ + cargo new --lib lib/dtmt-shared; \ + cargo new --lib lib/nexusmods; \ + cargo new --lib lib/sdk; \ + cargo new --lib lib/serde_sjson; \ + cargo new --lib lib/ansi-parser + +COPY Cargo.toml Cargo.lock /src/dtmt/ +COPY crates/dtmt/Cargo.toml /src/dtmt/crates/dtmt/ +COPY crates/dtmm/Cargo.toml /src/dtmt/crates/dtmm/ +COPY lib/dtmt-shared/Cargo.toml /src/dtmt/lib/dtmt-shared/ +COPY lib/nexusmods/Cargo.toml /src/dtmt/lib/nexusmods/ +COPY lib/sdk/Cargo.toml /src/dtmt/lib/sdk/ +COPY lib/serde_sjson/Cargo.toml /src/dtmt/lib/serde_sjson/ +COPY lib/ansi-parser/Cargo.toml /src/dtmt/lib/ansi-parser/ + +# Crates with build scripts cannot be split that way, but they shouldn't change too often +COPY lib/luajit2-sys /src/dtmt/lib/luajit2-sys +COPY lib/oodle /src/dtmt/lib/oodle +# color-eyre needs to be copied, too, then, as it's used by `oodle` +COPY lib/color-eyre /src/dtmt/lib/color-eyre +COPY --from=dtmt-ci-base-linux /src/*.lib /src/dtmt/lib/oodle/ + +RUN cargo build --release --locked +RUN rm -r crates lib + COPY . /src/dtmt COPY --from=dtmt-ci-base-linux /src/*.lib /src/*.so /src/dtmt/lib/oodle/ -RUN --mount=type=cache,id=cargo-registry,target=/cargo/registry \ - --mount=type=cache,id=cargo-target,target=/src/dtmt/target \ - cargo build --release --locked + +RUN cargo build --release --locked diff --git a/.ci/image/Dockerfile.linux b/.ci/image/Dockerfile.linux new file mode 100644 index 0000000..df10059 --- /dev/null +++ b/.ci/image/Dockerfile.linux @@ -0,0 +1,25 @@ +FROM rust:slim-bullseye + +RUN set -eux; \ + apt-get update; \ + apt-get install --no-install-recommends -y \ + build-essential \ + curl \ + git \ + gpg \ + jq \ + libatk1.0-dev \ + libclang-13-dev \ + libglib2.0-dev \ + libgtk-3-dev \ + libpango1.0-dev \ + libssl-dev \ + libzstd-dev \ + pkg-config; \ + apt-get remove -y --auto-remove; \ + rm -rf /var/lib/apt/lists/*; \ + rustup default nightly + +WORKDIR /src/dtmt + +COPY *.so *.a /src/ diff --git a/.ci/image/Dockerfile b/.ci/image/Dockerfile.msvc similarity index 73% rename from .ci/image/Dockerfile rename to .ci/image/Dockerfile.msvc index 4e3433b..a9eab62 100644 --- a/.ci/image/Dockerfile +++ b/.ci/image/Dockerfile.msvc @@ -1,68 +1,13 @@ # https://jake-shadle.github.io/xwin/ -FROM debian:bullseye-slim as xwin - -ARG XWIN_VERSION=0.5.0 -ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl" -ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz - -RUN set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - tar \ - ; \ - # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`. - tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/bin --strip-components=1 $XWIN_PREFIX/xwin; \ - apt-get remove -y --auto-remove; \ - rm -rf \ - /var/lib/apt/lists/* \ - /root/$XWIN_PREFIX.tar.gz; - -RUN set -eux; \ - # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively - xwin \ - --log-level debug \ - --cache-dir /root/.xwin-cache \ - --manifest-version 16 \ - --accept-license \ - splat \ - --output /xwin; \ - # Even though this build step only exists temporary, to copy the - # final data out of, it still generates a cache entry on the Docker host. - # And to keep that to a minimum, we still delete the stuff we don't need. - rm -rf /root/.xwin-cache; - -FROM rust:slim-bullseye as linux - -RUN set -eux; \ - apt-get update; \ - apt-get install --no-install-recommends -y \ - build-essential \ - curl \ - git \ - gpg \ - jq \ - libatk1.0-dev \ - libclang-13-dev \ - libglib2.0-dev \ - libgtk-3-dev \ - libpango1.0-dev \ - libssl-dev \ - libzstd-dev \ - pkg-config; \ - apt-get remove -y --auto-remove; \ - rm -rf /var/lib/apt/lists/*; \ - rustup default nightly - -WORKDIR /src/dtmt - -COPY lib/oodle/*.so lib/oodle/*.a /src/ - -FROM linux as msvc +FROM dtmt-ci-base-linux ENV KEYRINGS /usr/local/share/keyrings +ARG XWIN_VERSION=0.2.11 +ARG XWIN_PREFIX="xwin-$XWIN_VERSION-x86_64-unknown-linux-musl" ADD https://apt.llvm.org/llvm-snapshot.gpg.key /root/llvm-snapshot.gpg.key ADD https://dl.winehq.org/wine-builds/winehq.key /root/winehq.key +ADD https://github.com/Jake-Shadle/xwin/releases/download/$XWIN_VERSION/$XWIN_PREFIX.tar.gz /root/$XWIN_PREFIX.tar.gz RUN set -eux; \ mkdir -p $KEYRINGS; \ @@ -81,7 +26,7 @@ RUN set -eux; \ llvm-13 \ lld-13 \ winehq-staging \ - ; \ + tar; \ # ensure that clang/clang++ are callable directly ln -s clang-13 /usr/bin/clang && ln -s clang /usr/bin/clang++ && ln -s lld-13 /usr/bin/ld.lld; \ # We also need to setup symlinks ourselves for the MSVC shims because they aren't in the debian packages @@ -99,15 +44,19 @@ RUN set -eux; \ update-alternatives --install /usr/bin/ld ld /usr/bin/ld.lld 100; \ rustup target add x86_64-pc-windows-msvc; \ rustup component add rust-src; \ + # Install xwin to cargo/bin via github release. Note you could also just use `cargo install xwin`. + tar -xzv -f /root/$XWIN_PREFIX.tar.gz -C /usr/local/cargo/bin --strip-components=1 $XWIN_PREFIX/xwin; \ + # Splat the CRT and SDK files to /xwin/crt and /xwin/sdk respectively + xwin --accept-license splat --output /xwin; \ # Remove unneeded files to reduce image size apt-get remove -y --auto-remove; \ rm -rf \ + .xwin-cache \ + /usr/local/cargo/bin/xwin \ + /root/$XWIN_PREFIX.tar.gz \ /var/lib/apt/lists/* \ /root/*.key; -COPY lib/oodle/*.lib /src -COPY --from=xwin /xwin /xwin - # Note that we're using the full target triple for each variable instead of the # simple CC/CXX/AR shorthands to avoid issues when compiling any C/C++ code for # build dependencies that need to compile and execute in the host environment @@ -134,3 +83,7 @@ ENV CFLAGS_x86_64_pc_windows_msvc="$CL_FLAGS" \ # Run wineboot just to setup the default WINEPREFIX so we don't do it every # container run RUN wine wineboot --init + +WORKDIR /src/dtmt + +COPY *.lib /src diff --git a/.ci/pipelines/base-pipeline.yml b/.ci/pipelines/base-pipeline.yml new file mode 100644 index 0000000..f231fb7 --- /dev/null +++ b/.ci/pipelines/base-pipeline.yml @@ -0,0 +1,43 @@ +--- + +# The base pipeline that runs continuously, checks for branches and +# creates a new pipeline instance for each of them. + +resource_types: +- name: gitea-pr + type: registry-image + source: + repository: registry.local:5000/gitea-pr + +resources: +- name: repo-pr + type: gitea-pr + source: + access_token: ((gitea_api_key)) + owner: ((owner)) + repo: ((repo)) + url: https://git.sclu1034.dev +- name: repo + type: git + source: + uri: https://git.sclu1034.dev/bitsquid_dt/dtmt + +jobs: + - name: set-pipelines + plan: + - in_parallel: + - get: repo-pr + trigger: true + - get: repo + - load_var: prs + file: repo-pr/prs.json + - across: + - var: pr + values: ((.:prs)) + set_pipeline: dtmt-pr + file: repo/.ci/pipelines/pr.yml + vars: + pr: ((.:pr)) + gitea_api_key: ((gitea_api_key)) + instance_vars: + number: ((.:pr.number)) diff --git a/.ci/pipelines/base.yml b/.ci/pipelines/base.yml deleted file mode 100644 index 474c090..0000000 --- a/.ci/pipelines/base.yml +++ /dev/null @@ -1,206 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/cappyzawa/concourse-pipeline-jsonschema/master/concourse_jsonschema.json#/definitions/Config ---- - -# The actual CI pipeline that is run per branch -resource_types: -- name: gitea-package - type: registry-image - source: - repository: registry.local:5000/gitea-package - -- name: gitea-status - type: registry-image - source: - repository: registry.local:5000/gitea-status - -- name: gitea-pr - type: registry-image - source: - repository: registry.local:5000/gitea-pr - - -resources: -- name: repo - type: git - source: - uri: http://forgejo:3000/bitsquid_dt/dtmt - branch: master - -- name: repo-pr - type: gitea-pr - source: - access_token: ((gitea_api_key)) - owner: ((owner)) - repo: ((repo)) - url: https://git.sclu1034.dev - -- name: gitea-package - type: gitea-package - source: - access_token: ((gitea_api_key)) - url: http://forgejo:3000 - owner: bitsquid_dt - type: generic - name: dtmt - - -- name: status-build-msvc - type: gitea-status - source: - access_token: ((gitea_api_key)) - url: http://forgejo:3000 - owner: bitsquid_dt - repo: dtmt - context: build/msvc - description: "Build for the target platform: msvc" - -- name: status-build-linux - type: gitea-status - source: - access_token: ((gitea_api_key)) - url: http://forgejo:3000 - owner: bitsquid_dt - repo: dtmt - context: build/linux - description: "Build for the target platform: linux" - - -jobs: -- name: set-pipelines - plan: - - in_parallel: - - get: repo-pr - trigger: true - - - get: repo - - - load_var: prs - file: repo-pr/prs.json - - - across: - - var: pr - values: ((.:prs)) - set_pipeline: dtmt-pr - file: repo/.ci/pipelines/pr.yml - vars: - pr: ((.:pr)) - gitea_api_key: ((gitea_api_key)) - instance_vars: - number: ((.:pr.number)) - - -- name: build-msvc - on_success: - put: state-success - resource: status-build-msvc - no_get: true - params: - state: success - sha: ((.:git_sha)) - - on_failure: - put: state-failure - resource: status-build-msvc - no_get: true - params: - state: failure - sha: ((.:git_sha)) - - plan: - - get: repo - trigger: true - - - load_var: git_sha - file: repo/.git/ref - - - put: state-pending - resource: status-build-msvc - no_get: true - params: - state: pending - sha: ((.:git_sha)) - - - task: build - file: repo/.ci/tasks/build.yml - vars: - pr: "" - target: msvc - gitea_url: http://forgejo:3000 - gitea_api_key: ((gitea_api_key)) - - - load_var: version_number - reveal: true - file: artifact/version - - - put: package - resource: gitea-package - no_get: true - inputs: - - artifact - params: - version: ((.:version_number)) - fail_fast: true - override: true - globs: - - artifact/dtmt - - artifact/dtmm - - artifact/*.exe - - artifact/*.sha256 - -- name: build-linux - on_success: - put: state-success - resource: status-build-linux - no_get: true - params: - state: success - sha: ((.:git_sha)) - - on_failure: - put: state-failure - resource: status-build-linux - no_get: true - params: - state: failure - sha: ((.:git_sha)) - - plan: - - get: repo - trigger: true - - - load_var: git_sha - file: repo/.git/ref - - - put: state-pending - resource: status-build-linux - no_get: true - params: - state: pending - sha: ((.:git_sha)) - - - task: build - file: repo/.ci/tasks/build.yml - vars: - pr: "" - target: linux - gitea_url: http://forgejo:3000 - gitea_api_key: ((gitea_api_key)) - - - load_var: version_number - reveal: true - file: artifact/version - - - put: package - resource: gitea-package - no_get: true - inputs: - - artifact - params: - version: ((.:version_number)) - fail_fast: true - override: true - globs: - - artifact/dtmt - - artifact/dtmm - - artifact/*.exe - - artifact/*.sha256 diff --git a/.ci/tasks/build.sh b/.ci/tasks/build.sh index bb96775..5e60d7f 100755 --- a/.ci/tasks/build.sh +++ b/.ci/tasks/build.sh @@ -19,9 +19,7 @@ install_artifact() { cd "repo" -PR=${PR:-} - -if [ -n "$PR" ]; then +if [ -n "${PR:-}" ]; then title "PR: $(echo "$PR" | jq '.number') - $(echo "$PR" | jq '.title')" ref="pr-$(echo "$PR" | jq '.number')-$(git rev-parse --short "$(cat .git/ref || echo "HEAD")" 2>/dev/null || echo 'manual')" else diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 1fd318f..31910d3 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -16,7 +16,6 @@ - dtmt: add utility to migrate mod projects - dtmm: reset dtkit-patch installations - sdk: implement decompiling Lua files -- dtmm: fetch cover image for Nexus mods - dtmm: fetch file version for Nexus mods === Fixed diff --git a/Justfile b/Justfile index f9b37bc..70af61a 100644 --- a/Justfile +++ b/Justfile @@ -16,21 +16,22 @@ build-image-msvc: build-image-linux: docker build -f .ci/Dockerfile.linux . -ci-image: - # The MSVC image depends on the Linux image. So by building that first, - # we actually build both, and cache them, so that "building" the - # Linux image afterwards merely needs to pull the cache. - docker build --target msvc -t dtmt-ci-base-msvc -f .ci/image/Dockerfile . - docker build --target linux -t dtmt-ci-base-linux -f .ci/image/Dockerfile . +ci-image: ci-image-msvc ci-image-linux + +ci-image-msvc: ci-image-linux + docker build -t dtmt-ci-base-msvc -f .ci/image/Dockerfile.msvc .ci/image docker tag dtmt-ci-base-msvc registry.sclu1034.dev/dtmt-ci-base-msvc - docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux docker push registry.sclu1034.dev/dtmt-ci-base-msvc + +ci-image-linux: + docker build -t dtmt-ci-base-linux -f .ci/image/Dockerfile.linux .ci/image + docker tag dtmt-ci-base-linux registry.sclu1034.dev/dtmt-ci-base-linux docker push registry.sclu1034.dev/dtmt-ci-base-linux set-base-pipeline: fly -t {{fly_target}} set-pipeline \ - --pipeline dtmt \ - --config .ci/pipelines/base.yml \ + --pipeline dtmt-prs \ + --config .ci/pipelines/base-pipeline.yml \ -v gitea_api_key=${GITEA_API_KEY} \ -v owner=bitsquid_dt \ -v repo=dtmt diff --git a/crates/dtmm/src/controller/import.rs b/crates/dtmm/src/controller/import.rs index cef7ee4..68f2b05 100644 --- a/crates/dtmm/src/controller/import.rs +++ b/crates/dtmm/src/controller/import.rs @@ -27,16 +27,6 @@ fn find_archive_file( path } -fn image_data_to_buffer(data: impl AsRef<[u8]>) -> Result { - // Druid somehow doesn't return an error compatible with eyre, here. - // So we have to wrap through `Display` manually. - ImageBuf::from_data(data.as_ref()).map_err(|err| { - Report::msg(err.to_string()) - .wrap_err("Invalid image data") - .suggestion("Supported formats are: PNG, JPEG, Bitmap and WebP") - }) -} - // Runs the content of a `.mod` file to extract what data we can get // from legacy mods. // 1. Create a global function `new_mod` that stores @@ -448,8 +438,6 @@ pub(crate) async fn import_mod(state: ActionState, info: FileInfo) -> Result Result Result { - mod_cfg.image = Some(name.into()); - Some(img) - } + // Druid somehow doesn't return an error compatible with eyre, here. + // So we have to wrap through `Display` manually. + let img = match ImageBuf::from_data(&buf) { + Ok(img) => img, Err(err) => { - let err = Report::new(err).wrap_err(format!( - "Failed to write Nexus picture to file '{}'", - path.display() - )); - tracing::error!("{:?}", err); - None + let err = Report::msg(err.to_string()) + .wrap_err("Invalid image data") + .note("Supported formats are: PNG, JPEG, Bitmap and WebP") + .suggestion("Contact the mod author to fix this"); + return Err(err); } - } + }; + + Some(img) } else { None }; tracing::trace!(?image); - tracing::debug!(root, ?mod_cfg); + + let mod_dir = state.data_dir.join(state.mod_dir.as_ref()); + let dest = mod_dir.join(&mod_cfg.id); + + tracing::trace!("Creating mods directory {}", dest.display()); + fs::create_dir_all(&dest) + .await + .wrap_err_with(|| format!("Failed to create data directory '{}'", dest.display()))?; let packages = if mod_cfg.bundled { extract_bundled_mod(&mut archive, root, &mod_dir).wrap_err("Failed to extract mod")? diff --git a/crates/dtmm/src/state/data.rs b/crates/dtmm/src/state/data.rs index e5b70c4..64fdd28 100644 --- a/crates/dtmm/src/state/data.rs +++ b/crates/dtmm/src/state/data.rs @@ -78,7 +78,6 @@ pub(crate) struct NexusInfo { pub author: String, pub summary: Arc, pub description: Arc, - pub picture_url: Arc, } impl From for NexusInfo { @@ -90,7 +89,6 @@ impl From for NexusInfo { author: value.author, summary: Arc::new(value.summary), description: Arc::new(value.description), - picture_url: Arc::new(value.picture_url.into()), } } } diff --git a/lib/nexusmods/src/lib.rs b/lib/nexusmods/src/lib.rs index 5fb08e2..06c1f01 100644 --- a/lib/nexusmods/src/lib.rs +++ b/lib/nexusmods/src/lib.rs @@ -4,7 +4,7 @@ use std::convert::Infallible; use lazy_static::lazy_static; use regex::Regex; use reqwest::header::{HeaderMap, HeaderValue, InvalidHeaderValue}; -use reqwest::{Client, IntoUrl, RequestBuilder, Url}; +use reqwest::{Client, RequestBuilder, Url}; use serde::Deserialize; use thiserror::Error; @@ -126,16 +126,6 @@ impl Api { Ok(file.version) } - #[tracing::instrument(skip(self))] - pub async fn picture(&self, url: impl IntoUrl + std::fmt::Debug) -> Result> { - let res = self.client.get(url).send().await?.error_for_status()?; - - res.bytes() - .await - .map(|bytes| bytes.to_vec()) - .map_err(From::from) - } - pub fn parse_file_name>( name: S, ) -> Option<(String, u64, String, OffsetDateTime)> {