From 3cde8dde52586a22c92892c513200e4fe6e665de Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Mon, 20 Nov 2023 16:29:32 +0100 Subject: [PATCH] gitea: Implement package --- images/gitea/Cargo.lock | 118 +++++++++++- images/gitea/Cargo.toml | 9 +- images/gitea/Dockerfile | 19 +- images/gitea/shims/package/check | 2 + images/gitea/shims/package/in | 2 + images/gitea/shims/package/out | 2 + images/gitea/src/cmd/package.rs | 301 +++++++++++++++++++++++++++++++ images/gitea/src/cmd/pr.rs | 6 +- images/gitea/src/main.rs | 12 +- images/gitea/src/types.rs | 28 ++- images/gitea/test.sh | 38 ++++ 11 files changed, 514 insertions(+), 23 deletions(-) create mode 100755 images/gitea/shims/package/check create mode 100755 images/gitea/shims/package/in create mode 100755 images/gitea/shims/package/out create mode 100644 images/gitea/src/cmd/package.rs create mode 100755 images/gitea/test.sh diff --git a/images/gitea/Cargo.lock b/images/gitea/Cargo.lock index 52cf3f5..e0411d0 100644 --- a/images/gitea/Cargo.lock +++ b/images/gitea/Cargo.lock @@ -17,6 +17,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.2.6" @@ -90,6 +99,16 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bstr" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -185,8 +204,7 @@ dependencies = [ [[package]] name = "color-eyre" version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a667583cca8c4f8436db8de46ea8233c42a7d9ae424a82d338f2e4675229204" +source = "git+https://github.com/sclu1034/color-eyre.git?branch=fork#0fa05eba9954be223b06468c8760b97e660f9941" dependencies = [ "backtrace", "color-spantrace", @@ -372,11 +390,12 @@ checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "gitea" -version = "0.1.0" +version = "0.1.2" dependencies = [ "clap", "cli-table", "color-eyre", + "globwalk", "reqwest", "serde", "serde_json", @@ -384,6 +403,30 @@ dependencies = [ "url", ] +[[package]] +name = "globset" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759c97c1e17c55525b57192c06a267cda0ac5210b222d6b82189a2338fa1c13d" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + +[[package]] +name = "globwalk" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e3af942408868f6934a7b85134a3230832b9977cf66125df2f9edcfce4ddcc" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + [[package]] name = "h2" version = "0.3.16" @@ -511,6 +554,23 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "ignore" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" +dependencies = [ + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "indenter" version = "0.3.3" @@ -600,9 +660,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "mime" @@ -704,6 +764,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" + [[package]] name = "reqwest" version = "0.11.16" @@ -817,6 +906,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -1192,6 +1290,16 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" diff --git a/images/gitea/Cargo.toml b/images/gitea/Cargo.toml index 51e08d7..7ec1d78 100644 --- a/images/gitea/Cargo.toml +++ b/images/gitea/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gitea" -version = "0.1.0" +version = "0.1.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,7 +14,14 @@ serde = { version = "1.0.159", features = ["derive"] } time = { version = "0.3.20", features = ["formatting", "parsing", "serde"] } url = { version = "2.3.1", features = ["serde"] } cli-table = "0.4.7" +globwalk = "0.8.1" [profile.release] strip = "debuginfo" lto = true + +[profile.dev.package.backtrace] +opt-level = 3 + +[patch.crates-io] +color-eyre = { git = "https://github.com/sclu1034/color-eyre.git", branch = "fork" } diff --git a/images/gitea/Dockerfile b/images/gitea/Dockerfile index 1916764..8dc4bea 100644 --- a/images/gitea/Dockerfile +++ b/images/gitea/Dockerfile @@ -22,20 +22,19 @@ RUN cargo build --color always --release --locked COPY . . RUN touch -a -m ./src/main.rs && cargo build --color always --release --locked -FROM alpine AS pr -# FROM debian:bullseye-slim AS pr +FROM alpine AS final RUN apk add --no-cache ca-certificates -# RUN set -e; \ -# apt-get update; \ -# apt-get install -y --no-install-recommends \ -# ca-certificates; \ -# rm -fr /var/apt/lists/*; - -LABEL version="0.1.0" ENV RUST_BACKTRACE=1 ENV RUST_LOG=info -COPY ./shims/pr/* /opt/resource/ COPY --from=builder /app/target/release/gitea /bin/ + +# Put this last, as it's the only thing that's different between variants. +# This way, the previous layers can all be shared. + +ARG VARIANT="missing build arg 'VARIANT'" +ARG VERSION="0.1.0" +LABEL version="$VERSION" +COPY ./shims/${VARIANT}/* /opt/resource/ diff --git a/images/gitea/shims/package/check b/images/gitea/shims/package/check new file mode 100755 index 0000000..82430e1 --- /dev/null +++ b/images/gitea/shims/package/check @@ -0,0 +1,2 @@ +#!/bin/sh +/bin/gitea package check "$@" diff --git a/images/gitea/shims/package/in b/images/gitea/shims/package/in new file mode 100755 index 0000000..48c8805 --- /dev/null +++ b/images/gitea/shims/package/in @@ -0,0 +1,2 @@ +#!/bin/sh +/bin/gitea package in "$@" diff --git a/images/gitea/shims/package/out b/images/gitea/shims/package/out new file mode 100755 index 0000000..0ea48e9 --- /dev/null +++ b/images/gitea/shims/package/out @@ -0,0 +1,2 @@ +#!/bin/sh +/bin/gitea package out "$@" diff --git a/images/gitea/src/cmd/package.rs b/images/gitea/src/cmd/package.rs new file mode 100644 index 0000000..ec43d59 --- /dev/null +++ b/images/gitea/src/cmd/package.rs @@ -0,0 +1,301 @@ +use std::fs; +use std::io::{self, Read, Write}; +use std::path::Path; + +use cli_table::format::Justify; +use cli_table::{Cell, Style, Table}; +use color_eyre::eyre::{self, Context}; +use color_eyre::{Help, Report, Result}; +use globwalk::{DirEntry, GlobWalkerBuilder}; +use reqwest::blocking::Client; +use reqwest::header::HeaderMap; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use time::format_description::well_known::Iso8601; +use url::Url; + +use crate::types::{Package, PackageType}; +use crate::{Action, USER_AGENT}; + +#[derive(Deserialize, Serialize, Debug)] +struct Source { + access_token: String, + owner: String, + url: Url, + r#type: PackageType, + name: String, +} + +#[derive(Deserialize, Serialize, Debug)] +struct Version { + version: String, +} + +#[derive(Deserialize, Serialize, Debug)] +struct Params { + version: String, + globs: Vec, + #[serde(default)] + fail_fast: bool, + #[serde(default)] + r#override: bool, +} + +#[derive(Deserialize, Serialize, Debug)] +struct Config { + /// Resource configuration. + /// Passed verbatim from the definition in the pipeline. + source: Source, + /// For 'check': + /// Previous version of this resource. + /// Will be empty for the very first request. + /// For 'in': + /// The version metadata to create this as. + version: Option, + params: Option, +} + +fn make_client(src: &Source) -> Result { + let mut headers = HeaderMap::new(); + headers.insert( + "Authorization", + format!("token {}", src.access_token).try_into()?, + ); + + Client::builder() + .default_headers(headers) + .user_agent(USER_AGENT) + .build() + .map_err(From::from) +} + +fn action_check(conf: Config) -> Result<()> { + let client = make_client(&conf.source)?; + let url = conf + .source + .url + .join(&format!("api/v1/packages/{}", conf.source.owner)) + .wrap_err("Invalid URL")?; + + let mut pkgs: Vec = client + .get(url) + .query(&[ + ("type", conf.source.r#type.to_string()), + ("q", conf.source.name), + ]) + .send()? + .json()?; + + if pkgs.is_empty() { + return io::stdout().write_all(b"[]").map_err(From::from); + } + + pkgs.sort_unstable_by_key(|pkg| pkg.created_at); + + let table: Vec<_> = pkgs + .iter() + .map(|pkg| { + vec![ + pkg.id.cell().justify(Justify::Center), + pkg.name.clone().cell(), + pkg.owner.login.clone().cell(), + pkg.version.clone().cell(), + pkg.created_at + .format(&Iso8601::DEFAULT) + .unwrap_or_default() + .cell(), + ] + }) + .collect(); + + let table = table + .table() + .title(vec![ + "#".cell().bold(true), + "Name".cell().bold(true), + "Owner".cell().bold(true), + "Version".cell().bold(true), + "Created At".cell().bold(true), + ]) + .bold(true); + + let _ = cli_table::print_stderr(table); + + let newest = pkgs + .last() + .expect("List of packages should not be empty.") + .version + .clone(); + + let out = if let Some(prev) = conf.version { + if prev.version == newest { + vec![prev] + } else { + vec![prev, Version { version: newest }] + } + } else { + vec![Version { version: newest }] + }; + + serde_json::to_writer_pretty(io::stdout(), &out) + .wrap_err("Failed to write result to stdout")?; + + Ok(()) +} + +fn action_out(conf: Config, dir: impl AsRef) -> Result<()> { + let dir = dir + .as_ref() + .canonicalize() + .wrap_err_with(|| format!("Invalid file path '{}'", dir.as_ref().display()))?; + + let Some(params) = conf.params else { + eyre::bail!("Missing params"); + }; + + if params.globs.is_empty() { + eyre::bail!("`params.globs` must not be empty"); + } + + let client = make_client(&conf.source)?; + let url = conf + .source + .url + .join(&format!( + // The trailing slash is required, to make sure this entire URL is used + // when `join`ing the file name later. Otherwise, the last component + // would be considered a "file" part itself, and replaced by a future `join`. + "api/packages/{}/{}/{}/{}/", + conf.source.owner, conf.source.r#type, conf.source.name, params.version + )) + .wrap_err("Invalid URL")?; + + let walker = GlobWalkerBuilder::from_patterns(dir, ¶ms.globs) + .max_depth(3) + .file_type(globwalk::FileType::FILE) + .build() + .wrap_err("Failed to create glob walker")?; + + let handle_glob = |entry| { + let entry: DirEntry = entry?; + + let name = entry.file_name(); + let path = entry.path(); + let url = url.join(&name.to_string_lossy()).wrap_err("Invalid url")?; + + if params.r#override { + client + .delete(url.clone()) + .send() + .wrap_err_with(|| format!("Failed request 'DELETE {}'", &url))?; + } + + let data = fs::read(path) + .wrap_err_with(|| format!("Failed to read package file '{}'", path.display()))?; + + let res = client + .put(url.clone()) + .body(data) + .send() + .wrap_err_with(|| format!("Failed request 'PUT {}'", &url))?; + + match res.status() { + StatusCode::CREATED => { + eprintln!( + "Uploaded file '{}', version={}, package={}", + name.to_string_lossy(), + params.version, + conf.source.name + ); + } + StatusCode::BAD_REQUEST => { + eyre::bail!( + "Package, version or file name are invalid. package={}, version={}, file={}", + conf.source.name, + params.version, + name.to_string_lossy(), + ); + } + StatusCode::CONFLICT => { + eyre::bail!( + "File '{}' already exists. version={}, package={}", + name.to_string_lossy(), + params.version, + conf.source.name + ); + } + StatusCode::INTERNAL_SERVER_ERROR => { + eyre::bail!( + "Internal server error: {} {}", + res.status(), + res.text().unwrap_or_default() + ); + } + code => { + eyre::bail!("Unexpected status code {}\ntext = {:?}", code, res.text()); + } + } + + Ok::<_, Report>(()) + }; + + let mut has_entry = false; + + let mut errors = Vec::new(); + for entry in walker { + has_entry = true; + if let Err(err) = handle_glob(entry) { + errors.push(err); + if params.fail_fast { + break; + } + } + } + + if !errors.is_empty() { + let mut err = eyre::eyre!("Failed to upload package with {} errors", errors.len()); + + for e in errors { + err = err.report(e); + } + + return Err(err); + } + + if !has_entry { + eyre::bail!("Globs didn't produce any files"); + } + + let out = json!({ + "version": { + "version": params.version + } + }); + + serde_json::to_writer_pretty(io::stdout(), &out) + .wrap_err("Failed to write result to stdout")?; + + Ok(()) +} + +pub(crate) fn run(action: &Action) -> Result<()> { + let config: Config = { + let mut buf = String::new(); + io::stdin() + .read_to_string(&mut buf) + .wrap_err("Failed to read from stdin")?; + + if buf.is_empty() { + eyre::bail!("No data received on stdin"); + } + + serde_json::from_str(&buf).wrap_err("Failed to parse stdin")? + }; + + match action { + Action::Check => action_check(config), + Action::In { dest: _ } => Ok(()), + Action::Out { src } => action_out(config, src), + } +} diff --git a/images/gitea/src/cmd/pr.rs b/images/gitea/src/cmd/pr.rs index 2e5577f..eafbcff 100644 --- a/images/gitea/src/cmd/pr.rs +++ b/images/gitea/src/cmd/pr.rs @@ -14,9 +14,7 @@ use time::OffsetDateTime; use url::Url; use crate::types::PullRequest; -use crate::Action; - -static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); +use crate::{Action, USER_AGENT}; #[derive(Deserialize, Serialize, Debug)] struct Source { @@ -182,6 +180,6 @@ pub(crate) fn run(action: &Action) -> Result<()> { match action { Action::Check => action_check(config), Action::In { dest } => action_in(config, dest), - Action::Out => Ok(()), + Action::Out { src: _ } => Ok(()), } } diff --git a/images/gitea/src/main.rs b/images/gitea/src/main.rs index 1f71387..4e6e13f 100644 --- a/images/gitea/src/main.rs +++ b/images/gitea/src/main.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use clap::{Parser, Subcommand}; use color_eyre::Result; +static USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION")); + #[derive(Debug, Parser)] #[command(author, version, about, long_about = None)] #[command(propagate_version = true)] @@ -17,25 +19,31 @@ enum Commands { #[command(subcommand)] action: Action, }, + Package { + #[command(subcommand)] + action: Action, + }, } #[derive(Clone, Debug, Subcommand)] pub(crate) enum Action { Check, In { dest: PathBuf }, - Out, + Out { src: PathBuf }, } mod types; mod cmd { + pub mod package; pub mod pr; } -// #[tracing::instrument] fn main() -> Result<()> { + color_eyre::install()?; let cli = Cli::parse(); match &cli.command { Commands::Pr { action } => cmd::pr::run(action), + Commands::Package { action } => cmd::package::run(action), } } diff --git a/images/gitea/src/types.rs b/images/gitea/src/types.rs index 17f450e..23d7b6a 100644 --- a/images/gitea/src/types.rs +++ b/images/gitea/src/types.rs @@ -1,4 +1,5 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; #[derive(Clone, Debug, Deserialize)] pub struct Ref { @@ -18,3 +19,28 @@ pub struct PullRequest { pub base: Ref, pub head: Ref, } + +#[derive(Copy, Clone, Deserialize, Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum PackageType { + Generic, +} + +impl std::fmt::Display for PackageType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + PackageType::Generic => write!(f, "generic"), + } + } +} + +#[derive(Clone, Debug, Deserialize)] +pub struct Package { + #[serde(with = "time::serde::iso8601")] + pub created_at: OffsetDateTime, + pub id: u64, + pub name: String, + pub owner: User, + pub r#type: PackageType, + pub version: String, +} diff --git a/images/gitea/test.sh b/images/gitea/test.sh new file mode 100755 index 0000000..1dbfc6a --- /dev/null +++ b/images/gitea/test.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +cargo build + +run() { +./target/debug/gitea package "$1" <