From a6a9fe59ec61e982177e8fad83ea81c330b98272 Mon Sep 17 00:00:00 2001 From: Lucas Schwiderski Date: Thu, 29 Aug 2024 11:58:39 +0200 Subject: [PATCH] feat: Implement repository mirror The `oci-build-task` seems to be doing more requests to the registry than local `docker build`, which frequently results in Docker Hub's rate limits kicking in. By proxying `docker.io`, this should hopefully be avoided. A somewhat elaborate setup and a custom build of `oci-build-task` are needed: https://github.com/concourse/oci-build-task/pull/121. Fixes: #39. --- Justfile | 20 +++++++++++++++++++- pipelines/build-and-push.yml.j2 | 13 ++++++------- pipelines/jobs/gitea.yml.j2 | 23 ++++++++++++----------- pipelines/jobs/rust-xwin.yml.j2 | 16 ++++++++-------- pipelines/jobs/simple-image.yml.j2 | 7 ++++--- tasks/build-image.yml | 3 ++- 6 files changed, 51 insertions(+), 31 deletions(-) diff --git a/Justfile b/Justfile index 82920d1..4d7588d 100644 --- a/Justfile +++ b/Justfile @@ -3,6 +3,7 @@ target := 'main' # Internal endpoint of the Docker registry, where no authentication is necessary registry_url := env_var_or_default('REGISTRY_URL', 'docker.io/') +registry_mirror_url := env_var_or_default('REGISTRY_MIRROR_URL', '') forgejo_api_key := env_var("FORGEJO_API_KEY") forgejo_url := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\).git|https://\\1/\\2|' | cut -d'/' -f-3") repo_owner := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\).git|https://\\1/\\2|' | cut -d'/' -f4") @@ -10,6 +11,15 @@ repo_name := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\). pipeline_file := shell('mktemp') +buildkit_config_tmpl := ''' +[registry."docker.io"] + mirrors = ["{{registry_mirror_url}}"] + +[registry."{{registry_mirror_url}}"] + http = true +''' +buildkit_config := replace(buildkit_config_tmpl, "{{registry_mirror_url}}", registry_mirror_url) + build context dockerfile='' image_target='' *args='': fly -t {{target}} execute \ --config=./tasks/build-image.yml \ @@ -19,6 +29,8 @@ build context dockerfile='' image_target='' *args='': -v context=repo/images/{{context}} \ -v dockerfile={{dockerfile}} \ -v target={{image_target}} \ + -v registry_url={{registry_url}} \ + -v 'buildkit_config={{buildkit_config}}' \ {{args}} make-pipeline file: @@ -26,13 +38,19 @@ make-pipeline file: fly -t {{target}} validate-pipeline \ --strict \ --config "{{file}}" \ - -v registry_url={{registry_url}} + -v registry_url={{registry_url}} \ + -v 'buildkit_config={{buildkit_config}}' \ + -v forgejo_api_key={{forgejo_api_key}} \ + -v forgejo_url={{forgejo_url}} \ + -v repo_owner={{repo_owner}} \ + -v repo_name={{repo_name}} set-pipeline: (make-pipeline pipeline_file) fly -t {{target}} set-pipeline \ --pipeline {{pipeline_name}} \ --config "{{pipeline_file}}" \ -v registry_url={{registry_url}} \ + -v 'buildkit_config={{buildkit_config}}' \ -v forgejo_api_key={{forgejo_api_key}} \ -v forgejo_url={{forgejo_url}} \ -v repo_owner={{repo_owner}} \ diff --git a/pipelines/build-and-push.yml.j2 b/pipelines/build-and-push.yml.j2 index edec459..06937a1 100644 --- a/pipelines/build-and-push.yml.j2 +++ b/pipelines/build-and-push.yml.j2 @@ -1,5 +1,5 @@ {%- import 'jobs/simple-image.yml.j2' as simple_image %} -{%- macro simple_image_resource(name, registry_url) -%} +{%- macro simple_image_resource(name) -%} - name: status-{{name}} type: gitea-status source: @@ -15,7 +15,7 @@ type: registry-image icon: docker source: - repository: "{{ registry_url }}/{{ name }}" + repository: "((registry_url))/{{ name }}" tag: latest {% endmacro -%} --- @@ -39,23 +39,22 @@ resources: uri: ((forgejo_url))/((repo_owner))/((repo_name)) branch: master - {%- set registry_url = "((registry_url))" %} {% for img in images -%} {%- if img.type == "simple" %} - {{ simple_image_resource(img.name, registry_url) }} + {{ simple_image_resource(img.name) }} {%- elif img.type == "file" %} {%- import "jobs/" + img.name + ".yml.j2" as job %} - {{ job.resources(registry_url) }} + {{ job.resources() }} {%- endif %} {%- endfor %} jobs: {%- for img in images %} {%- if img.type == "simple" %} - {{ simple_image.jobs(img, registry_url) }} + {{ simple_image.jobs(img) }} {%- elif img.type == "file" %} {%- import "jobs/" + img.name + ".yml.j2" as job %} - {{ job.jobs(registry_url) }} + {{ job.jobs() }} {%- endif %} {% endfor %} diff --git a/pipelines/jobs/gitea.yml.j2 b/pipelines/jobs/gitea.yml.j2 index 116e1e0..1a93ba1 100644 --- a/pipelines/jobs/gitea.yml.j2 +++ b/pipelines/jobs/gitea.yml.j2 @@ -1,13 +1,13 @@ -{% macro resource(variant, registry_url) -%} +{% macro resource(variant) -%} - name: image-gitea-{{ variant }} type: registry-image icon: docker source: - repository: "{{ registry_url }}/gitea-{{ variant }}" + repository: "((registry_url))/gitea-{{ variant }}" tag: latest {% endmacro %} -{% macro build(variant, registry_url) -%} +{% macro build(variant) -%} - task: build-gitea-{{ variant }} file: repo/tasks/build-image.yml privileged: true @@ -15,9 +15,10 @@ context: repo/images/gitea target: '' dockerfile: '' + registry_url: "((registry_url))" + buildkit_config: "((buildkit_config))" params: BUILD_ARG_VARIANT: {{ variant }} - REGISTRY_MIRRORS: {{ registry_url }} output_mapping: image: image-gitea-{{ variant }} {% endmacro %} @@ -29,7 +30,7 @@ no_get: true {% endmacro %} -{% macro resources(registry_url) -%} +{% macro resources() -%} - name: status-gitea type: gitea-status source: @@ -41,9 +42,9 @@ context: build/gitea description: Building images gitea-package, gitea-pr, gitea-status - {{ resource('package', registry_url) }} - {{ resource('status', registry_url) }} - {{ resource('pr', registry_url) }} + {{ resource('package') }} + {{ resource('status') }} + {{ resource('pr') }} {% endmacro %} {% macro jobs(registry_url) -%} @@ -76,9 +77,9 @@ params: state: pending sha: ((.:git_sha)) - {{ build('package', registry_url) }} - {{ build('status', registry_url) }} - {{ build('pr', registry_url) }} + {{ build('package') }} + {{ build('status') }} + {{ build('pr') }} - in_parallel: steps: {{ put('package') }} diff --git a/pipelines/jobs/rust-xwin.yml.j2 b/pipelines/jobs/rust-xwin.yml.j2 index aea0ad3..4391f38 100644 --- a/pipelines/jobs/rust-xwin.yml.j2 +++ b/pipelines/jobs/rust-xwin.yml.j2 @@ -1,4 +1,4 @@ -{% macro resources(registry_url) -%} +{% macro resources() -%} - name: status-rust-xwin type: gitea-status source: @@ -14,18 +14,18 @@ type: registry-image icon: docker source: - repository: "{{ registry_url }}/rust-xwin" + repository: "((registry_url))/rust-xwin" tag: latest - name: image-rust-xwin-ci type: registry-image icon: docker source: - repository: "{{ registry_url }}/rust-xwin-ci" + repository: "((registry_url))/rust-xwin-ci" tag: latest {% endmacro %} -{% macro jobs(registry_url) -%} +{% macro jobs() -%} - name: rust-xwin serial: true on_success: @@ -62,8 +62,8 @@ context: repo/images/rust-xwin target: rust-xwin dockerfile: '' - params: - REGISTRY_MIRRORS: {{ registry_url }} + registry_url: "((registry_url))" + buildkit_config: "((buildkit_config))" output_mapping: image: image-rust-xwin - task: build-rust-xwin-ci @@ -73,8 +73,8 @@ context: repo/images/rust-xwin target: rust-xwin-ci dockerfile: '' - params: - REGISTRY_MIRRORS: {{ registry_url }} + registry_url: "((registry_url))" + buildkit_config: "((buildkit_config))" output_mapping: image: image-rust-xwin-ci - in_parallel: diff --git a/pipelines/jobs/simple-image.yml.j2 b/pipelines/jobs/simple-image.yml.j2 index 2eea0d9..649f06f 100644 --- a/pipelines/jobs/simple-image.yml.j2 +++ b/pipelines/jobs/simple-image.yml.j2 @@ -1,4 +1,4 @@ -{% macro jobs(job, registry_url) -%} +{% macro jobs(job) -%} - name: {{ job.name }} serial: true on_success: @@ -35,9 +35,10 @@ context: repo/images/{{ job.name }} dockerfile: '' target: '' - params: - REGISTRY_MIRROR: {{ registry_url }} + registry_url: "((registry_url))" + buildkit_config: "((buildkit_config))" {%- if "args" in job %} + params: {%- for name, value in job.args.items() %} BUILD_ARG_{{name}}: "{{value}}" {%- endfor %} diff --git a/tasks/build-image.yml b/tasks/build-image.yml index 69ca9d5..b83c7d9 100644 --- a/tasks/build-image.yml +++ b/tasks/build-image.yml @@ -6,7 +6,7 @@ image_resource: name: image type: registry-image source: - repository: concourse/oci-build-task + repository: ((registry_url))/oci-build-task tag: latest inputs: @@ -23,6 +23,7 @@ params: CONTEXT: ((context)) DOCKERFILE: ((dockerfile)) TARGET: ((target)) + BUILDKIT_EXTRA_CONFIG: ((buildkit_config)) run: path: build