feat: Implement repository mirror

The `oci-build-task` seems to be doing more requests to the registry
than local `docker build`, which frequently results in Docker Hub's
rate limits kicking in.

By proxying `docker.io`, this should hopefully be avoided.

A somewhat elaborate setup and a custom build of `oci-build-task` are
needed: https://github.com/concourse/oci-build-task/pull/121.

Fixes: #39.

Justfile

@@ -3,6 +3,7 @@ target := 'main'
 
 # Internal endpoint of the Docker registry, where no authentication is necessary
 registry_url := env_var_or_default('REGISTRY_URL', 'docker.io/')
+registry_mirror_url := env_var_or_default('REGISTRY_MIRROR_URL', '')
 forgejo_api_key := env_var("FORGEJO_API_KEY")
 forgejo_url := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\).git|https://\\1/\\2|' | cut -d'/' -f-3")
 repo_owner := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\).git|https://\\1/\\2|' | cut -d'/' -f4")
@@ -10,6 +11,15 @@ repo_name := shell("git remote get-url origin | sed 's|git@\\([^:]*\\):\\(.*\\).
 
 pipeline_file := shell('mktemp')
 
+buildkit_config_tmpl := '''
+[registry."docker.io"]
+  mirrors = ["{{registry_mirror_url}}"]
+
+[registry."{{registry_mirror_url}}"]
+  http = true
+'''
+buildkit_config := replace(buildkit_config_tmpl, "{{registry_mirror_url}}", registry_mirror_url)
+
 build context dockerfile='' image_target='' *args='':
     fly -t {{target}} execute \
         --config=./tasks/build-image.yml \
@@ -19,6 +29,8 @@ build context dockerfile='' image_target='' *args='':
         -v context=repo/images/{{context}} \
         -v dockerfile={{dockerfile}} \
         -v target={{image_target}} \
+        -v registry_url={{registry_url}} \
+        -v 'buildkit_config={{buildkit_config}}' \
         {{args}}
 
 make-pipeline file:
@@ -26,13 +38,19 @@ make-pipeline file:
     fly -t {{target}} validate-pipeline \
         --strict \
         --config "{{file}}" \
-        -v registry_url={{registry_url}}
+        -v registry_url={{registry_url}} \
+        -v 'buildkit_config={{buildkit_config}}' \
+        -v forgejo_api_key={{forgejo_api_key}} \
+        -v forgejo_url={{forgejo_url}} \
+        -v repo_owner={{repo_owner}} \
+        -v repo_name={{repo_name}}
 
 set-pipeline: (make-pipeline pipeline_file)
     fly -t {{target}} set-pipeline \
         --pipeline {{pipeline_name}} \
         --config "{{pipeline_file}}" \
         -v registry_url={{registry_url}} \
+        -v 'buildkit_config={{buildkit_config}}' \
         -v forgejo_api_key={{forgejo_api_key}} \
         -v forgejo_url={{forgejo_url}} \
         -v repo_owner={{repo_owner}} \

pipelines/build-and-push.yml.j2

@@ -1,5 +1,5 @@
 {%- import 'jobs/simple-image.yml.j2' as simple_image %}
-{%- macro simple_image_resource(name, registry_url) -%}
+{%- macro simple_image_resource(name) -%}
   - name: status-{{name}}
     type: gitea-status
     source:
@@ -15,7 +15,7 @@
     type: registry-image
     icon: docker
     source:
-      repository: "{{ registry_url }}/{{ name }}"
+      repository: "((registry_url))/{{ name }}"
       tag: latest
 {% endmacro -%}
 ---
@@ -39,23 +39,22 @@ resources:
       uri: ((forgejo_url))/((repo_owner))/((repo_name))
       branch: master
 
-  {%- set registry_url = "((registry_url))" %}
   {% for img in images -%}
   {%- if img.type == "simple" %}
-  {{ simple_image_resource(img.name, registry_url) }}
+  {{ simple_image_resource(img.name) }}
   {%- elif img.type == "file" %}
   {%- import "jobs/" + img.name + ".yml.j2" as job %}
-  {{ job.resources(registry_url) }}
+  {{ job.resources() }}
   {%- endif %}
   {%- endfor %}
 
 jobs:
   {%- for img in images %}
   {%- if img.type == "simple" %}
-  {{ simple_image.jobs(img, registry_url) }}
+  {{ simple_image.jobs(img) }}
   {%- elif img.type == "file" %}
   {%- import "jobs/" + img.name + ".yml.j2" as job %}
-  {{ job.jobs(registry_url) }}
+  {{ job.jobs() }}
   {%- endif %}
   {% endfor %}
 

pipelines/jobs/gitea.yml.j2

@@ -1,13 +1,13 @@
-{% macro resource(variant,  registry_url) -%}
+{% macro resource(variant) -%}
   - name: image-gitea-{{ variant }}
     type: registry-image
     icon: docker
     source:
-      repository: "{{ registry_url }}/gitea-{{ variant }}"
+      repository: "((registry_url))/gitea-{{ variant }}"
       tag: latest
 {% endmacro %}
 
-{% macro build(variant, registry_url) -%}
+{% macro build(variant) -%}
       - task: build-gitea-{{ variant }}
         file: repo/tasks/build-image.yml
         privileged: true
@@ -15,9 +15,10 @@
           context: repo/images/gitea
           target: ''
           dockerfile: ''
+          registry_url: "((registry_url))"
+          buildkit_config: "((buildkit_config))"
         params:
           BUILD_ARG_VARIANT: {{ variant }}
-          REGISTRY_MIRRORS: {{ registry_url }}
         output_mapping:
           image: image-gitea-{{ variant }}
 {% endmacro %}
@@ -29,7 +30,7 @@
               no_get: true
 {% endmacro %}
 
-{% macro resources(registry_url) -%}
+{% macro resources() -%}
   - name: status-gitea
     type: gitea-status
     source:
@@ -41,9 +42,9 @@
       context: build/gitea
       description: Building images gitea-package, gitea-pr, gitea-status
 
-  {{ resource('package', registry_url) }}
-  {{ resource('status', registry_url) }}
-  {{ resource('pr', registry_url) }}
+  {{ resource('package') }}
+  {{ resource('status') }}
+  {{ resource('pr') }}
 {% endmacro %}
 
 {% macro jobs(registry_url) -%}
@@ -76,9 +77,9 @@
         params:
           state: pending
           sha: ((.:git_sha))
-      {{ build('package', registry_url) }}
-      {{ build('status', registry_url) }}
-      {{ build('pr', registry_url) }}
+      {{ build('package') }}
+      {{ build('status') }}
+      {{ build('pr') }}
       - in_parallel:
           steps:
             {{ put('package') }}

pipelines/jobs/rust-xwin.yml.j2

@@ -1,4 +1,4 @@
-{% macro resources(registry_url) -%}
+{% macro resources() -%}
   - name: status-rust-xwin
     type: gitea-status
     source:
@@ -14,18 +14,18 @@
     type: registry-image
     icon: docker
     source:
-      repository: "{{ registry_url }}/rust-xwin"
+      repository: "((registry_url))/rust-xwin"
       tag: latest
 
   - name: image-rust-xwin-ci
     type: registry-image
     icon: docker
     source:
-      repository: "{{ registry_url }}/rust-xwin-ci"
+      repository: "((registry_url))/rust-xwin-ci"
       tag: latest
 {% endmacro %}
 
-{% macro jobs(registry_url) -%}
+{% macro jobs() -%}
   - name: rust-xwin
     serial: true
     on_success:
@@ -62,8 +62,8 @@
           context: repo/images/rust-xwin
           target: rust-xwin
           dockerfile: ''
-        params:
-          REGISTRY_MIRRORS: {{ registry_url }}
+          registry_url: "((registry_url))"
+          buildkit_config: "((buildkit_config))"
         output_mapping:
           image: image-rust-xwin
       - task: build-rust-xwin-ci
@@ -73,8 +73,8 @@
           context: repo/images/rust-xwin
           target: rust-xwin-ci
           dockerfile: ''
-        params:
-          REGISTRY_MIRRORS: {{ registry_url }}
+          registry_url: "((registry_url))"
+          buildkit_config: "((buildkit_config))"
         output_mapping:
           image: image-rust-xwin-ci
       - in_parallel:

pipelines/jobs/simple-image.yml.j2

@@ -1,4 +1,4 @@
-{% macro jobs(job, registry_url) -%}
+{% macro jobs(job) -%}
   - name: {{ job.name }}
     serial: true
     on_success:
@@ -35,9 +35,10 @@
           context: repo/images/{{ job.name }}
           dockerfile: ''
           target: ''
-        params:
-          REGISTRY_MIRROR: {{ registry_url }}
+          registry_url: "((registry_url))"
+          buildkit_config: "((buildkit_config))"
         {%- if "args" in job %}
+        params:
         {%- for name, value in job.args.items() %}
           BUILD_ARG_{{name}}: "{{value}}"
         {%- endfor %}

tasks/build-image.yml

@@ -6,7 +6,7 @@ image_resource:
   name: image
   type: registry-image
   source:
-    repository: concourse/oci-build-task
+    repository: ((registry_url))/oci-build-task
     tag: latest
 
 inputs:
@@ -23,6 +23,7 @@ params:
   CONTEXT: ((context))
   DOCKERFILE: ((dockerfile))
   TARGET: ((target))
+  BUILDKIT_EXTRA_CONFIG: ((buildkit_config))
 
 run:
   path: build