From 5ff1fafcea6e91956d19722031bb60ec19f302a3 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Tue, 23 Dec 2025 13:49:46 +0100 Subject: [PATCH 1/5] feat: add image_tag_suffix support --- .../build-docker-artifacts-config.schema.json | 16 ++++++++++++++++ .github/workflows/build-docker-artifacts.yml | 17 ++++++++++++----- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker-artifacts-config.schema.json b/.github/workflows/build-docker-artifacts-config.schema.json index 8604a918..acccae59 100644 --- a/.github/workflows/build-docker-artifacts-config.schema.json +++ b/.github/workflows/build-docker-artifacts-config.schema.json @@ -40,6 +40,14 @@ "type": "string", "description": "ECR repository to push the image to" }, + "image_tag_suffix": { + "type": "string", + "description": "Suffix to append to the image tag" + }, + "image_tag_prefix": { + "type": "string", + "description": "Prefix to prepend to the image tag" + }, "scan_high_severity": { "type": "boolean", "default": true, @@ -95,6 +103,14 @@ "type": "object", "additionalProperties": false, "properties": { + "image_tag_suffix": { + "type": "string", + "description": "Suffix to append to the image tag" + }, + "image_tag_prefix": { + "type": "string", + "description": "Prefix to prepend to the image tag" + }, "source_repository": { "type": "string", "description": "Internal ECR repository name" diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 35840a08..3a80f340 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -118,7 +118,6 @@ jobs: build_time: buildTime, image_tag: imageTag, image_tag_branch_name: imageTagBranchName, - ecr_repositories: flavor.components.map(component => component.ecr_repository), test_images: { enabled: testImageEnabled, script_path: testImagesHookScript, @@ -247,6 +246,8 @@ jobs: # Disable provenance as it creates weird multi-arch images: https://github.com/docker/build-push-action/issues/755 provenance: false # Disable the cache to avoid outdated (base) images + # TODO: are we sure we want this? We might benefit a lot from caching! But also, our base images like datavisyn/base/python:main are updated frequently and have no version tag... + # maybe no-cache-filters works? Or we do this on our self-hosted runners: https://docs.docker.com/build/ci/github-actions/cache/#local-cache no-cache: true build-args: | GIT_BRANCH=${{ steps.get-branch.outputs.branch }} @@ -432,13 +433,19 @@ jobs: echo "image_tag=$image_tag" echo "image_tag_branch_name=$image_tag_branch_name" - for repository_name in $(jq -r '.ecr_repositories[]' <<< "$FLAVOR"); do - IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index( $var )') - if [[ -z "${IMAGE_META}" || ${IMAGE_META} == "null" ]]; then + for component in $(jq -c '.components[]' <<< "$FLAVOR"); do + repository_name=$(jq -r '.ecr_repository' <<< "$component") + image_tag=$(jq -r '.image_tag' <<< "$component") + image_tag_branch_name=$(jq -r '.image_tag_branch_name' <<< "$component") + + echo "Processing repository: $repository_name, image_tag: $image_tag, image_tag_branch_name: $image_tag_branch_name" + + IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index($var)') + if [[ -z "${IMAGE_META}" || "${IMAGE_META}" == "null" ]]; then MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest') aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_branch_name" --image-manifest "$MANIFEST" else - echo "image already tagged!" + echo "Image already tagged for repository: $repository_name!" fi done; env: From 775a4e3cb2360253000ba777833c756331c144f5 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Tue, 23 Dec 2025 13:53:00 +0100 Subject: [PATCH 2/5] Get rid of additional image_tag --- .github/workflows/build-docker-artifacts.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 3a80f340..72482d31 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -116,8 +116,6 @@ jobs: skip_retag: skip_push, // Add metadata to the flavor object (will be used as matrix input) build_time: buildTime, - image_tag: imageTag, - image_tag_branch_name: imageTagBranchName, test_images: { enabled: testImageEnabled, script_path: testImagesHookScript, @@ -427,12 +425,6 @@ jobs: - name: Retag images shell: bash run: | - image_tag="${{ matrix.flavor.image_tag }}" - image_tag_branch_name="${{ matrix.flavor.image_tag_branch_name }}" - - echo "image_tag=$image_tag" - echo "image_tag_branch_name=$image_tag_branch_name" - for component in $(jq -c '.components[]' <<< "$FLAVOR"); do repository_name=$(jq -r '.ecr_repository' <<< "$component") image_tag=$(jq -r '.image_tag' <<< "$component") From d49f52a8edbf5eb1b1e72198ab684d9da8db6a7f Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Tue, 23 Dec 2025 14:02:29 +0100 Subject: [PATCH 3/5] Add suffix and prefix support --- .../workflows/build-docker-artifacts-push.yml | 4 +-- .../build-docker-artifacts-trigger-push.yml | 4 +-- .github/workflows/build-docker-artifacts.yml | 29 ++++++++++--------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-docker-artifacts-push.yml b/.github/workflows/build-docker-artifacts-push.yml index 04c6e2c2..2bba307e 100644 --- a/.github/workflows/build-docker-artifacts-push.yml +++ b/.github/workflows/build-docker-artifacts-push.yml @@ -67,8 +67,8 @@ jobs: const filteredRepositories = customer.repositories.filter(repo => !repo.skip); // Comma separated list of source images, incl. registry, repository and tag - const sourceImages = filteredRepositories.map(repo => `${process.env.SOURCE_ECR_REGISTRY}/${repo.source_repository}:${repo.source_image_tag || process.env.IMAGE_TAG}`).join(','); - const destinationImages = filteredRepositories.map(repo => `${customer.registry}/${repo.target_repository}:${repo.target_image_tag || process.env.IMAGE_TAG}`).join(','); + const sourceImages = filteredRepositories.map(repo => `${process.env.SOURCE_ECR_REGISTRY}/${repo.source_repository}:${repo.image_tag_prefix || ''}${repo.source_image_tag || process.env.IMAGE_TAG}${repo.image_tag_suffix || ''}`).join(','); + const destinationImages = filteredRepositories.map(repo => `${customer.registry}/${repo.target_repository}:${repo.image_tag_prefix || ''}${repo.target_image_tag || process.env.IMAGE_TAG}${repo.image_tag_suffix || ''}`).join(','); const result = { customer, diff --git a/.github/workflows/build-docker-artifacts-trigger-push.yml b/.github/workflows/build-docker-artifacts-trigger-push.yml index 6006eb71..169187aa 100644 --- a/.github/workflows/build-docker-artifacts-trigger-push.yml +++ b/.github/workflows/build-docker-artifacts-trigger-push.yml @@ -75,13 +75,13 @@ jobs: const customers = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {}); - const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); + const imageTagAfterRetag = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); const result = customers.map((c) => ({ repository: process.env.REPOSITORY, customer: c, customer_json: JSON.stringify(config.push[c]), - image_tag: imageTagBranchName, + image_tag: imageTagAfterRetag, })); console.log(result); return result; diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 72482d31..d55b2dcd 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -97,8 +97,6 @@ jobs: const config = require('./deploy/build/config.json'); const buildTime = new Date().toISOString().replace(/:/g, '').replace(/\..+/, 'Z'); - const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); - const imageTag = `tagged-${imageTagBranchName}-${buildTime}`; const builds = process.env.BUILDS ? process.env.BUILDS.split(',') : Object.keys(config.build); const push_to = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {}); @@ -126,6 +124,9 @@ jobs: const formattedBuildArgs = component.build_args ? Object.entries(component.build_args).map(([key, value]) => `${key}=${value}`).join('\n') : ''; + const imageTagAfterRetag = (component.image_tag_prefix || '') + "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-') + (component.image_tag_suffix || ''); + const imageTagBeforeRetag = `tagged-${imageTagAfterRetag}-${buildTime}`; + return { ...component, // We can skip the push only if we don't want to test the image below @@ -135,9 +136,9 @@ jobs: flavor_id: id, flavor_directory: `./deploy/build/${flavor.directory}`, build_time: buildTime, - image_tag: imageTag, - image_ref: `${{ vars.DV_AWS_ECR_REGISTRY }}/${component.ecr_repository}:${imageTag}`, - image_tag_branch_name: imageTagBranchName, + image_tag_before_retag: imageTagBeforeRetag, + image_ref: `${{ vars.DV_AWS_ECR_REGISTRY }}/${component.ecr_repository}:${imageTagBeforeRetag}`, + image_tag_after_retag: imageTagAfterRetag, formatted_build_args: formattedBuildArgs, }; }), @@ -160,7 +161,7 @@ jobs: PUSH_TO: ${{ inputs.push_to }} build-flavors: - name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag }}) + name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag_before_retag }}) needs: get-flavors strategy: fail-fast: ${{ inputs.fail_fast }} @@ -264,12 +265,12 @@ jobs: ${{ matrix.component.image_ref }} labels: | name=${{ matrix.component.ecr_repository }} - version=${{ matrix.component.image_tag_branch_name }} + version=${{ matrix.component.image_tag_after_retag }} org.opencontainers.image.description=Image for ${{ matrix.component.ecr_repository }} org.opencontainers.image.source=${{ github.event.repository.html_url }} org.opencontainers.image.url=${{ github.event.repository.html_url }} org.opencontainers.image.title=${{ matrix.component.ecr_repository }} - org.opencontainers.image.version=${{ matrix.component.image_tag_branch_name }} + org.opencontainers.image.version=${{ matrix.component.image_tag_after_retag }} org.opencontainers.image.created=${{ matrix.component.build_time }} org.opencontainers.image.revision=${{ github.sha }} env: @@ -427,15 +428,15 @@ jobs: run: | for component in $(jq -c '.components[]' <<< "$FLAVOR"); do repository_name=$(jq -r '.ecr_repository' <<< "$component") - image_tag=$(jq -r '.image_tag' <<< "$component") - image_tag_branch_name=$(jq -r '.image_tag_branch_name' <<< "$component") + image_tag_before_retag=$(jq -r '.image_tag_before_retag' <<< "$component") + image_tag_after_retag=$(jq -r '.image_tag_after_retag' <<< "$component") - echo "Processing repository: $repository_name, image_tag: $image_tag, image_tag_branch_name: $image_tag_branch_name" + echo "Processing repository: $repository_name, image_tag_before_retag: $image_tag_before_retag, image_tag_after_retag: $image_tag_after_retag" - IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index($var)') + IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag_before_retag" --output json | jq --arg var "${image_tag_after_retag}" '.imageDetails[0].imageTags | index($var)') if [[ -z "${IMAGE_META}" || "${IMAGE_META}" == "null" ]]; then - MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest') - aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_branch_name" --image-manifest "$MANIFEST" + MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag_before_retag" --output json | jq --raw-output --join-output '.images[0].imageManifest') + aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_after_retag" --image-manifest "$MANIFEST" else echo "Image already tagged for repository: $repository_name!" fi From 9094435ebc74e3513ab05448fe86e61ae15449e1 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Tue, 23 Dec 2025 14:07:19 +0100 Subject: [PATCH 4/5] Use mp/image_tag_suffix --- .github/workflows/build-docker-artifacts-push.yml | 2 +- .github/workflows/build-docker-artifacts-trigger-push.yml | 4 ++-- .github/workflows/build-docker-artifacts.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker-artifacts-push.yml b/.github/workflows/build-docker-artifacts-push.yml index 2bba307e..2163f718 100644 --- a/.github/workflows/build-docker-artifacts-push.yml +++ b/.github/workflows/build-docker-artifacts-push.yml @@ -33,7 +33,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "main" + WORKFLOW_BRANCH: "mp/image_tag_suffix" permissions: contents: read diff --git a/.github/workflows/build-docker-artifacts-trigger-push.yml b/.github/workflows/build-docker-artifacts-trigger-push.yml index 169187aa..78d4ff85 100644 --- a/.github/workflows/build-docker-artifacts-trigger-push.yml +++ b/.github/workflows/build-docker-artifacts-trigger-push.yml @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "main" + WORKFLOW_BRANCH: "mp/image_tag_suffix" permissions: contents: read @@ -106,6 +106,6 @@ jobs: repo: "infrastructure-k8s" github_token: ${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} workflow_file_name: "push-docker-artifact-to-customer.yml" - ref: "main" + ref: "mp/image_tag_suffix" github_user: ${{ secrets.DV_BOT_USER }} client_payload: ${{ toJson(matrix.customer) }} diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index d55b2dcd..74811c4c 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -46,7 +46,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "main" + WORKFLOW_BRANCH: "mp/image_tag_suffix" DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main" DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main" @@ -454,7 +454,7 @@ jobs: # if? When should we do this? Always? Only for certain branches? If so, how should we define that, in the config.json? # We need the always() && !cancelled() && !failure() because the test-images may have been skipped (which is fine), but this transitvely propagates through retag-images to here. See https://github.com/actions/runner/issues/491#issuecomment-1507495166 if: ${{ always() && !cancelled() && !failure() && fromJson(needs.get-flavors.outputs.result).skip_push != true && fromJson(needs.get-flavors.outputs.result).push_to != '' }} - uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@main + uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@mp/image_tag_suffix secrets: inherit with: push_to: ${{ fromJson(needs.get-flavors.outputs.result).push_to }} From 1d2c4d42948032d70beebf8cc70e06456eb7dd75 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Tue, 23 Dec 2025 14:32:33 +0100 Subject: [PATCH 5/5] Revert --- .github/workflows/build-docker-artifacts-push.yml | 2 +- .github/workflows/build-docker-artifacts-trigger-push.yml | 4 ++-- .github/workflows/build-docker-artifacts.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker-artifacts-push.yml b/.github/workflows/build-docker-artifacts-push.yml index 2163f718..2bba307e 100644 --- a/.github/workflows/build-docker-artifacts-push.yml +++ b/.github/workflows/build-docker-artifacts-push.yml @@ -33,7 +33,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "mp/image_tag_suffix" + WORKFLOW_BRANCH: "main" permissions: contents: read diff --git a/.github/workflows/build-docker-artifacts-trigger-push.yml b/.github/workflows/build-docker-artifacts-trigger-push.yml index 78d4ff85..169187aa 100644 --- a/.github/workflows/build-docker-artifacts-trigger-push.yml +++ b/.github/workflows/build-docker-artifacts-trigger-push.yml @@ -28,7 +28,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "mp/image_tag_suffix" + WORKFLOW_BRANCH: "main" permissions: contents: read @@ -106,6 +106,6 @@ jobs: repo: "infrastructure-k8s" github_token: ${{ secrets.DATAVISYN_BOT_REPO_TOKEN }} workflow_file_name: "push-docker-artifact-to-customer.yml" - ref: "mp/image_tag_suffix" + ref: "main" github_user: ${{ secrets.DV_BOT_USER }} client_payload: ${{ toJson(matrix.customer) }} diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 74811c4c..d55b2dcd 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -46,7 +46,7 @@ concurrency: cancel-in-progress: true env: - WORKFLOW_BRANCH: "mp/image_tag_suffix" + WORKFLOW_BRANCH: "main" DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main" DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main" @@ -454,7 +454,7 @@ jobs: # if? When should we do this? Always? Only for certain branches? If so, how should we define that, in the config.json? # We need the always() && !cancelled() && !failure() because the test-images may have been skipped (which is fine), but this transitvely propagates through retag-images to here. See https://github.com/actions/runner/issues/491#issuecomment-1507495166 if: ${{ always() && !cancelled() && !failure() && fromJson(needs.get-flavors.outputs.result).skip_push != true && fromJson(needs.get-flavors.outputs.result).push_to != '' }} - uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@mp/image_tag_suffix + uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@main secrets: inherit with: push_to: ${{ fromJson(needs.get-flavors.outputs.result).push_to }}