diff --git a/.github/workflows/build-docker-artifacts-config.schema.json b/.github/workflows/build-docker-artifacts-config.schema.json index 8604a918..acccae59 100644 --- a/.github/workflows/build-docker-artifacts-config.schema.json +++ b/.github/workflows/build-docker-artifacts-config.schema.json @@ -40,6 +40,14 @@ "type": "string", "description": "ECR repository to push the image to" }, + "image_tag_suffix": { + "type": "string", + "description": "Suffix to append to the image tag" + }, + "image_tag_prefix": { + "type": "string", + "description": "Prefix to prepend to the image tag" + }, "scan_high_severity": { "type": "boolean", "default": true, @@ -95,6 +103,14 @@ "type": "object", "additionalProperties": false, "properties": { + "image_tag_suffix": { + "type": "string", + "description": "Suffix to append to the image tag" + }, + "image_tag_prefix": { + "type": "string", + "description": "Prefix to prepend to the image tag" + }, "source_repository": { "type": "string", "description": "Internal ECR repository name" diff --git a/.github/workflows/build-docker-artifacts-push.yml b/.github/workflows/build-docker-artifacts-push.yml index 04c6e2c2..2bba307e 100644 --- a/.github/workflows/build-docker-artifacts-push.yml +++ b/.github/workflows/build-docker-artifacts-push.yml @@ -67,8 +67,8 @@ jobs: const filteredRepositories = customer.repositories.filter(repo => !repo.skip); // Comma separated list of source images, incl. registry, repository and tag - const sourceImages = filteredRepositories.map(repo => `${process.env.SOURCE_ECR_REGISTRY}/${repo.source_repository}:${repo.source_image_tag || process.env.IMAGE_TAG}`).join(','); - const destinationImages = filteredRepositories.map(repo => `${customer.registry}/${repo.target_repository}:${repo.target_image_tag || process.env.IMAGE_TAG}`).join(','); + const sourceImages = filteredRepositories.map(repo => `${process.env.SOURCE_ECR_REGISTRY}/${repo.source_repository}:${repo.image_tag_prefix || ''}${repo.source_image_tag || process.env.IMAGE_TAG}${repo.image_tag_suffix || ''}`).join(','); + const destinationImages = filteredRepositories.map(repo => `${customer.registry}/${repo.target_repository}:${repo.image_tag_prefix || ''}${repo.target_image_tag || process.env.IMAGE_TAG}${repo.image_tag_suffix || ''}`).join(','); const result = { customer, diff --git a/.github/workflows/build-docker-artifacts-trigger-push.yml b/.github/workflows/build-docker-artifacts-trigger-push.yml index 6006eb71..169187aa 100644 --- a/.github/workflows/build-docker-artifacts-trigger-push.yml +++ b/.github/workflows/build-docker-artifacts-trigger-push.yml @@ -75,13 +75,13 @@ jobs: const customers = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {}); - const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); + const imageTagAfterRetag = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); const result = customers.map((c) => ({ repository: process.env.REPOSITORY, customer: c, customer_json: JSON.stringify(config.push[c]), - image_tag: imageTagBranchName, + image_tag: imageTagAfterRetag, })); console.log(result); return result; diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 35840a08..d55b2dcd 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -97,8 +97,6 @@ jobs: const config = require('./deploy/build/config.json'); const buildTime = new Date().toISOString().replace(/:/g, '').replace(/\..+/, 'Z'); - const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); - const imageTag = `tagged-${imageTagBranchName}-${buildTime}`; const builds = process.env.BUILDS ? process.env.BUILDS.split(',') : Object.keys(config.build); const push_to = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {}); @@ -116,9 +114,6 @@ jobs: skip_retag: skip_push, // Add metadata to the flavor object (will be used as matrix input) build_time: buildTime, - image_tag: imageTag, - image_tag_branch_name: imageTagBranchName, - ecr_repositories: flavor.components.map(component => component.ecr_repository), test_images: { enabled: testImageEnabled, script_path: testImagesHookScript, @@ -129,6 +124,9 @@ jobs: const formattedBuildArgs = component.build_args ? Object.entries(component.build_args).map(([key, value]) => `${key}=${value}`).join('\n') : ''; + const imageTagAfterRetag = (component.image_tag_prefix || '') + "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-') + (component.image_tag_suffix || ''); + const imageTagBeforeRetag = `tagged-${imageTagAfterRetag}-${buildTime}`; + return { ...component, // We can skip the push only if we don't want to test the image below @@ -138,9 +136,9 @@ jobs: flavor_id: id, flavor_directory: `./deploy/build/${flavor.directory}`, build_time: buildTime, - image_tag: imageTag, - image_ref: `${{ vars.DV_AWS_ECR_REGISTRY }}/${component.ecr_repository}:${imageTag}`, - image_tag_branch_name: imageTagBranchName, + image_tag_before_retag: imageTagBeforeRetag, + image_ref: `${{ vars.DV_AWS_ECR_REGISTRY }}/${component.ecr_repository}:${imageTagBeforeRetag}`, + image_tag_after_retag: imageTagAfterRetag, formatted_build_args: formattedBuildArgs, }; }), @@ -163,7 +161,7 @@ jobs: PUSH_TO: ${{ inputs.push_to }} build-flavors: - name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag }}) + name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag_before_retag }}) needs: get-flavors strategy: fail-fast: ${{ inputs.fail_fast }} @@ -247,6 +245,8 @@ jobs: # Disable provenance as it creates weird multi-arch images: https://github.com/docker/build-push-action/issues/755 provenance: false # Disable the cache to avoid outdated (base) images + # TODO: are we sure we want this? We might benefit a lot from caching! But also, our base images like datavisyn/base/python:main are updated frequently and have no version tag... + # maybe no-cache-filters works? Or we do this on our self-hosted runners: https://docs.docker.com/build/ci/github-actions/cache/#local-cache no-cache: true build-args: | GIT_BRANCH=${{ steps.get-branch.outputs.branch }} @@ -265,12 +265,12 @@ jobs: ${{ matrix.component.image_ref }} labels: | name=${{ matrix.component.ecr_repository }} - version=${{ matrix.component.image_tag_branch_name }} + version=${{ matrix.component.image_tag_after_retag }} org.opencontainers.image.description=Image for ${{ matrix.component.ecr_repository }} org.opencontainers.image.source=${{ github.event.repository.html_url }} org.opencontainers.image.url=${{ github.event.repository.html_url }} org.opencontainers.image.title=${{ matrix.component.ecr_repository }} - org.opencontainers.image.version=${{ matrix.component.image_tag_branch_name }} + org.opencontainers.image.version=${{ matrix.component.image_tag_after_retag }} org.opencontainers.image.created=${{ matrix.component.build_time }} org.opencontainers.image.revision=${{ github.sha }} env: @@ -426,19 +426,19 @@ jobs: - name: Retag images shell: bash run: | - image_tag="${{ matrix.flavor.image_tag }}" - image_tag_branch_name="${{ matrix.flavor.image_tag_branch_name }}" + for component in $(jq -c '.components[]' <<< "$FLAVOR"); do + repository_name=$(jq -r '.ecr_repository' <<< "$component") + image_tag_before_retag=$(jq -r '.image_tag_before_retag' <<< "$component") + image_tag_after_retag=$(jq -r '.image_tag_after_retag' <<< "$component") - echo "image_tag=$image_tag" - echo "image_tag_branch_name=$image_tag_branch_name" + echo "Processing repository: $repository_name, image_tag_before_retag: $image_tag_before_retag, image_tag_after_retag: $image_tag_after_retag" - for repository_name in $(jq -r '.ecr_repositories[]' <<< "$FLAVOR"); do - IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index( $var )') - if [[ -z "${IMAGE_META}" || ${IMAGE_META} == "null" ]]; then - MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest') - aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_branch_name" --image-manifest "$MANIFEST" + IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag_before_retag" --output json | jq --arg var "${image_tag_after_retag}" '.imageDetails[0].imageTags | index($var)') + if [[ -z "${IMAGE_META}" || "${IMAGE_META}" == "null" ]]; then + MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag_before_retag" --output json | jq --raw-output --join-output '.images[0].imageManifest') + aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_after_retag" --image-manifest "$MANIFEST" else - echo "image already tagged!" + echo "Image already tagged for repository: $repository_name!" fi done; env: