From 26cb6daeb17a6d4c590009e858ef53bf1bbe562f Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Sun, 12 Oct 2025 14:40:15 +0200 Subject: [PATCH 1/2] feat: make TRIVY_CACHE_DIR configurable --- .github/workflows/build-docker-artifacts.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index d2f9a281..16a169b3 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -275,7 +275,7 @@ jobs: DOCKER_BUILD_SUMMARY: false - name: Determine trivy scan severity levels - id: set_severity + id: configure_trivy run: | if [[ "${{ github.event.inputs.scan_high_severity }}" == "false" ]] || \ [[ "${{ vars.SCAN_HIGH_SEVERITY }}" == "false" ]] || \ @@ -284,6 +284,14 @@ jobs: else echo "severity=HIGH,CRITICAL" >> "$GITHUB_OUTPUT" fi + + # If the TRIVY_CACHE_DIR is in the environment variables, we set it as output to be used in the trivy-action + if [[ -n "$TRIVY_CACHE_DIR" ]]; then + echo "cache_dir=$TRIVY_CACHE_DIR" >> "$GITHUB_OUTPUT" + else + echo "cache_dir=${{ github.workspace }}/.cache/trivy" >> "$GITHUB_OUTPUT" + fi + - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@0.33.1 with: @@ -294,9 +302,11 @@ jobs: exit-code: "1" ignore-unfixed: false vuln-type: "os,library" - severity: ${{ steps.set_severity.outputs.severity }} + severity: ${{ steps.configure_trivy.outputs.severity }} # The cache update takes quite long, so let's try to disable it for now: https://github.com/aquasecurity/trivy-action#cache cache: "false" + # Explicitly set the cache directory to make it persistent between jobs + cache-dir: ${{ steps.configure_trivy.outputs.cache_dir }} continue-on-error: false - name: Push image From c8e98a7527161562bfd0b143b7f2a6fea572d9e5 Mon Sep 17 00:00:00 2001 From: Michael Puehringer Date: Sun, 12 Oct 2025 14:49:52 +0200 Subject: [PATCH 2/2] Add logic to determine if self-hosted --- .github/actions/build-node-python/action.yml | 17 ++++++++++++++--- .github/workflows/build-docker-artifacts.yml | 2 ++ .github/workflows/build-node-python.yml | 8 -------- .github/workflows/build-node.yml | 1 - .github/workflows/build-python.yml | 1 - 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/actions/build-node-python/action.yml b/.github/actions/build-node-python/action.yml index 0132f687..086bbc39 100644 --- a/.github/actions/build-node-python/action.yml +++ b/.github/actions/build-node-python/action.yml @@ -26,7 +26,7 @@ inputs: enable_node_cache: description: "enables the yarn cache download and upload" required: false - default: "true" + default: "false" download_yarn_lock: description: "downloads the node-yarn-lock artifact (only available when build-node was called before)" required: false @@ -99,6 +99,17 @@ inputs: runs: using: "composite" steps: + - name: Determine runner configuration + id: runner_configuration + shell: bash + run: | + # If the GH_ACTIONS_SELF_HOSTED is in the environment variables, we handle things like caching differently + if [[ -n "$GH_ACTIONS_SELF_HOSTED" ]]; then + echo "self_hosted=true" >> "$GITHUB_OUTPUT" + else + echo "self_hosted=false" >> "$GITHUB_OUTPUT" + fi + # Node - uses: ./tmp/github-workflows/.github/actions/get-node-version if: inputs.enable_node == 'true' && inputs.node_version == '' @@ -185,7 +196,7 @@ runs: # Enable yarn download cache, @see https://github.com/actions/cache/tree/main/save#always-save-cache and https://github.com/actions/setup-node/issues/325 - name: Restore yarn cache uses: actions/cache/restore@v4 - if: inputs.enable_node == 'true' && inputs.enable_node_cache == 'true' + if: inputs.enable_node == 'true' && inputs.enable_node_cache == 'true' && steps.runner_configuration.outputs.self_hosted == 'false' with: # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder path: ~/.yarn/berry/cache/ @@ -290,7 +301,7 @@ runs: # Node - name: Save yarn cache uses: actions/cache/save@v4 - if: inputs.enable_node == 'true' && steps.build.outcome == 'success' && inputs.enable_node_cache == 'true' + if: inputs.enable_node == 'true' && steps.build.outcome == 'success' && inputs.enable_node_cache == 'true' && steps.runner_configuration.outputs.self_hosted == 'false' with: # This path is the global yarn cache, because for some reason the local .yarn/cache is not used. Maybe we need to set the cacheFolder, enableGlobalCache, ... options differently? @see https://yarnpkg.com/configuration/yarnrc#cacheFolder path: ~/.yarn/berry/cache/ diff --git a/.github/workflows/build-docker-artifacts.yml b/.github/workflows/build-docker-artifacts.yml index 16a169b3..997bd67e 100644 --- a/.github/workflows/build-docker-artifacts.yml +++ b/.github/workflows/build-docker-artifacts.yml @@ -243,6 +243,7 @@ jobs: # Disable provenance as it creates weird multi-arch images: https://github.com/docker/build-push-action/issues/755 provenance: false # Disable the cache to avoid outdated (base) images + # TODO: are we sure we want this? We might benefit a lot from caching! But also, our base images like datavisyn/base/python:main are updated frequently and have no version tag... no-cache: true build-args: | GIT_BRANCH=${{ steps.get-branch.outputs.branch }} @@ -276,6 +277,7 @@ jobs: - name: Determine trivy scan severity levels id: configure_trivy + shell: bash run: | if [[ "${{ github.event.inputs.scan_high_severity }}" == "false" ]] || \ [[ "${{ vars.SCAN_HIGH_SEVERITY }}" == "false" ]] || \ diff --git a/.github/workflows/build-node-python.yml b/.github/workflows/build-node-python.yml index 16b02ed2..b9250df6 100644 --- a/.github/workflows/build-node-python.yml +++ b/.github/workflows/build-node-python.yml @@ -157,8 +157,6 @@ jobs: python_version: ${{ vars.PYTHON_VERSION || inputs.python_version }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_node_bundle: ${{ inputs.node_run_webpack }} - enable_node_cache: ${{ inputs.runs_on != 'self-hosted' }} - enable_python_cache: ${{ inputs.runs_on != 'self-hosted' }} chromatic_enable: ${{ inputs.chromatic_enable }} chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} upload_yarn_lock: "node-yarn-lock-${{ inputs.branch || github.sha }}" @@ -198,8 +196,6 @@ jobs: python_version: ${{ vars.PYTHON_VERSION || inputs.python_version }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_node_bundle: ${{ inputs.node_run_webpack }} - enable_node_cache: ${{ inputs.runs_on != 'self-hosted' }} - enable_python_cache: ${{ inputs.runs_on != 'self-hosted' }} # If cypress is used, build node and python sequentially as it is avoiding the duplicate install overhead build-node-python-cypress: @@ -289,8 +285,6 @@ jobs: python_version: ${{ vars.PYTHON_VERSION || inputs.python_version }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_node_bundle: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install - enable_node_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} - enable_python_cache: ${{ inputs.cypress_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} chromatic_enable: ${{ inputs.chromatic_enable }} chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} - name: Decrypt .env.enc and /.env.enc @@ -433,8 +427,6 @@ jobs: github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_node_bundle: false # Disable the build here and call afterwards, as otherwise the yarn run env:decrypt will fail due to a missing yarn install run_playwright_browser_install: true - enable_node_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} - enable_python_cache: ${{ inputs.playwright_runs_on != 'self-hosted' && inputs.runs_on != 'self-hosted' }} chromatic_enable: false # Set to false as we run chromatic below w/ playwright integration chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} upload_yarn_lock: "node-yarn-lock-${{ inputs.branch || github.sha }}" diff --git a/.github/workflows/build-node.yml b/.github/workflows/build-node.yml index fe6338b5..8faf1764 100644 --- a/.github/workflows/build-node.yml +++ b/.github/workflows/build-node.yml @@ -73,6 +73,5 @@ jobs: npm_registry: ${{ vars.NPM_REGISTRY }} github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} run_node_bundle: ${{ inputs.node_run_webpack }} - enable_node_cache: ${{ inputs.runs_on != 'self-hosted' }} chromatic_enable: ${{ inputs.chromatic_enable }} chromatic_project_token: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} diff --git a/.github/workflows/build-python.yml b/.github/workflows/build-python.yml index 21919253..90fb80fa 100644 --- a/.github/workflows/build-python.yml +++ b/.github/workflows/build-python.yml @@ -58,4 +58,3 @@ jobs: enable_python: true github_ro_token: ${{ github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} python_version: ${{ vars.PYTHON_VERSION || inputs.python_version }} - enable_python_cache: ${{ inputs.runs_on != 'self-hosted' }}