diff --git a/.github/actions/path-filter/action.yaml b/.github/actions/path-filter/action.yaml new file mode 100644 index 00000000..1bf2f06b --- /dev/null +++ b/.github/actions/path-filter/action.yaml @@ -0,0 +1,34 @@ +name: path-filter +description: Detect changed files by provideding a list of glob patterns +inputs: + patterns: + description: A list of glob patterns to check for changes. Defaults to '*'. + default: "*" +outputs: + matched: + description: Returns if any changed files were matched by the provided glob patterns. + value: ${{ steps.check.outputs.MATCHED }} +runs: + using: composite + steps: + - name: Check for changed files + id: check + shell: bash + env: + PATTERNS: ${{ inputs.patterns }} + run: | + set -euo pipefail + + # This is cursed... we use `.[] | .` to remove quotes and any yaml formatting. + readarray -t GLOBS < <(echo -n "$PATTERNS" | yq '.[] | .') + + MATCHED="false" + for GLOB in "${GLOBS[@]}"; do + if ! git diff --exit-code --name-only "${BASE_SHA}.." -- "$GLOB"; then + # When git diff exist with an error, that means there was a diff + # for the glob. + MATCHED="true" + fi + done + + echo "MATCHED=$MATCHED" | tee -a "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c68f8e0d..82f01d98 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -11,22 +11,24 @@ on: push: branches: - main + - ci/fix-path-filter tags: - - '[0-9][0-9].[0-9]+.[0-9]+-rc[0-9]+' - - '[0-9][0-9].[0-9]+.[0-9]+' + - "[0-9][0-9].[0-9]+.[0-9]+-rc[0-9]+" + - "[0-9][0-9].[0-9]+.[0-9]+" schedule: # Run every Saturday morning: https://crontab.guru/#15_3_*_*_6 - - cron: '15 3 * * 6' - pull_request: - paths: - - '.github/workflows/build.yaml' - - 'rust-toolchain.toml' - - '.dockerignore' - - 'deploy/**' - - '.cargo/**' - - 'docker/**' - - 'Cargo.*' - - '*.rs' + - cron: "15 3 * * 6" + # pull_request: + # We cannot use path filters here, because this workflow contains a "required job". + # paths: + # - '.github/workflows/build.yaml' + # - 'rust-toolchain.toml' + # - '.dockerignore' + # - 'deploy/**' + # - '.cargo/**' + # - 'docker/**' + # - 'Cargo.*' + # - '*.rs' merge_group: env: @@ -39,8 +41,32 @@ env: CARGO_TERM_COLOR: always jobs: + # This workflow contains a "required job", and GitHub Actions isn't clever + # enough to detect that it should be skipped, and therefore pass (like they + # allow for skipping jobs in a workflow). + # Therefore, we have to move path filters/globs down to an actual job, and + # emit an output that can be used to skip irrelevant jobs. + path-filter: + name: Testing path filters + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + submodules: recursive + fetch-depth: 0 + + - name: Check for changed files + id: check + uses: stackabletech/actions/detect-changes@a1ab9bf951cf97728f0efaa58ba2105ad955e2e3 # vx.x.x + outputs: + matched: ${{ steps.check.outputs.detected }} + cargo-udeps: name: Run cargo-udeps + needs: [path-filter] + if: needs.path-filter.outputs.matched == 'true' runs-on: ubuntu-latest env: RUSTC_BOOTSTRAP: 1 @@ -74,193 +100,199 @@ jobs: - name: Run cargo-udeps run: cargo udeps --workspace --all-targets - build-container-image: - name: Build/Publish ${{ matrix.runner.arch }} Image - if: github.event_name != 'merge_group' - permissions: - id-token: write - strategy: - fail-fast: false - matrix: - runner: - - { name: "ubuntu-latest", arch: "amd64" } - - { name: "ubicloud-standard-8-arm", arch: "arm64" } - runs-on: ${{ matrix.runner.name }} - outputs: - operator-version: ${{ steps.version.outputs.OPERATOR_VERSION }} - steps: - - name: Install host dependencies - uses: awalsh128/cache-apt-pkgs-action@2c09a5e66da6c8016428a2172bd76e5e4f14bb17 # v1.5.3 - with: - packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https - version: ${{ matrix.runner.name }} + # build-container-image: + # name: Build/Publish ${{ matrix.runner.arch }} Image + # needs: [path-filter] + # if: github.event_name != 'merge_group' + # permissions: + # id-token: write + # strategy: + # fail-fast: false + # matrix: + # runner: + # - { name: "ubuntu-latest", arch: "amd64" } + # - { name: "ubicloud-standard-8-arm", arch: "arm64" } + # runs-on: ${{ matrix.runner.name }} + # outputs: + # operator-version: ${{ steps.version.outputs.OPERATOR_VERSION }} + # steps: + # - name: Install host dependencies + # uses: awalsh128/cache-apt-pkgs-action@2c09a5e66da6c8016428a2172bd76e5e4f14bb17 # v1.5.3 + # with: + # packages: protobuf-compiler krb5-user libkrb5-dev libclang-dev liblzma-dev libssl-dev pkg-config apt-transport-https + # version: ${{ matrix.runner.name }} - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - submodules: recursive + # - name: Checkout Repository + # uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + # with: + # persist-credentials: false + # submodules: recursive - - name: Update/Extract Operator Version - id: version - env: - PR_BASE_REF: ${{ github.event.pull_request.base.ref }} - PR_NUMBER: ${{ github.event.pull_request.number }} - GITHUB_EVENT_NAME: ${{ github.event_name }} - GITHUB_DEBUG: ${{ runner.debug }} - shell: bash - run: | - set -euo pipefail - [ -n "$GITHUB_DEBUG" ] && set -x + # - name: Update/Extract Operator Version + # id: version + # env: + # PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + # PR_NUMBER: ${{ github.event.pull_request.number }} + # GITHUB_EVENT_NAME: ${{ github.event_name }} + # GITHUB_DEBUG: ${{ runner.debug }} + # shell: bash + # run: | + # set -euo pipefail + # [ -n "$GITHUB_DEBUG" ] && set -x - CURRENT_VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version') + # CURRENT_VERSION=$(cargo metadata --format-version 1 --no-deps | jq -r '.packages[0].version') - if [ "$GITHUB_EVENT_NAME" == 'pull_request' ]; then - # Include a PR suffix if this workflow is triggered by a PR - if [ "$PR_BASE_REF" == 'main' ]; then - NEW_VERSION="0.0.0-pr$PR_NUMBER" - else - NEW_VERSION="$CURRENT_VERSION-pr$PR_NUMBER" - fi - else - # Just use the current version if this workflow is run on push, schedule, etc... - NEW_VERSION="$CURRENT_VERSION" - fi + # if [ "$GITHUB_EVENT_NAME" == 'pull_request' ]; then + # # Include a PR suffix if this workflow is triggered by a PR + # if [ "$PR_BASE_REF" == 'main' ]; then + # NEW_VERSION="0.0.0-pr$PR_NUMBER" + # else + # NEW_VERSION="$CURRENT_VERSION-pr$PR_NUMBER" + # fi + # else + # # Just use the current version if this workflow is run on push, schedule, etc... + # NEW_VERSION="$CURRENT_VERSION" + # fi - sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NEW_VERSION}\"/" Cargo.toml - echo "OPERATOR_VERSION=$NEW_VERSION" | tee -a "$GITHUB_OUTPUT" + # sed -i "s/version = \"${CURRENT_VERSION}\"/version = \"${NEW_VERSION}\"/" Cargo.toml + # echo "OPERATOR_VERSION=$NEW_VERSION" | tee -a "$GITHUB_OUTPUT" - - name: Install Nix - uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31.5.2 + # - name: Install Nix + # uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31.5.2 - - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} Toolchain - uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b - with: - toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} + # - name: Install Rust ${{ env.RUST_TOOLCHAIN_VERSION }} Toolchain + # uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b + # with: + # toolchain: ${{ env.RUST_TOOLCHAIN_VERSION }} - - name: Build Container Image - id: build - uses: stackabletech/actions/build-container-image@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - image-name: ${{ env.OPERATOR_NAME }} - image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} - build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }} - container-file: docker/Dockerfile + # - name: Build Container Image + # id: build + # uses: stackabletech/actions/build-container-image@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # image-name: ${{ env.OPERATOR_NAME }} + # image-index-manifest-tag: ${{ steps.version.outputs.OPERATOR_VERSION }} + # build-arguments: VERSION=${{ steps.version.outputs.OPERATOR_VERSION }} + # container-file: docker/Dockerfile - - name: Publish Container Image - uses: stackabletech/actions/publish-image@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$sdp+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ env.OPERATOR_NAME }} - image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} - source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + # - name: Publish Container Image + # uses: stackabletech/actions/publish-image@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # image-registry-uri: oci.stackable.tech + # image-registry-username: robot$sdp+github-action-build + # image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + # image-repository: sdp/${{ env.OPERATOR_NAME }} + # image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + # source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} - publish-index-manifest: - name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index - if: github.event_name != 'merge_group' - needs: - - build-container-image - permissions: - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false + # publish-index-manifest: + # name: Publish/Sign ${{ needs.build-container-image.outputs.operator-version }} Index + # if: github.event_name != 'merge_group' + # needs: + # - path-filter + # - build-container-image + # permissions: + # id-token: write + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Repository + # uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + # with: + # persist-credentials: false - - name: Publish and Sign Image Index - uses: stackabletech/actions/publish-image-index-manifest@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$sdp+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} - image-repository: sdp/${{ env.OPERATOR_NAME }} - image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} + # - name: Publish and Sign Image Index + # uses: stackabletech/actions/publish-image-index-manifest@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # image-registry-uri: oci.stackable.tech + # image-registry-username: robot$sdp+github-action-build + # image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + # image-repository: sdp/${{ env.OPERATOR_NAME }} + # image-index-manifest-tag: ${{ needs.build-container-image.outputs.operator-version }} - publish-helm-chart: - name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart - if: github.event_name != 'merge_group' - needs: - - build-container-image - permissions: - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - submodules: recursive + # publish-helm-chart: + # name: Package/Publish ${{ needs.build-container-image.outputs.operator-version }} Helm Chart + # if: github.event_name != 'merge_group' + # needs: + # - path-filter + # - build-container-image + # permissions: + # id-token: write + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Repository + # uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + # with: + # persist-credentials: false + # submodules: recursive - - name: Package, Publish, and Sign Helm Chart - uses: stackabletech/actions/publish-helm-chart@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - chart-registry-uri: oci.stackable.tech - chart-registry-username: robot$sdp-charts+github-action-build - chart-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} - chart-repository: sdp-charts - chart-directory: deploy/helm/${{ env.OPERATOR_NAME }} - chart-version: ${{ needs.build-container-image.outputs.operator-version }} - app-version: ${{ needs.build-container-image.outputs.operator-version }} + # - name: Package, Publish, and Sign Helm Chart + # uses: stackabletech/actions/publish-helm-chart@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # chart-registry-uri: oci.stackable.tech + # chart-registry-username: robot$sdp-charts+github-action-build + # chart-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_CHARTS_GITHUB_ACTION_BUILD_SECRET }} + # chart-repository: sdp-charts + # chart-directory: deploy/helm/${{ env.OPERATOR_NAME }} + # chart-version: ${{ needs.build-container-image.outputs.operator-version }} + # app-version: ${{ needs.build-container-image.outputs.operator-version }} - openshift-preflight-check: - name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} - if: github.event_name != 'merge_group' - needs: - - build-container-image - - publish-index-manifest - strategy: - fail-fast: false - matrix: - arch: - - amd64 - - arm64 - runs-on: ubuntu-latest - steps: - - name: Run OpenShift Preflight Check - uses: stackabletech/actions/run-openshift-preflight@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} - image-architecture: ${{ matrix.arch }} + # openshift-preflight-check: + # name: Run OpenShift Preflight Check for ${{ needs.build-container-image.outputs.operator-version }}-${{ matrix.arch }} + # if: github.event_name != 'merge_group' + # needs: + # - path-filter + # - build-container-image + # - publish-index-manifest + # strategy: + # fail-fast: false + # matrix: + # arch: + # - amd64 + # - arm64 + # runs-on: ubuntu-latest + # steps: + # - name: Run OpenShift Preflight Check + # uses: stackabletech/actions/run-openshift-preflight@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # image-index-uri: oci.stackable.tech/sdp/${{ env.OPERATOR_NAME }}:${{ needs.build-container-image.outputs.operator-version }} + # image-architecture: ${{ matrix.arch }} - # This job is a required check in GitHub Settings for this repository. - # It saves us having to list many required jobs, or work around dynamically - # named jobs (since there is no concept of required settings). - finished: - # WARNING: Do not change the name unless you will also be changing the - # Required Checks (in branch protections) in GitHub settings. - name: Finished Build and Publish - needs: - - cargo-udeps - - openshift-preflight-check - - publish-helm-chart - runs-on: ubuntu-latest - steps: - - run: echo "We are done here" + # # This job is a required check in GitHub Settings for this repository. + # # It saves us having to list many required jobs, or work around dynamically + # # named jobs (since there is no concept of required settings). + # finished: + # # WARNING: Do not change the name unless you will also be changing the + # # Required Checks (in branch protections) in GitHub settings. + # name: Finished Build and Publish + # needs: + # - path-filter + # - cargo-udeps + # - openshift-preflight-check + # - publish-helm-chart + # runs-on: ubuntu-latest + # steps: + # - run: echo "We are done here" - notify: - name: Failure Notification - if: (failure() || github.run_attempt > 1) && github.event_name != 'merge_group' - needs: - - build-container-image - - publish-index-manifest - - publish-helm-chart - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false + # notify: + # name: Failure Notification + # if: (failure() || github.run_attempt > 1) && github.event_name != 'merge_group' + # needs: + # - path-filter + # - build-container-image + # - publish-index-manifest + # - publish-helm-chart + # runs-on: ubuntu-latest + # steps: + # - name: Checkout Repository + # uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + # with: + # persist-credentials: false - - name: Send Notification - uses: stackabletech/actions/send-slack-notification@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 - with: - publish-helm-chart-result: ${{ needs.publish-helm-chart.result }} - publish-manifests-result: ${{ needs.publish-index-manifest.result }} - build-result: ${{ needs.build-container-image.result }} - slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} - channel-id: C07UG6JH44F # notifications-container-images - type: container-image-build + # - name: Send Notification + # uses: stackabletech/actions/send-slack-notification@29bea1b451c0c2e994bd495969286f95bf49ed6a # v0.11.0 + # with: + # publish-helm-chart-result: ${{ needs.publish-helm-chart.result }} + # publish-manifests-result: ${{ needs.publish-index-manifest.result }} + # build-result: ${{ needs.build-container-image.result }} + # slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} + # channel-id: C07UG6JH44F # notifications-container-images + # type: container-image-build diff --git a/.github/workflows/pr_pre-commit.yaml b/.github/workflows/pr_pre-commit.yaml index acbc963e..e947e42e 100644 --- a/.github/workflows/pr_pre-commit.yaml +++ b/.github/workflows/pr_pre-commit.yaml @@ -3,6 +3,8 @@ name: pre-commit on: pull_request: + paths: + - omg merge_group: env: diff --git a/docs/modules/hive/partials/supported-versions.adoc b/docs/modules/hive/partials/supported-versions.adoc index 3ec74a45..bab1d495 100644 --- a/docs/modules/hive/partials/supported-versions.adoc +++ b/docs/modules/hive/partials/supported-versions.adoc @@ -12,7 +12,10 @@ Hive 4 has known compatibility issues, especially when using it with Iceberg or Trino. The missing compatibility with Iceberg also means that Spark jobs using this storage may fail. -**Workaround:** If you encounter issues with Hive 4, use Hive 3.1.3 instead until these upstream issues are resolved. +Be aware of upgrading Hive (e.g. 4.0.0 to 4.0.1 or 4.0.1 to Hive 4.1.0), as this upgrade is not easily reversible. +Test the new version before upgrading your production workloads and take backups of your database. + +**Workaround:** If you encounter issues with Hive 4.x, use Hive 3.1.3 instead until these upstream issues are resolved. For more details, see: