From 772eba170a9062988ceae67e3576d466e31057dd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 1 Dec 2025 16:51:43 +0100 Subject: [PATCH 01/15] wip:ssh for gitsync --- .pre-commit-config.yaml | 16 +++---- Cargo.lock | 16 +++---- Cargo.nix | 44 +++++++++--------- Cargo.toml | 8 ++-- crate-hashes.json | 14 +++--- deploy/helm/airflow-operator/crds/crds.yaml | 27 ++++++++++- ...aml => example-airflow-gitsync-https.yaml} | 9 ++++ .../examples/example-airflow-gitsync-ssh.yaml | 19 ++++++++ .../pages/usage-guide/mounting-dags.adoc | 33 ++++++++++---- nix/sources.json | 6 +-- .../operator-binary/src/airflow_controller.rs | 3 ++ rust/operator-binary/src/crd/mod.rs | 4 ++ .../30-install-airflow-cluster.yaml.j2 | 45 ++++++++++++++----- .../mount-dags-gitsync/31-assert.yaml.j2 | 4 ++ tests/test-definition.yaml | 5 +++ 15 files changed, 180 insertions(+), 73 deletions(-) rename docs/modules/airflow/examples/{example-airflow-gitsync.yaml => example-airflow-gitsync-https.yaml} (87%) create mode 100644 docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5461297f..f88ad40f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0 + rev: v6.0.0 # 6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -16,26 +16,26 @@ repos: - repo: https://github.com/EmbarkStudios/cargo-deny # Failing to compile cargo-deny with version >0.18.3, needs rust 1.88.0 - rev: baa02b0a0c54e0578aae6bb7c7181ad00dc290af # 0.18.3 + rev: 0.18.6 # 0.18.3 hooks: - id: cargo-deny args: ["--all-features", "check", "advisories", "bans", "licenses", "sources"] - repo: https://github.com/adrienverge/yamllint - rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1 + rev: v1.37.1 # 1.37.1 hooks: - id: yamllint args: ["--strict"] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0 + rev: v0.46.0 # 0.45.0 hooks: - id: markdownlint types: [text] files: \.md(\.j2)*$ - repo: https://github.com/koalaman/shellcheck-precommit - rev: 99470f5e12208ff0fb17ab81c3c494f7620a1d8d # 0.11.0 + rev: v0.11.0 # 0.11.0 hooks: - id: shellcheck args: ["--severity=info"] @@ -44,7 +44,7 @@ repos: # If you do not, you will need to delete the cached ruff binary shown in the # error message - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 3b4bc031619cde2e0a9f3c4441ac7cc8227245a4 # 0.14.1 + rev: v0.14.7 # 0.14.1 hooks: # Run the linter. - id: ruff-check @@ -52,12 +52,12 @@ repos: - id: ruff-format - repo: https://github.com/rhysd/actionlint - rev: e7d448ef7507c20fc4c88a95d0c448b848cd6127 # 1.7.8 + rev: v1.7.9 # 1.7.8 hooks: - id: actionlint - repo: https://github.com/hadolint/hadolint - rev: 57e1618d78fd469a92c1e584e8c9313024656623 # 2.14.0 + rev: v2.14.0 # 2.14.0 hooks: - id: hadolint diff --git a/Cargo.lock b/Cargo.lock index 12987668..e6318618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "regex", @@ -2481,8 +2481,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.100.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +version = "0.100.3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "clap", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "proc-macro2", @@ -2531,7 +2531,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "k8s-openapi", @@ -2548,7 +2548,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "axum", "clap", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "schemars", "serde", @@ -2585,7 +2585,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index e04eb219..e06f37de 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4187,9 +4187,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "k8s_version"; authors = [ @@ -8089,13 +8089,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.100.1"; + version = "0.100.3"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_operator"; authors = [ @@ -8262,9 +8262,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8297,9 +8297,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_shared"; authors = [ @@ -8379,9 +8379,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_telemetry"; authors = [ @@ -8489,9 +8489,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_versioned"; authors = [ @@ -8533,9 +8533,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/Cargo.toml b/Cargo.toml index 8945301c..be57cf89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" } anyhow = "1.0" base64 = "0.22" @@ -32,6 +32,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -# [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/gitsync-ssh" } +#stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index 1c58f755..576d5189 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#k8s-version@0.1.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator-derive@0.3.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator@0.100.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-shared@0.0.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-telemetry@0.6.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned-macros@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index b8e45ff8..e728bcd2 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -680,6 +680,7 @@ spec: The referenced Secret must include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the git-sync [documentation]. + This cannot be provided if `ssh_secret` is also provided. [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual nullable: true @@ -711,9 +712,19 @@ spec: [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example type: object repo: - description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.' + description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string + sshSecret: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + This cannot be provided if `credentials_secret` is also provided. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string wait: default: 20s description: |- @@ -1602,6 +1613,20 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array schedulers: description: |- The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database. diff --git a/docs/modules/airflow/examples/example-airflow-gitsync.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml similarity index 87% rename from docs/modules/airflow/examples/example-airflow-gitsync.yaml rename to docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 6a4b5e46..49cabcfe 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -24,3 +24,12 @@ spec: --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11> webservers: ... +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-credentials # <8> +type: Opaque +data: + user: c3Rh... + password: Z2l0a... diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml new file mode 100644 index 00000000..9ed7d7de --- /dev/null +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: +... + dagsGitSync: + - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> + sshSecret: git-sync-ssh # <2> +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-sync-ssh # <2> +type: Opaque +data: + key: LS0tL... + knownHosts: Z2l0a... diff --git a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc index 4c1f3e1c..8b7aa8ff 100644 --- a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc +++ b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc @@ -1,6 +1,6 @@ = Mounting DAGs :description: Mount DAGs in Airflow via ConfigMap for single DAGs or use git-sync for multiple DAGs. git-sync pulls from a Git repo and handles updates automatically. -:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.1 +:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.4 DAGs can be mounted by using a ConfigMap or `git-sync`. This is best illustrated with an example of each, shown in the sections below. @@ -24,21 +24,31 @@ include::example$example-airflow-dags-configmap.yaml[] <7> The resource has to be defined using `subPath`: this is to prevent the versioning of ConfigMap elements which may cause a conflict with how Airflow propagates DAGs between its components. <8> If the mount path described above is anything other than the standard location (the default is `$AIRFLOW_HOME/dags`), then the location should be defined using the relevant environment variable. -WARNING: If a DAG mounted via ConfigMap consists of modularized files then using the standard location is mandatory as Python uses this as a "root" directory when looking for referenced files. +[WARNING] +-- +If a DAG mounted via ConfigMap consists of modularized files, Python uses this as a "root" directory when looking for referenced files. +If this is the case, then either the standard DAGs location should be used, or `PYTHONPATH` should be overriden to point to the new location (it is also necessary to include the logging configuration in the path) as shown below: +[source,yaml] +---- + envOverrides: &envOverrides + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + PYTHONPATH: "/stackable/app/log_config:/dags" +---- +-- The advantage of this approach is that DAGs are provided "in-line". However, handling multiple DAGs this way becomes cumbersome, as each must be mapped individually. -For multiple DAGs, it is easier to expose them via a mounted volume, as shown below. +For multiple DAGs, it is easier to expose them via `gitsync`, as shown below. == Via `git-sync` {git-sync}[git-sync] is a command that pulls a git repository into a local directory and is supplied as a sidecar container for use within Kubernetes. The Stackable Airflow images already ship with git-sync included, and the operator takes care of calling the tool and mounting volumes, so that only the repository and synchronization details are required: -.git-sync usage example +.git-sync usage example: https [source,yaml] ---- -include::example$example-airflow-gitsync.yaml[] +include::example$example-airflow-gitsync-https.yaml[] ---- <1> A Secret used for accessing database and admin user details (included here to illustrate where different credential secrets are defined) @@ -60,10 +70,17 @@ include::example$example-airflow-gitsync.yaml[] <11> Git-sync settings can be provided inline, although some of these (`--dest`, `--root`) are specified internally in the operator and are ignored if provided by the user. Git-config settings can also be specified, although a warning is logged if `safe.directory` is specified as this is defined internally, and should not be defined by the user. +.git-sync usage example: ssh +[source,yaml] +---- +include::example$example-airflow-gitsync-ssh.yaml[] +---- + +<1> The name of the Secret used to access the repository if it is not public. + This should include two fields: `key` and `knownHosts`, both of which can contain multiple entries. +<2> The secret referenced above. -IMPORTANT: The example shows a _list_ of git-sync definitions, with a single element. -This is to avoid breaking-changes in future releases. -Currently, only one such git-sync definition is considered and processed. +IMPORTANT: Gitsync will not allow both `credentialsSecret` and `sshSecret` to be declared and the operator will throw an exception if this is attempted. NOTE: git-sync can be used with DAGs that make use of Python modules, as Python is configured to use the git-sync target folder as the "root" location when looking for referenced files. See the xref:usage-guide/applying-custom-resources.adoc[] example for more details. diff --git a/nix/sources.json b/nix/sources.json index f79c2cf0..dcfe59fa 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -29,10 +29,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a7fc11be66bdfb5cdde611ee5ce381c183da8386", - "sha256": "0h3gvjbrlkvxhbxpy01n603ixv0pjy19n9kf73rdkchdvqcn70j2", + "rev": "5c46f3bd98147c8d82366df95bbef2cab3a967ea", + "sha256": "1q085irshssrpnryw9kixd0smc2c0rhqs20niklybaywpv2zlywx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/a7fc11be66bdfb5cdde611ee5ce381c183da8386.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/5c46f3bd98147c8d82366df95bbef2cab3a967ea.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index e9e17d34..5f161213 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -436,6 +436,7 @@ pub async fn reconcile_airflow( AIRFLOW_CONTROLLER_NAME, &airflow.object_ref(&()), ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), + airflow.spec.object_overrides.clone(), ) .context(CreateClusterResourcesSnafu)?; @@ -1452,6 +1453,8 @@ fn add_git_sync_resources( } pb.add_volumes(git_sync_resources.git_content_volumes.to_owned()) .context(AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ssh_volumes.to_owned()) + .context(AddVolumeSnafu)?; cb.add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) .context(AddVolumeMountSnafu)?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index aa1a234f..a7529513 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -20,6 +20,7 @@ use stackable_operator::{ merge::Merge, }, crd::git_sync, + deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, @@ -198,6 +199,9 @@ pub mod versioned { // no doc string - See ProductImage struct pub image: ProductImage, + #[serde(default)] + pub object_overrides: ObjectOverrides, + /// Configuration that applies to all roles and role groups. /// This includes settings for authentication, git sync, service exposition and volumes, among other things. pub cluster_config: v1alpha1::AirflowClusterConfig, diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 7d5d098f..d3bfa1ca 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -3,23 +3,20 @@ kind: TestStep metadata: name: install-airflow timeout: 480 +{% if test_scenario['values']['access'] == 'ssh' %} --- apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: git-sync-ssh type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +data: + # This is a combination of a private key and known hosts (github.com) for the owner of the repo (stackable-airflow/dags). + # Contact github users @razvan or @adwk67 for details. + key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBRFdmd0hlaHhaQXdqcEk0OXN0eEM2SmJNWVJLODEvRVdueFh2NnV4aUhLUUFBQUpCSUEzTWJTQU56Ckd3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQURXZndIZWh4WkF3anBJNDlzdHhDNkpiTVlSSzgxL0VXbnhYdjZ1eGlIS1EKQUFBRUFRcVZSOW9hNi91MzZTeC9DSFRxbExMa3A2dmVnc2dha3d1RHRJNFdJcHRnTlovQWQ2SEZrRENPa2pqMnkzRUxvbApzeGhFcnpYOFJhZkZlL3E3R0ljcEFBQUFEV2RwZEMxemVXNWpMWFJsYzNRPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K + knownHosts: Z2l0aHViLmNvbSBzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNqN25kTnhRb3dnY1FuanNoY0xycVBFaWlwaG50K1ZUVHZEUDZtSEJMOWoxYU5Va1k0VWUxZ3Z3bkdMVmxPaEdlWXJuWmFNZ1JLNitQS0NVWGFEYkM3cXRiVzhnSWtoTDdhR0NzT3IvQzU2U0pNeS9CQ1pmeGQxbld6QU94U0RQZ1ZzbWVyT0JZZk5xbHRWOS9oV0NxQnl3SU5JUis1ZElnNkpUSjcycGNFcEVqY1lnWGtFMllFRlhWMUpIbnNLZ2JMV05saFNjcWIyVW15UmtReXl0Ukx0TCszOFRHeGt4Q2ZsbU8rNVo4Q1NTTlk3R2lkak1JWjdRNHpNakEybjFuR3JsVERrendEQ3N3K3dxRlBHUUExNzljbmZHV09XUlZydWoxNno2WHl2eHZqSndiejB3UVo3NVhLNXRLU2I3Rk55ZUlFczRUVDRqaytTNGRoUGVBVUM1eStiRFlpcllnTTRHQzd1RW56dG5aeWFWV1E3QjM4MUFLNFFkcnd0NTFacUV4S2JRcFRVTm4rRWpxb1R3dnFOajRrcXg1UVVDSTBUaFMvWWtPeEpDWG1QVVdaYmhqcENnNTZpKzJhQjZDbUsySkdobjU3SzVtajBNTmRCWEE0L1dud0g2WG9QV0p6SzVOeXUyekIzbkFacCtTNWhwUXMrcDF2TjEvd3Nqaz0KZ2l0aHViLmNvbSBlY2RzYS1zaGEyLW5pc3RwMjU2IEFBQUFFMlZqWkhOaExYTm9ZVEl0Ym1semRIQXlOVFlBQUFBSWJtbHpkSEF5TlRZQUFBQkJCRW1LU0VOalFFZXpPbXhrWk15N29wS2d3RkI5bmt0NVlScllNak51RzVOODd1UmdnNkNMcmJvNXdBZFQveTZ2MG1LVjBVMncwV1oyWUIvKytUcG9ja2c9CmdpdGh1Yi5jb20gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9NcXFua1Z6cm0wU2RHNlVPb3FLTHNhYmdINUM5b2tXaTBkaDJsOUdLSmwK {% endif %} +{% if test_scenario['values']['access'] == 'https' %} --- apiVersion: v1 kind: Secret @@ -32,6 +29,24 @@ data: # This token doesn't expire. user: c3RhY2thYmxlLWFpcmZsb3c= password: Z2l0aHViX3BhdF8xMUJLUURCRVkwSk1EWlNVQk1RYTdoX0c2OGlhbWtpRkpFV1RMTTF0ajFwbHFTVFNyZ3p3dHZneXI5b2tubGRXaGpVRDZITFRFV0JJcm9yT0dXCg== +{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +{% if test_scenario['values']['executor'] == 'celery' %} + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +{% endif %} --- apiVersion: v1 kind: ConfigMap @@ -60,13 +75,19 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials dagsGitSync: +{% if test_scenario['values']['access'] == 'ssh' %} + - repo: ssh://git@github.com/stackable-airflow/dags.git + sshSecret: git-sync-ssh +{% endif %} +{% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags + credentialsSecret: git-credentials +{% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) branch: main wait: 5s {% endif %} - credentialsSecret: git-credentials gitSyncConf: # supply some config to check that safe.directory is correctly set --git-config: http.sslVerify:false diff --git a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 index 8c7e90a8..cbe76bf8 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 @@ -11,8 +11,10 @@ commands: # will expect 2 (two containers, base and gitsync) - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "AIRFLOW_TEST_VAR" | wc -l | grep 2 # will expect 1 (one container, gitsync) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_USERNAME" | wc -l | grep 1 - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_PASSWORD" | wc -l | grep 1 +{% endif %} {% else %} # check that the statefulset contains mounts and envs # will expect 6 (2 from from the volume declaration + mounts to 3 containers, base and 2 gitsyncs, plus configmap restarter) @@ -20,6 +22,8 @@ commands: # will expect 3 (two containers, base and gitsync-1, and one initContainer gitsync-0) - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 3 # will expect 2 (one container, gitsync-1, and one initContainer gitsync-0) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 2 - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 2 {% endif %} +{% endif %} diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 39c632b3..367ddeae 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -33,6 +33,10 @@ dimensions: values: - celery - kubernetes + - name: access + values: + - https + - ssh tests: - name: smoke dimensions: @@ -49,6 +53,7 @@ tests: - airflow-latest - openshift - executor + - access - name: ldap dimensions: - airflow-latest From 2beef1d70f4953031c6d35210dd565fa761feff3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 2 Dec 2025 10:25:08 +0100 Subject: [PATCH 02/15] linting, use deploy key --- .../modules/airflow/examples/example-airflow-gitsync-ssh.yaml | 4 +++- .../mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml index 9ed7d7de..52c0a0f9 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -4,10 +4,12 @@ kind: AirflowCluster metadata: name: airflow spec: -... + clusterConfig: dagsGitSync: - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> sshSecret: git-sync-ssh # <2> +... + --- apiVersion: v1 kind: Secret diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index d3bfa1ca..3897ad62 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -11,9 +11,9 @@ metadata: name: git-sync-ssh type: Opaque data: - # This is a combination of a private key and known hosts (github.com) for the owner of the repo (stackable-airflow/dags). + # This is a combination of a read-only deploy key and known hosts (github.com) for the repo (stackable-airflow/dags). # Contact github users @razvan or @adwk67 for details. - key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBRFdmd0hlaHhaQXdqcEk0OXN0eEM2SmJNWVJLODEvRVdueFh2NnV4aUhLUUFBQUpCSUEzTWJTQU56Ckd3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQURXZndIZWh4WkF3anBJNDlzdHhDNkpiTVlSSzgxL0VXbnhYdjZ1eGlIS1EKQUFBRUFRcVZSOW9hNi91MzZTeC9DSFRxbExMa3A2dmVnc2dha3d1RHRJNFdJcHRnTlovQWQ2SEZrRENPa2pqMnkzRUxvbApzeGhFcnpYOFJhZkZlL3E3R0ljcEFBQUFEV2RwZEMxemVXNWpMWFJsYzNRPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K + key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNDTUpoLzdaUDFUWDBHem5sZG0zV3p3TW9Ed0xTVzh4dm9SUmpIZGVYTVViQUFBQUtDc3QzQ1NyTGR3CmtnQUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQ01KaC83WlAxVFgwR3pubGRtM1d6d01vRHdMU1c4eHZvUlJqSGRlWE1VYkEKQUFBRUQwSUtCOG1wMWlScXFVSlVjTDh5Y0tKWkRwZkRnYWRXUHVhcUxYQkkvTVY0d21IL3RrL1ZOZlFiT2VWMmJkYlBBeQpnUEF0SmJ6RytoRkdNZDE1Y3hSc0FBQUFGbVJsY0d4dmVTMXJaWGt0Wm05eUxXZHBkSE41Ym1NQkFnTUVCUVlICi0tLS0tRU5EIE9QRU5TU0ggUFJJVkFURSBLRVktLS0tLQo= knownHosts: Z2l0aHViLmNvbSBzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNqN25kTnhRb3dnY1FuanNoY0xycVBFaWlwaG50K1ZUVHZEUDZtSEJMOWoxYU5Va1k0VWUxZ3Z3bkdMVmxPaEdlWXJuWmFNZ1JLNitQS0NVWGFEYkM3cXRiVzhnSWtoTDdhR0NzT3IvQzU2U0pNeS9CQ1pmeGQxbld6QU94U0RQZ1ZzbWVyT0JZZk5xbHRWOS9oV0NxQnl3SU5JUis1ZElnNkpUSjcycGNFcEVqY1lnWGtFMllFRlhWMUpIbnNLZ2JMV05saFNjcWIyVW15UmtReXl0Ukx0TCszOFRHeGt4Q2ZsbU8rNVo4Q1NTTlk3R2lkak1JWjdRNHpNakEybjFuR3JsVERrendEQ3N3K3dxRlBHUUExNzljbmZHV09XUlZydWoxNno2WHl2eHZqSndiejB3UVo3NVhLNXRLU2I3Rk55ZUlFczRUVDRqaytTNGRoUGVBVUM1eStiRFlpcllnTTRHQzd1RW56dG5aeWFWV1E3QjM4MUFLNFFkcnd0NTFacUV4S2JRcFRVTm4rRWpxb1R3dnFOajRrcXg1UVVDSTBUaFMvWWtPeEpDWG1QVVdaYmhqcENnNTZpKzJhQjZDbUsySkdobjU3SzVtajBNTmRCWEE0L1dud0g2WG9QV0p6SzVOeXUyekIzbkFacCtTNWhwUXMrcDF2TjEvd3Nqaz0KZ2l0aHViLmNvbSBlY2RzYS1zaGEyLW5pc3RwMjU2IEFBQUFFMlZqWkhOaExYTm9ZVEl0Ym1semRIQXlOVFlBQUFBSWJtbHpkSEF5TlRZQUFBQkJCRW1LU0VOalFFZXpPbXhrWk15N29wS2d3RkI5bmt0NVlScllNak51RzVOODd1UmdnNkNMcmJvNXdBZFQveTZ2MG1LVjBVMncwV1oyWUIvKytUcG9ja2c9CmdpdGh1Yi5jb20gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9NcXFua1Z6cm0wU2RHNlVPb3FLTHNhYmdINUM5b2tXaTBkaDJsOUdLSmwK {% endif %} {% if test_scenario['values']['access'] == 'https' %} From 5c6881e2e9b65881d0ece59666e5cd7605e350f1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 11:25:15 +0100 Subject: [PATCH 03/15] change credentials fields in line with operator-rs --- deploy/helm/airflow-operator/crds/crds.yaml | 101 +++++++----------- .../example-airflow-gitsync-https.yaml | 3 +- .../examples/example-airflow-gitsync-ssh.yaml | 3 +- .../pages/usage-guide/mounting-dags.adoc | 2 - .../30-install-airflow-cluster.yaml.j2 | 6 +- 5 files changed, 45 insertions(+), 70 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index e728bcd2..3dd59cbc 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,12 +115,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -143,7 +142,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -163,7 +161,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -385,12 +382,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -413,7 +409,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -433,7 +428,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -673,18 +667,34 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentialsSecret: - description: |- - The name of the Secret used to access the repository if it is not public. + credentials: + anyOf: + - required: + - basicAuthSecretName + - required: + - sshPrivateKeySecretName + description: An optional secret used for git access. + nullable: true + properties: + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - This cannot be provided if `ssh_secret` is also provided. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + type: object depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -715,16 +725,6 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string - sshSecret: - description: |- - The name of the Secret used for SSH access to the repository. - - The referenced Secret must include two fields: `key` and `knownHosts`. - This cannot be provided if `credentials_secret` is also provided. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string wait: default: 20s description: |- @@ -890,12 +890,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -918,7 +917,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -938,7 +936,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1160,12 +1157,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1188,7 +1184,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1208,7 +1203,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1458,12 +1452,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1486,7 +1479,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1506,7 +1498,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1707,12 +1698,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1735,7 +1725,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1755,7 +1744,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1977,12 +1965,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2005,7 +1992,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2025,7 +2011,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2220,12 +2205,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2248,7 +2232,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2268,7 +2251,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2490,12 +2472,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2518,7 +2499,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2538,7 +2518,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2733,12 +2712,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2761,7 +2739,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2781,7 +2758,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3008,12 +2984,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3036,7 +3011,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3056,7 +3030,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 49cabcfe..bfc5be08 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -16,7 +16,8 @@ spec: gitFolder: "tests/templates/kuttl/mount-dags-gitsync/dags" # <5> depth: 10 # <6> wait: 20s # <7> - credentialsSecret: git-credentials # <8> + credentials: + basicAuthSecretName: git-credentials # <8> gitSyncConf: # <9> --rev: HEAD # <10> # --rev: git-sync-tag # N.B. tag must be covered by "depth" (the number of commits to clone) diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml index 52c0a0f9..f415f7c4 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -7,7 +7,8 @@ spec: clusterConfig: dagsGitSync: - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> - sshSecret: git-sync-ssh # <2> + credentials: + sshPrivateKeySecretName: git-sync-ssh # <2> ... --- diff --git a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc index 8b7aa8ff..cd03fc3b 100644 --- a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc +++ b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc @@ -80,7 +80,5 @@ include::example$example-airflow-gitsync-ssh.yaml[] This should include two fields: `key` and `knownHosts`, both of which can contain multiple entries. <2> The secret referenced above. -IMPORTANT: Gitsync will not allow both `credentialsSecret` and `sshSecret` to be declared and the operator will throw an exception if this is attempted. - NOTE: git-sync can be used with DAGs that make use of Python modules, as Python is configured to use the git-sync target folder as the "root" location when looking for referenced files. See the xref:usage-guide/applying-custom-resources.adoc[] example for more details. diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 3897ad62..7b2b888b 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -77,11 +77,13 @@ spec: dagsGitSync: {% if test_scenario['values']['access'] == 'ssh' %} - repo: ssh://git@github.com/stackable-airflow/dags.git - sshSecret: git-sync-ssh + credentials: + sshPrivateKeySecretName: git-sync-ssh {% endif %} {% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags - credentialsSecret: git-credentials + credentials: + basicAuthSecretName: git-credentials {% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) From 436abeb9cee55ead70abf941ea926a1f7c25fe2c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 11:33:31 +0100 Subject: [PATCH 04/15] regenerate charts --- deploy/helm/airflow-operator/crds/crds.yaml | 101 +++++++++++++------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 3dd59cbc..e728bcd2 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,11 +115,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -142,6 +143,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -161,6 +163,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -382,11 +385,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -409,6 +413,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -428,6 +433,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -667,34 +673,18 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentials: - anyOf: - - required: - - basicAuthSecretName - - required: - - sshPrivateKeySecretName - description: An optional secret used for git access. - nullable: true - properties: - basicAuthSecretName: - description: |- - The name of the Secret used to access the repository via Basic Authentication if it is not public. - - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - sshPrivateKeySecretName: - description: |- - The name of the Secret used for SSH access to the repository. + credentialsSecret: + description: |- + The name of the Secret used to access the repository if it is not public. - The referenced Secret must include two fields: `key` and `knownHosts`. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. + This cannot be provided if `ssh_secret` is also provided. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - type: object + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -725,6 +715,16 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string + sshSecret: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + This cannot be provided if `credentials_secret` is also provided. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string wait: default: 20s description: |- @@ -890,11 +890,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -917,6 +918,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -936,6 +938,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1157,11 +1160,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1184,6 +1188,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1203,6 +1208,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1452,11 +1458,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1479,6 +1486,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1498,6 +1506,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1698,11 +1707,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1725,6 +1735,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1744,6 +1755,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1965,11 +1977,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1992,6 +2005,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2011,6 +2025,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2205,11 +2220,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2232,6 +2248,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2251,6 +2268,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2472,11 +2490,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2499,6 +2518,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2518,6 +2538,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2712,11 +2733,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2739,6 +2761,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2758,6 +2781,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2984,11 +3008,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3011,6 +3036,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -3030,6 +3056,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object From e5a910ce41c0a85e8319e26ff2462accc1f8ddf0 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 12:21:20 +0100 Subject: [PATCH 05/15] bring enum in line with framework definition --- deploy/helm/airflow-operator/crds/crds.yaml | 111 +++++++++----------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index e728bcd2..f3352ae3 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,12 +115,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -143,7 +142,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -163,7 +161,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -385,12 +382,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -413,7 +409,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -433,7 +428,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -673,18 +667,44 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentialsSecret: - description: |- - The name of the Secret used to access the repository if it is not public. + credentials: + description: An optional secret used for git access. + nullable: true + oneOf: + - required: + - basicAuth + - required: + - ssh + properties: + basicAuth: + properties: + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - This cannot be provided if `ssh_secret` is also provided. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + required: + - basicAuthSecretName + type: object + ssh: + properties: + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + required: + - sshPrivateKeySecretName + type: object + type: object depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -715,16 +735,6 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string - sshSecret: - description: |- - The name of the Secret used for SSH access to the repository. - - The referenced Secret must include two fields: `key` and `knownHosts`. - This cannot be provided if `credentials_secret` is also provided. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string wait: default: 20s description: |- @@ -890,12 +900,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -918,7 +927,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -938,7 +946,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1160,12 +1167,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1188,7 +1194,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1208,7 +1213,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1458,12 +1462,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1486,7 +1489,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1506,7 +1508,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1707,12 +1708,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1735,7 +1735,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1755,7 +1754,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1977,12 +1975,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2005,7 +2002,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2025,7 +2021,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2220,12 +2215,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2248,7 +2242,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2268,7 +2261,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2490,12 +2482,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2518,7 +2509,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2538,7 +2528,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2733,12 +2722,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2761,7 +2749,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2781,7 +2768,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3008,12 +2994,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3036,7 +3021,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3056,7 +3040,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object From 576baf9c26d0b04d54ac481caf40855f33fad93d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 12:50:27 +0100 Subject: [PATCH 06/15] bring enum in line with framework definition: correction --- deploy/helm/airflow-operator/crds/crds.yaml | 42 ++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index f3352ae3..76c505e3 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -672,38 +672,28 @@ spec: nullable: true oneOf: - required: - - basicAuth + - basicAuthSecretName - required: - - ssh + - sshPrivateKeySecretName properties: - basicAuth: - properties: - basicAuthSecretName: - description: |- - The name of the Secret used to access the repository via Basic Authentication if it is not public. + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - required: - - basicAuthSecretName - type: object - ssh: - properties: - sshPrivateKeySecretName: - description: |- - The name of the Secret used for SSH access to the repository. + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. - The referenced Secret must include two fields: `key` and `knownHosts`. + The referenced Secret must include two fields: `key` and `knownHosts`. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - required: - - sshPrivateKeySecretName - type: object + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string type: object depth: default: 1 From 5e4d83879db551982c18e054af92aac24b6c771b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 14:08:08 +0100 Subject: [PATCH 07/15] missed files --- Cargo.lock | 29 +++++++++------------- Cargo.nix | 63 +++++++++++++++++++++++++++++++++-------------- crate-hashes.json | 19 ++++++++------ 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6318618..188416e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "darling", "regex", @@ -1348,8 +1348,7 @@ dependencies = [ [[package]] name = "kube" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e7bb0b6a46502cc20e4575b6ff401af45cfea150b34ba272a3410b78aa014e" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "k8s-openapi", "kube-client", @@ -1361,8 +1360,7 @@ dependencies = [ [[package]] name = "kube-client" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4987d57a184d2b5294fdad3d7fc7f278899469d21a4da39a8f6ca16426567a36" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "base64", "bytes", @@ -1397,8 +1395,7 @@ dependencies = [ [[package]] name = "kube-core" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914bbb770e7bb721a06e3538c0edd2babed46447d128f7c21caa68747060ee73" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "chrono", "derive_more", @@ -1416,8 +1413,7 @@ dependencies = [ [[package]] name = "kube-derive" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dee8252be137772a6ab3508b81cd797dee62ee771112a2453bc85cbbe150d2" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "darling", "proc-macro2", @@ -1430,8 +1426,7 @@ dependencies = [ [[package]] name = "kube-runtime" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea4de4b562c5cc89ab10300bb63474ae1fa57ff5a19275f2e26401a323e3fd" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "ahash", "async-broadcast", @@ -2482,7 +2477,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.100.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "chrono", "clap", @@ -2520,7 +2515,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "darling", "proc-macro2", @@ -2531,7 +2526,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "chrono", "k8s-openapi", @@ -2548,7 +2543,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "axum", "clap", @@ -2572,7 +2567,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "schemars", "serde", @@ -2585,7 +2580,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index e06f37de..9aa43a86 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4188,8 +4188,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "k8s_version"; authors = [ @@ -4220,7 +4220,12 @@ rec { crateName = "kube"; version = "2.0.1"; edition = "2024"; - sha256 = "0kh1m9w0nhd3fai4pcshl7z5rx0s83zvcxa51v12ql26d85vprs8"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4293,7 +4298,12 @@ rec { crateName = "kube-client"; version = "2.0.1"; edition = "2024"; - sha256 = "0dksaqk698bciyda6k8ss9lr92bqyb3pygddzna54asd31xdb1s9"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_client"; authors = [ "clux " @@ -4526,7 +4536,12 @@ rec { crateName = "kube-core"; version = "2.0.1"; edition = "2024"; - sha256 = "0wzfc1q78s5a3k1gfa6i8xjd9gmssbnw0f1mdsh23dvv1rvvnjwi"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_core"; authors = [ "clux " @@ -4608,7 +4623,12 @@ rec { crateName = "kube-derive"; version = "2.0.1"; edition = "2024"; - sha256 = "1ljhw6xmrj1v8ni144bpxrifwzbrrn0qnl5kd8m7fdz15cjyiph3"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; procMacro = true; libName = "kube_derive"; authors = [ @@ -4657,7 +4677,12 @@ rec { crateName = "kube-runtime"; version = "2.0.1"; edition = "2024"; - sha256 = "1zg34fih2r72y9sr58gmgyjizbkl6jv0nc0hmf4wrib2npj4vska"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_runtime"; authors = [ "clux " @@ -8094,8 +8119,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_operator"; authors = [ @@ -8263,8 +8288,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8298,8 +8323,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_shared"; authors = [ @@ -8380,8 +8405,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_telemetry"; authors = [ @@ -8490,8 +8515,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_versioned"; authors = [ @@ -8534,8 +8559,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/crate-hashes.json b/crate-hashes.json index 576d5189..79f6e88b 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,15 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-runtime@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file From 9c284ad59cccb9c78051785adf50bb559dac49f4 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 19 Dec 2025 10:55:27 +0100 Subject: [PATCH 08/15] switch gitsync to v2 --- Cargo.lock | 14 +++++----- Cargo.nix | 28 +++++++++---------- crate-hashes.json | 14 +++++----- .../operator-binary/src/airflow_controller.rs | 12 ++++---- rust/operator-binary/src/crd/mod.rs | 2 +- rust/operator-binary/src/env_vars.rs | 8 +++--- 6 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 188416e2..3f834c3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "darling", "regex", @@ -2477,7 +2477,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.100.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "chrono", "clap", @@ -2515,7 +2515,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "darling", "proc-macro2", @@ -2526,7 +2526,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "chrono", "k8s-openapi", @@ -2543,7 +2543,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "axum", "clap", @@ -2567,7 +2567,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "schemars", "serde", @@ -2580,7 +2580,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index 9aa43a86..fcfe6b8c 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4188,8 +4188,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "k8s_version"; authors = [ @@ -8119,8 +8119,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_operator"; authors = [ @@ -8288,8 +8288,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8323,8 +8323,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_shared"; authors = [ @@ -8405,8 +8405,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_telemetry"; authors = [ @@ -8515,8 +8515,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_versioned"; authors = [ @@ -8559,8 +8559,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; - sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/crate-hashes.json b/crate-hashes.json index 79f6e88b..299e3d15 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,11 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 5f161213..432c2648 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -224,7 +224,7 @@ pub enum Error { }, #[snafu(display("invalid git-sync specification"))] - InvalidGitSyncSpec { source: git_sync::v1alpha1::Error }, + InvalidGitSyncSpec { source: git_sync::v1alpha2::Error }, #[snafu(display("failed to create cluster resources"))] CreateClusterResources { @@ -530,7 +530,7 @@ pub async fn reconcile_airflow( .merged_config(&airflow_role, &rolegroup) .context(FailedToResolveConfigSnafu)?; - let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new( + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, &resolved_product_image, &env_vars_from_rolegroup_config(rolegroup_config), @@ -717,7 +717,7 @@ async fn build_executor_template( rolegroup: rolegroup.clone(), })?; - let git_sync_resources = git_sync::v1alpha1::GitSyncResources::new( + let git_sync_resources = git_sync::v1alpha2::GitSyncResources::new( &airflow.spec.cluster_config.dags_git_sync, resolved_product_image, &env_vars_from(&common_config.env_overrides), @@ -934,7 +934,7 @@ fn build_server_rolegroup_statefulset( service_account: &ServiceAccount, merged_airflow_config: &AirflowConfig, executor: &AirflowExecutor, - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { let binding = airflow.get_role(airflow_role); let role = binding.as_ref().context(NoAirflowRoleSnafu)?; @@ -1255,7 +1255,7 @@ fn build_executor_template_config_map( env_overrides: &HashMap, pod_overrides: &PodTemplateSpec, rolegroup_ref: &RoleGroupRef, - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { let mut pb = PodBuilder::new(); let pb_metadata = ObjectMetaBuilder::new() @@ -1437,7 +1437,7 @@ fn add_authentication_volumes_and_volume_mounts( fn add_git_sync_resources( pb: &mut PodBuilder, cb: &mut ContainerBuilder, - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, add_sidecar_containers: bool, add_init_containers: bool, ) -> Result<()> { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a7529513..41639513 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -252,7 +252,7 @@ pub mod versioned { /// Learn more in the /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). #[serde(default)] - pub dags_git_sync: Vec, + pub dags_git_sync: Vec, /// for internal use only - not for production use. #[serde(default)] diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index b04144bc..d2a75921 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -80,7 +80,7 @@ pub fn build_airflow_statefulset_envs( executor: &AirflowExecutor, auth_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, resolved_product_image: &ResolvedProductImage, ) -> Result, Error> { let mut env: BTreeMap = BTreeMap::new(); @@ -288,7 +288,7 @@ pub fn build_airflow_statefulset_envs( Ok(transform_map_to_vec(env)) } -pub fn get_dags_folder(git_sync_resources: &git_sync::v1alpha1::GitSyncResources) -> String { +pub fn get_dags_folder(git_sync_resources: &git_sync::v1alpha2::GitSyncResources) -> String { let git_sync_count = git_sync_resources.git_content_folders.len(); if git_sync_count > 1 { tracing::warn!( @@ -310,7 +310,7 @@ pub fn get_dags_folder(git_sync_resources: &git_sync::v1alpha1::GitSyncResources // This set of environment variables is a standard set that is not dependent on any // conditional logic and should be applied to the statefulset or the executor template config map. fn static_envs( - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> BTreeMap { let mut env: BTreeMap = BTreeMap::new(); @@ -372,7 +372,7 @@ pub fn build_airflow_template_envs( airflow: &v1alpha1::AirflowCluster, env_overrides: &HashMap, config: &ExecutorConfig, - git_sync_resources: &git_sync::v1alpha1::GitSyncResources, + git_sync_resources: &git_sync::v1alpha2::GitSyncResources, resolved_product_image: &ResolvedProductImage, ) -> Vec { let mut env: BTreeMap = BTreeMap::new(); From fd9726b66e5e2ba52f5d174f6f761a43513f6a7f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 22 Dec 2025 10:12:14 +0100 Subject: [PATCH 09/15] wip: airflow versioning --- Cargo.lock | 454 ++- Cargo.nix | 1729 ++++++++- Cargo.toml | 2 +- crate-hashes.json | 2 + deploy/helm/airflow-operator/crds/crds.yaml | 3186 ++++++++++++++++- .../templates/deployment.yaml | 4 + .../airflow-operator/templates/roles.yaml | 9 + .../airflow-operator/templates/service.yaml | 4 +- deploy/helm/airflow-operator/values.yaml | 9 + .../operator-binary/src/airflow_controller.rs | 40 +- rust/operator-binary/src/crd/affinity.rs | 10 +- rust/operator-binary/src/crd/authorization.rs | 6 +- .../src/crd/internal_secret.rs | 4 +- rust/operator-binary/src/crd/mod.rs | 64 +- rust/operator-binary/src/env_vars.rs | 10 +- rust/operator-binary/src/main.rs | 42 +- rust/operator-binary/src/operations/pdb.rs | 4 +- rust/operator-binary/src/service.rs | 16 +- .../src/webhooks/conversion.rs | 53 + rust/operator-binary/src/webhooks/mod.rs | 1 + .../30-install-airflow-cluster.yaml.j2 | 5 +- 21 files changed, 5525 insertions(+), 129 deletions(-) create mode 100644 rust/operator-binary/src/webhooks/conversion.rs create mode 100644 rust/operator-binary/src/webhooks/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 3f834c3f..8a8096ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,6 +101,12 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -221,12 +227,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" + [[package]] name = "bit-set" version = "0.8.0" @@ -364,6 +382,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const_format" version = "0.2.35" @@ -442,6 +466,18 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -498,6 +534,30 @@ dependencies = [ "syn 2.0.108", ] +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "deranged" version = "0.5.5" @@ -534,7 +594,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", ] [[package]] @@ -574,6 +636,20 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + [[package]] name = "educe" version = "0.6.0" @@ -592,6 +668,26 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pem-rfc7468", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.35" @@ -674,12 +770,28 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flate2" version = "1.1.5" @@ -821,6 +933,7 @@ checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -877,6 +990,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.4.12" @@ -919,6 +1043,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.12" @@ -1454,6 +1587,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "libc" @@ -1473,6 +1609,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "libz-sys" version = "1.1.22" @@ -1563,12 +1705,48 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1576,6 +1754,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1684,7 +1863,7 @@ dependencies = [ "futures-util", "opentelemetry", "percent-encoding", - "rand", + "rand 0.9.2", "thiserror 2.0.17", "tokio", "tokio-stream", @@ -1699,6 +1878,18 @@ dependencies = [ "num-traits", ] +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -1738,6 +1929,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1819,6 +2019,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -1849,6 +2070,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -1921,14 +2151,34 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha", - "rand_core", + "rand_chacha 0.9.0", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", ] [[package]] @@ -1938,7 +2188,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.16", ] [[package]] @@ -2048,6 +2307,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + [[package]] name = "ring" version = "0.17.14" @@ -2062,6 +2331,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rsa" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rstest" version = "0.26.1" @@ -2200,6 +2490,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secrecy" version = "0.10.3" @@ -2338,6 +2642,17 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2373,6 +2688,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -2443,6 +2768,22 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -2462,7 +2803,7 @@ dependencies = [ "futures 0.3.31", "indoc", "product-config", - "rand", + "rand 0.9.2", "rstest", "serde", "serde_json", @@ -2474,6 +2815,30 @@ dependencies = [ "tracing", ] +[[package]] +name = "stackable-certs" +version = "0.4.0" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +dependencies = [ + "const-oid", + "ecdsa", + "k8s-openapi", + "kube", + "p256", + "rand 0.9.2", + "rand_core 0.6.4", + "rsa", + "sha2", + "signature", + "snafu 0.8.9", + "stackable-shared", + "tokio", + "tokio-rustls", + "tracing", + "x509-cert", + "zeroize", +] + [[package]] name = "stackable-operator" version = "0.100.3" @@ -2504,6 +2869,7 @@ dependencies = [ "stackable-shared", "stackable-telemetry", "stackable-versioned", + "stackable-webhook", "strum", "tokio", "tracing", @@ -2594,6 +2960,35 @@ dependencies = [ "syn 2.0.108", ] +[[package]] +name = "stackable-webhook" +version = "0.7.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +dependencies = [ + "arc-swap", + "axum", + "futures-util", + "hyper", + "hyper-util", + "k8s-openapi", + "kube", + "opentelemetry", + "opentelemetry-semantic-conventions", + "rand 0.9.2", + "serde_json", + "snafu 0.8.9", + "stackable-certs", + "stackable-shared", + "stackable-telemetry", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tracing", + "tracing-opentelemetry", + "x509-cert", +] + [[package]] name = "strsim" version = "0.11.1" @@ -2759,6 +3154,27 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] + [[package]] name = "tokio" version = "1.48.0" @@ -3496,6 +3912,20 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "sha1", + "signature", + "spki", + "tls_codec", +] + [[package]] name = "xml" version = "1.0.1" @@ -3572,6 +4002,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.108", +] [[package]] name = "zerotrie" diff --git a/Cargo.nix b/Cargo.nix index fcfe6b8c..c81df7e1 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -339,6 +339,19 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "arc-swap" = rec { + crateName = "arc-swap"; + version = "1.7.1"; + edition = "2018"; + sha256 = "0mrl9a9r9p9bln74q6aszvf22q1ijiw089jkrmabfqkbj31zixv9"; + libName = "arc_swap"; + authors = [ + "Michal 'vorner' Vaner " + ]; + features = { + "serde" = [ "dep:serde" ]; + }; + }; "async-broadcast" = rec { crateName = "async-broadcast"; version = "0.7.2"; @@ -757,6 +770,19 @@ rec { }; resolvedDefaultFeatures = [ "default" "gloo-timers" "gloo-timers-sleep" "std" "std-blocking-sleep" "tokio" "tokio-sleep" ]; }; + "base16ct" = rec { + crateName = "base16ct"; + version = "0.2.0"; + edition = "2021"; + sha256 = "1kylrjhdzk7qpknrvlphw8ywdnvvg39dizw9622w3wk5xba04zsc"; + authors = [ + "RustCrypto Developers" + ]; + features = { + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "base64" = rec { crateName = "base64"; version = "0.22.1"; @@ -771,6 +797,19 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; + "base64ct" = rec { + crateName = "base64ct"; + version = "1.8.1"; + edition = "2024"; + sha256 = "12h6iwd0ib6xxwd0814wf3x6nd91r851xcycvlkpm199cii0y18f"; + authors = [ + "RustCrypto Developers" + ]; + features = { + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "bit-set" = rec { crateName = "bit-set"; version = "0.8.0"; @@ -1166,6 +1205,20 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "const-oid" = rec { + crateName = "const-oid"; + version = "0.9.6"; + edition = "2021"; + sha256 = "1y0jnqaq7p2wvspnx7qj76m7hjcqpz73qzvr9l2p9n2s51vr6if2"; + libName = "const_oid"; + authors = [ + "RustCrypto Developers" + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + }; + resolvedDefaultFeatures = [ "db" "std" ]; + }; "const_format" = rec { crateName = "const_format"; version = "0.2.35"; @@ -1380,6 +1433,58 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "crypto-bigint" = rec { + crateName = "crypto-bigint"; + version = "0.5.5"; + edition = "2021"; + sha256 = "0xmbdff3g6ii5sbxjxc31xfkv9lrmyril4arh3dzckd4gjsjzj8d"; + libName = "crypto_bigint"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "generic-array"; + packageId = "generic-array"; + optional = true; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + optional = true; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + features = [ "std" ]; + } + ]; + features = { + "alloc" = [ "serdect?/alloc" ]; + "default" = [ "rand" ]; + "der" = [ "dep:der" ]; + "generic-array" = [ "dep:generic-array" ]; + "rand" = [ "rand_core/std" ]; + "rand_core" = [ "dep:rand_core" ]; + "rlp" = [ "dep:rlp" ]; + "serde" = [ "dep:serdect" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "generic-array" "rand_core" "zeroize" ]; + }; "crypto-common" = rec { crateName = "crypto-common"; version = "0.1.6"; @@ -1404,6 +1509,7 @@ rec { "getrandom" = [ "rand_core/getrandom" ]; "rand_core" = [ "dep:rand_core" ]; }; + resolvedDefaultFeatures = [ "std" ]; }; "darling" = rec { crateName = "darling"; @@ -1525,6 +1631,83 @@ rec { } ]; + }; + "der" = rec { + crateName = "der"; + version = "0.7.10"; + edition = "2021"; + sha256 = "1jyxacyxdx6mxbkfw99jz59dzvcd9k17rq01a7xvn1dr6wl87hg7"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "const-oid"; + packageId = "const-oid"; + optional = true; + } + { + name = "der_derive"; + packageId = "der_derive"; + optional = true; + } + { + name = "flagset"; + packageId = "flagset"; + optional = true; + } + { + name = "pem-rfc7468"; + packageId = "pem-rfc7468"; + optional = true; + features = [ "alloc" ]; + } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "zeroize?/alloc" ]; + "arbitrary" = [ "dep:arbitrary" "const-oid?/arbitrary" "std" ]; + "bytes" = [ "dep:bytes" "alloc" ]; + "derive" = [ "dep:der_derive" ]; + "flagset" = [ "dep:flagset" ]; + "oid" = [ "dep:const-oid" ]; + "pem" = [ "dep:pem-rfc7468" "alloc" "zeroize" ]; + "std" = [ "alloc" ]; + "time" = [ "dep:time" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "alloc" "derive" "flagset" "oid" "pem" "std" "zeroize" ]; + }; + "der_derive" = rec { + crateName = "der_derive"; + version = "0.7.3"; + edition = "2021"; + sha256 = "065d2wy7zd0dank99hh58l5x7lv50hxnr7j6f3sphlb7i4ihjd40"; + procMacro = true; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.108"; + features = [ "extra-traits" ]; + } + ]; + }; "deranged" = rec { crateName = "deranged"; @@ -1656,10 +1839,21 @@ rec { packageId = "block-buffer"; optional = true; } + { + name = "const-oid"; + packageId = "const-oid"; + optional = true; + } { name = "crypto-common"; packageId = "crypto-common"; } + { + name = "subtle"; + packageId = "subtle"; + optional = true; + usesDefaultFeatures = false; + } ]; features = { "blobby" = [ "dep:blobby" ]; @@ -1674,7 +1868,7 @@ rec { "std" = [ "alloc" "crypto-common/std" ]; "subtle" = [ "dep:subtle" ]; }; - resolvedDefaultFeatures = [ "block-buffer" "core-api" "default" ]; + resolvedDefaultFeatures = [ "alloc" "block-buffer" "const-oid" "core-api" "default" "mac" "oid" "std" "subtle" ]; }; "displaydoc" = rec { crateName = "displaydoc"; @@ -1763,6 +1957,79 @@ rec { ]; }; + "ecdsa" = rec { + crateName = "ecdsa"; + version = "0.16.9"; + edition = "2021"; + sha256 = "1jhb0bcbkaz4001sdmfyv8ajrv8a1cg7z7aa5myrd4jjbhmz69zf"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "der"; + packageId = "der"; + optional = true; + } + { + name = "digest"; + packageId = "digest"; + optional = true; + usesDefaultFeatures = false; + features = [ "oid" ]; + } + { + name = "elliptic-curve"; + packageId = "elliptic-curve"; + usesDefaultFeatures = false; + features = [ "digest" "sec1" ]; + } + { + name = "rfc6979"; + packageId = "rfc6979"; + optional = true; + } + { + name = "signature"; + packageId = "signature"; + usesDefaultFeatures = false; + features = [ "rand_core" ]; + } + { + name = "spki"; + packageId = "spki"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "elliptic-curve"; + packageId = "elliptic-curve"; + usesDefaultFeatures = false; + features = [ "dev" ]; + } + ]; + features = { + "alloc" = [ "elliptic-curve/alloc" "signature/alloc" "spki/alloc" ]; + "arithmetic" = [ "elliptic-curve/arithmetic" ]; + "default" = [ "digest" ]; + "der" = [ "dep:der" ]; + "dev" = [ "arithmetic" "digest" "elliptic-curve/dev" "hazmat" ]; + "digest" = [ "dep:digest" "signature/digest" ]; + "pem" = [ "elliptic-curve/pem" "pkcs8" ]; + "pkcs8" = [ "digest" "elliptic-curve/pkcs8" "der" ]; + "rfc6979" = [ "dep:rfc6979" ]; + "serde" = [ "elliptic-curve/serde" "serdect" ]; + "serdect" = [ "dep:serdect" ]; + "sha2" = [ "dep:sha2" ]; + "signing" = [ "arithmetic" "digest" "hazmat" "rfc6979" ]; + "spki" = [ "dep:spki" ]; + "std" = [ "alloc" "elliptic-curve/std" "signature/std" ]; + "verifying" = [ "arithmetic" "digest" "hazmat" ]; + }; + resolvedDefaultFeatures = [ "alloc" "arithmetic" "default" "der" "digest" "hazmat" "pem" "pkcs8" "rfc6979" "signing" "spki" "std" "verifying" ]; + }; "educe" = rec { crateName = "educe"; version = "0.6.0"; @@ -1820,6 +2087,104 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" "use_std" ]; }; + "elliptic-curve" = rec { + crateName = "elliptic-curve"; + version = "0.13.8"; + edition = "2021"; + sha256 = "0ixx4brgnzi61z29r3g1606nh2za88hzyz8c5r3p6ydzhqq09rmm"; + libName = "elliptic_curve"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "base16ct"; + packageId = "base16ct"; + } + { + name = "crypto-bigint"; + packageId = "crypto-bigint"; + usesDefaultFeatures = false; + features = [ "rand_core" "generic-array" "zeroize" ]; + } + { + name = "digest"; + packageId = "digest"; + optional = true; + } + { + name = "ff"; + packageId = "ff"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "generic-array"; + packageId = "generic-array"; + usesDefaultFeatures = false; + features = [ "zeroize" ]; + } + { + name = "group"; + packageId = "group"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "pem-rfc7468"; + packageId = "pem-rfc7468"; + optional = true; + features = [ "alloc" ]; + } + { + name = "pkcs8"; + packageId = "pkcs8"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + usesDefaultFeatures = false; + } + { + name = "sec1"; + packageId = "sec1"; + optional = true; + features = [ "subtle" "zeroize" ]; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + } + { + name = "zeroize"; + packageId = "zeroize"; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "base16ct/alloc" "ff?/alloc" "group?/alloc" "pkcs8?/alloc" "sec1?/alloc" "zeroize/alloc" ]; + "arithmetic" = [ "group" ]; + "bits" = [ "arithmetic" "ff/bits" "dep:tap" ]; + "default" = [ "arithmetic" ]; + "dev" = [ "arithmetic" "dep:hex-literal" "pem" "pkcs8" ]; + "digest" = [ "dep:digest" ]; + "ecdh" = [ "arithmetic" "digest" "dep:hkdf" ]; + "ff" = [ "dep:ff" ]; + "group" = [ "dep:group" "ff" ]; + "hash2curve" = [ "arithmetic" "digest" ]; + "jwk" = [ "dep:base64ct" "dep:serde_json" "alloc" "serde" "zeroize/alloc" ]; + "pem" = [ "dep:pem-rfc7468" "alloc" "arithmetic" "pkcs8" "sec1/pem" ]; + "pkcs8" = [ "dep:pkcs8" "sec1" ]; + "sec1" = [ "dep:sec1" ]; + "serde" = [ "dep:serdect" "alloc" "pkcs8" "sec1/serde" ]; + "std" = [ "alloc" "rand_core/std" "pkcs8?/std" "sec1?/std" ]; + "voprf" = [ "digest" ]; + }; + resolvedDefaultFeatures = [ "alloc" "arithmetic" "digest" "ff" "group" "hazmat" "pem" "pkcs8" "sec1" "std" ]; + }; "encoding_rs" = rec { crateName = "encoding_rs"; version = "0.8.35"; @@ -2031,6 +2396,40 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "std" ]; }; + "ff" = rec { + crateName = "ff"; + version = "0.13.1"; + edition = "2021"; + sha256 = "14v3bc6q24gbcjnxjfbq2dddgf4as2z2gd4mj35gjlrncpxhpdf0"; + authors = [ + "Sean Bowe " + "Jack Grigg " + ]; + dependencies = [ + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + usesDefaultFeatures = false; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "bits" = [ "bitvec" ]; + "bitvec" = [ "dep:bitvec" ]; + "byteorder" = [ "dep:byteorder" ]; + "default" = [ "bits" "std" ]; + "derive" = [ "byteorder" "ff_derive" ]; + "derive_bits" = [ "bits" "ff_derive/bits" ]; + "ff_derive" = [ "dep:ff_derive" ]; + "std" = [ "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "find-msvc-tools" = rec { crateName = "find-msvc-tools"; version = "0.1.4"; @@ -2039,11 +2438,23 @@ rec { libName = "find_msvc_tools"; }; - "flate2" = rec { - crateName = "flate2"; - version = "1.1.5"; - edition = "2018"; - sha256 = "1yrvxgxyg7mzksmmcd9i7vc3023kbv3zhdsf8mkjm8c5ivfkxqxz"; + "flagset" = rec { + crateName = "flagset"; + version = "0.4.7"; + edition = "2021"; + sha256 = "1zplx30g76kl5la3ayl9ns5p3diqd9zphbcggqcm4nm7411q5b5p"; + authors = [ + "Nathaniel McCallum " + ]; + features = { + "serde" = [ "dep:serde" ]; + }; + }; + "flate2" = rec { + crateName = "flate2"; + version = "1.1.5"; + edition = "2018"; + sha256 = "1yrvxgxyg7mzksmmcd9i7vc3023kbv3zhdsf8mkjm8c5ivfkxqxz"; authors = [ "Alex Crichton " "Josh Triplett " @@ -2462,6 +2873,12 @@ rec { name = "typenum"; packageId = "typenum"; } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } ]; buildDependencies = [ { @@ -2473,7 +2890,7 @@ rec { "serde" = [ "dep:serde" ]; "zeroize" = [ "dep:zeroize" ]; }; - resolvedDefaultFeatures = [ "more_lengths" ]; + resolvedDefaultFeatures = [ "more_lengths" "zeroize" ]; }; "getrandom 0.2.16" = rec { crateName = "getrandom"; @@ -2509,6 +2926,7 @@ rec { "rustc-dep-of-std" = [ "compiler_builtins" "core" "libc/rustc-dep-of-std" "wasi/rustc-dep-of-std" ]; "wasm-bindgen" = [ "dep:wasm-bindgen" ]; }; + resolvedDefaultFeatures = [ "std" ]; }; "getrandom 0.3.4" = rec { crateName = "getrandom"; @@ -2677,6 +3095,42 @@ rec { }; resolvedDefaultFeatures = [ "default" "futures" "futures-channel" "futures-core" ]; }; + "group" = rec { + crateName = "group"; + version = "0.13.0"; + edition = "2021"; + sha256 = "0qqs2p5vqnv3zvq9mfjkmw3qlvgqb0c3cm6p33srkh7pc9sfzygh"; + authors = [ + "Sean Bowe " + "Jack Grigg " + ]; + dependencies = [ + { + name = "ff"; + packageId = "ff"; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + usesDefaultFeatures = false; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "alloc" ]; + "memuse" = [ "dep:memuse" ]; + "rand" = [ "dep:rand" ]; + "rand_xorshift" = [ "dep:rand_xorshift" ]; + "tests" = [ "alloc" "rand" "rand_xorshift" ]; + "wnaf-memuse" = [ "alloc" "memuse" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "h2" = rec { crateName = "h2"; version = "0.4.12"; @@ -2820,6 +3274,33 @@ rec { sha256 = "1sjmpsdl8czyh9ywl3qcsfsq9a307dg4ni2vnlwgnzzqhc4y0113"; }; + "hmac" = rec { + crateName = "hmac"; + version = "0.12.1"; + edition = "2018"; + sha256 = "0pmbr069sfg76z7wsssfk5ddcqd9ncp79fyz6zcm6yn115yc6jbc"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "digest"; + packageId = "digest"; + features = [ "mac" ]; + } + ]; + devDependencies = [ + { + name = "digest"; + packageId = "digest"; + features = [ "dev" ]; + } + ]; + features = { + "std" = [ "digest/std" ]; + }; + resolvedDefaultFeatures = [ "reset" ]; + }; "home" = rec { crateName = "home"; version = "0.5.12"; @@ -3086,7 +3567,7 @@ rec { "server" = [ "dep:httpdate" "dep:pin-project-lite" "dep:smallvec" ]; "tracing" = [ "dep:tracing" ]; }; - resolvedDefaultFeatures = [ "client" "default" "http1" "http2" "server" ]; + resolvedDefaultFeatures = [ "client" "default" "full" "http1" "http2" "server" ]; }; "hyper-rustls" = rec { crateName = "hyper-rustls"; @@ -4805,10 +5286,20 @@ rec { authors = [ "Marvin Löbel " ]; + dependencies = [ + { + name = "spin"; + packageId = "spin"; + optional = true; + usesDefaultFeatures = false; + features = [ "once" ]; + } + ]; features = { "spin" = [ "dep:spin" ]; "spin_no_std" = [ "spin" ]; }; + resolvedDefaultFeatures = [ "spin" "spin_no_std" ]; }; "libc" = rec { crateName = "libc"; @@ -4870,6 +5361,20 @@ rec { "zlib-ng-compat" = [ "libz-sys/zlib-ng" "libssh2-sys?/zlib-ng-compat" ]; }; }; + "libm" = rec { + crateName = "libm"; + version = "0.2.15"; + edition = "2021"; + sha256 = "1plpzf0p829viazdj57yw5dhmlr8ywf3apayxc2f2bq5a6mvryzr"; + authors = [ + "Jorge Aparicio " + ]; + features = { + "default" = [ "arch" ]; + "unstable" = [ "unstable-intrinsics" "unstable-float" ]; + }; + resolvedDefaultFeatures = [ "arch" "default" ]; + }; "libz-sys" = rec { crateName = "libz-sys"; version = "1.1.22"; @@ -5142,6 +5647,81 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "num-bigint-dig" = rec { + crateName = "num-bigint-dig"; + version = "0.8.6"; + edition = "2021"; + sha256 = "1dxh3d8pzjc5k0kpy8gy2qhhhqs7zw8a7m564zl3ib8gcjkdsqg6"; + libName = "num_bigint_dig"; + authors = [ + "dignifiedquire " + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "lazy_static"; + packageId = "lazy_static"; + usesDefaultFeatures = false; + features = [ "spin_no_std" ]; + } + { + name = "libm"; + packageId = "libm"; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-iter"; + packageId = "num-iter"; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "rand"; + packageId = "rand 0.8.5"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "smallvec"; + packageId = "smallvec"; + usesDefaultFeatures = false; + } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "rand"; + packageId = "rand 0.8.5"; + features = [ "small_rng" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" ]; + "default" = [ "std" "u64_digit" ]; + "fuzz" = [ "arbitrary" "smallvec/arbitrary" ]; + "prime" = [ "rand/std_rng" ]; + "rand" = [ "dep:rand" ]; + "serde" = [ "dep:serde" ]; + "std" = [ "num-integer/std" "num-traits/std" "smallvec/write" "rand/std" "serde/std" ]; + "zeroize" = [ "dep:zeroize" ]; + }; + resolvedDefaultFeatures = [ "i128" "prime" "rand" "u64_digit" "zeroize" ]; + }; "num-conv" = rec { crateName = "num-conv"; version = "0.1.0"; @@ -5153,6 +5733,63 @@ rec { ]; }; + "num-integer" = rec { + crateName = "num-integer"; + version = "0.1.46"; + edition = "2018"; + sha256 = "13w5g54a9184cqlbsq80rnxw4jj4s0d8wv75jsq5r2lms8gncsbr"; + libName = "num_integer"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "num-traits/std" ]; + }; + resolvedDefaultFeatures = [ "i128" ]; + }; + "num-iter" = rec { + crateName = "num-iter"; + version = "0.1.45"; + edition = "2018"; + sha256 = "1gzm7vc5g9qsjjl3bqk9rz1h6raxhygbrcpbfl04swlh0i506a8l"; + libName = "num_iter"; + authors = [ + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "i128" ]; + } + ]; + buildDependencies = [ + { + name = "autocfg"; + packageId = "autocfg"; + } + ]; + features = { + "default" = [ "std" ]; + "std" = [ "num-integer/std" "num-traits/std" ]; + }; + }; "num-traits" = rec { crateName = "num-traits"; version = "0.2.19"; @@ -5162,6 +5799,13 @@ rec { authors = [ "The Rust Project Developers" ]; + dependencies = [ + { + name = "libm"; + packageId = "libm"; + optional = true; + } + ]; buildDependencies = [ { name = "autocfg"; @@ -5172,7 +5816,7 @@ rec { "default" = [ "std" ]; "libm" = [ "dep:libm" ]; }; - resolvedDefaultFeatures = [ "std" ]; + resolvedDefaultFeatures = [ "i128" "libm" "std" ]; }; "once_cell" = rec { crateName = "once_cell"; @@ -5603,7 +6247,7 @@ rec { } { name = "rand"; - packageId = "rand"; + packageId = "rand 0.9.2"; optional = true; usesDefaultFeatures = false; features = [ "std" "std_rng" "small_rng" "os_rng" "thread_rng" ]; @@ -5685,6 +6329,79 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "p256" = rec { + crateName = "p256"; + version = "0.13.2"; + edition = "2021"; + sha256 = "0jyd3c3k239ybs59ixpnl7dqkmm072fr1js8kh7ldx58bzc3m1n9"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "ecdsa"; + packageId = "ecdsa"; + rename = "ecdsa-core"; + optional = true; + usesDefaultFeatures = false; + features = [ "der" ]; + } + { + name = "elliptic-curve"; + packageId = "elliptic-curve"; + usesDefaultFeatures = false; + features = [ "hazmat" "sec1" ]; + } + { + name = "primeorder"; + packageId = "primeorder"; + optional = true; + } + { + name = "sha2"; + packageId = "sha2"; + optional = true; + usesDefaultFeatures = false; + } + ]; + devDependencies = [ + { + name = "ecdsa"; + packageId = "ecdsa"; + rename = "ecdsa-core"; + usesDefaultFeatures = false; + features = [ "dev" ]; + } + { + name = "primeorder"; + packageId = "primeorder"; + features = [ "dev" ]; + } + ]; + features = { + "alloc" = [ "ecdsa-core?/alloc" "elliptic-curve/alloc" ]; + "arithmetic" = [ "dep:primeorder" "elliptic-curve/arithmetic" ]; + "bits" = [ "arithmetic" "elliptic-curve/bits" ]; + "default" = [ "arithmetic" "ecdsa" "pem" "std" ]; + "digest" = [ "ecdsa-core/digest" "ecdsa-core/hazmat" ]; + "ecdh" = [ "arithmetic" "elliptic-curve/ecdh" ]; + "ecdsa" = [ "arithmetic" "ecdsa-core/signing" "ecdsa-core/verifying" "sha256" ]; + "ecdsa-core" = [ "dep:ecdsa-core" ]; + "expose-field" = [ "arithmetic" ]; + "hash2curve" = [ "arithmetic" "elliptic-curve/hash2curve" ]; + "jwk" = [ "elliptic-curve/jwk" ]; + "pem" = [ "elliptic-curve/pem" "ecdsa-core/pem" "pkcs8" ]; + "pkcs8" = [ "ecdsa-core?/pkcs8" "elliptic-curve/pkcs8" ]; + "serde" = [ "ecdsa-core?/serde" "elliptic-curve/serde" "primeorder?/serde" "serdect" ]; + "serdect" = [ "dep:serdect" ]; + "sha2" = [ "dep:sha2" ]; + "sha256" = [ "digest" "sha2" ]; + "std" = [ "alloc" "ecdsa-core?/std" "elliptic-curve/std" ]; + "test-vectors" = [ "dep:hex-literal" ]; + "voprf" = [ "elliptic-curve/voprf" "sha2" ]; + }; + resolvedDefaultFeatures = [ "alloc" "arithmetic" "default" "digest" "ecdsa" "ecdsa-core" "pem" "pkcs8" "sha2" "sha256" "std" ]; + }; "parking" = rec { crateName = "parking"; version = "2.2.1"; @@ -5793,6 +6510,27 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "pem-rfc7468" = rec { + crateName = "pem-rfc7468"; + version = "0.7.0"; + edition = "2021"; + sha256 = "04l4852scl4zdva31c1z6jafbak0ni5pi0j38ml108zwzjdrrcw8"; + libName = "pem_rfc7468"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "base64ct"; + packageId = "base64ct"; + } + ]; + features = { + "alloc" = [ "base64ct/alloc" ]; + "std" = [ "alloc" "base64ct/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" ]; + }; "percent-encoding" = rec { crateName = "percent-encoding"; version = "2.3.2"; @@ -5989,6 +6727,74 @@ rec { ]; }; + "pkcs1" = rec { + crateName = "pkcs1"; + version = "0.7.5"; + edition = "2021"; + sha256 = "0zz4mil3nchnxljdfs2k5ab1cjqn7kq5lqp62n9qfix01zqvkzy8"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "der"; + packageId = "der"; + features = [ "oid" ]; + } + { + name = "pkcs8"; + packageId = "pkcs8"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "spki"; + packageId = "spki"; + } + ]; + features = { + "alloc" = [ "der/alloc" "zeroize" "pkcs8?/alloc" ]; + "pem" = [ "alloc" "der/pem" "pkcs8?/pem" ]; + "pkcs8" = [ "dep:pkcs8" ]; + "std" = [ "der/std" "alloc" ]; + "zeroize" = [ "der/zeroize" ]; + }; + resolvedDefaultFeatures = [ "alloc" "pem" "pkcs8" "std" "zeroize" ]; + }; + "pkcs8" = rec { + crateName = "pkcs8"; + version = "0.10.2"; + edition = "2021"; + sha256 = "1dx7w21gvn07azszgqd3ryjhyphsrjrmq5mmz1fbxkj5g0vv4l7r"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "der"; + packageId = "der"; + features = [ "oid" ]; + } + { + name = "spki"; + packageId = "spki"; + } + ]; + features = { + "3des" = [ "encryption" "pkcs5/3des" ]; + "alloc" = [ "der/alloc" "der/zeroize" "spki/alloc" ]; + "des-insecure" = [ "encryption" "pkcs5/des-insecure" ]; + "encryption" = [ "alloc" "pkcs5/alloc" "pkcs5/pbes2" "rand_core" ]; + "getrandom" = [ "rand_core/getrandom" ]; + "pem" = [ "alloc" "der/pem" "spki/pem" ]; + "pkcs5" = [ "dep:pkcs5" ]; + "rand_core" = [ "dep:rand_core" ]; + "sha1-insecure" = [ "encryption" "pkcs5/sha1-insecure" ]; + "std" = [ "alloc" "der/std" "spki/std" ]; + "subtle" = [ "dep:subtle" ]; + }; + resolvedDefaultFeatures = [ "alloc" "pem" "std" ]; + }; "pkg-config" = rec { crateName = "pkg-config"; version = "0.3.32"; @@ -6060,7 +6866,30 @@ rec { }; resolvedDefaultFeatures = [ "simd" "std" ]; }; - "proc-macro-crate" = rec { + "primeorder" = rec { + crateName = "primeorder"; + version = "0.13.6"; + edition = "2021"; + sha256 = "1rp16710mxksagcjnxqjjq9r9wf5vf72fs8wxffnvhb6i6hiqgim"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "elliptic-curve"; + packageId = "elliptic-curve"; + usesDefaultFeatures = false; + features = [ "arithmetic" "sec1" ]; + } + ]; + features = { + "alloc" = [ "elliptic-curve/alloc" ]; + "serde" = [ "elliptic-curve/serde" "serdect" ]; + "serdect" = [ "dep:serdect" ]; + "std" = [ "alloc" "elliptic-curve/std" ]; + }; + }; + "proc-macro-crate" = rec { crateName = "proc-macro-crate"; version = "3.4.0"; edition = "2021"; @@ -6255,7 +7084,44 @@ rec { "rustc-dep-of-std" = [ "core" ]; }; }; - "rand" = rec { + "rand 0.8.5" = rec { + crateName = "rand"; + version = "0.8.5"; + edition = "2018"; + sha256 = "013l6931nn7gkc23jz5mm3qdhf93jjf0fg64nz2lp4i51qd8vbrl"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "rand_chacha"; + packageId = "rand_chacha 0.3.1"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + } + ]; + features = { + "alloc" = [ "rand_core/alloc" ]; + "default" = [ "std" "std_rng" ]; + "getrandom" = [ "rand_core/getrandom" ]; + "libc" = [ "dep:libc" ]; + "log" = [ "dep:log" ]; + "packed_simd" = [ "dep:packed_simd" ]; + "rand_chacha" = [ "dep:rand_chacha" ]; + "serde" = [ "dep:serde" ]; + "serde1" = [ "serde" "rand_core/serde1" ]; + "simd_support" = [ "packed_simd" ]; + "std" = [ "rand_core/std" "rand_chacha/std" "alloc" "getrandom" "libc" ]; + "std_rng" = [ "rand_chacha" ]; + }; + resolvedDefaultFeatures = [ "rand_chacha" "std_rng" ]; + }; + "rand 0.9.2" = rec { crateName = "rand"; version = "0.9.2"; edition = "2021"; @@ -6267,13 +7133,13 @@ rec { dependencies = [ { name = "rand_chacha"; - packageId = "rand_chacha"; + packageId = "rand_chacha 0.9.0"; optional = true; usesDefaultFeatures = false; } { name = "rand_core"; - packageId = "rand_core"; + packageId = "rand_core 0.9.3"; usesDefaultFeatures = false; } ]; @@ -6288,7 +7154,36 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "os_rng" "small_rng" "std" "std_rng" "thread_rng" ]; }; - "rand_chacha" = rec { + "rand_chacha 0.3.1" = rec { + crateName = "rand_chacha"; + version = "0.3.1"; + edition = "2018"; + sha256 = "123x2adin558xbhvqb8w4f6syjsdkmqff8cxwhmjacpsl1ihmhg6"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + "The CryptoCorrosion Contributors" + ]; + dependencies = [ + { + name = "ppv-lite86"; + packageId = "ppv-lite86"; + usesDefaultFeatures = false; + features = [ "simd" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + } + ]; + features = { + "default" = [ "std" ]; + "serde" = [ "dep:serde" ]; + "serde1" = [ "serde" ]; + "std" = [ "ppv-lite86/std" ]; + }; + }; + "rand_chacha 0.9.0" = rec { crateName = "rand_chacha"; version = "0.9.0"; edition = "2021"; @@ -6307,13 +7202,13 @@ rec { } { name = "rand_core"; - packageId = "rand_core"; + packageId = "rand_core 0.9.3"; } ]; devDependencies = [ { name = "rand_core"; - packageId = "rand_core"; + packageId = "rand_core 0.9.3"; features = [ "os_rng" ]; } ]; @@ -6325,7 +7220,31 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "rand_core" = rec { + "rand_core 0.6.4" = rec { + crateName = "rand_core"; + version = "0.6.4"; + edition = "2018"; + sha256 = "0b4j2v4cb5krak1pv6kakv4sz6xcwbrmy2zckc32hsigbrwy82zc"; + authors = [ + "The Rand Project Developers" + "The Rust Project Developers" + ]; + dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.2.16"; + optional = true; + } + ]; + features = { + "getrandom" = [ "dep:getrandom" ]; + "serde" = [ "dep:serde" ]; + "serde1" = [ "serde" ]; + "std" = [ "alloc" "getrandom" "getrandom/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "getrandom" "std" ]; + }; + "rand_core 0.9.3" = rec { crateName = "rand_core"; version = "0.9.3"; edition = "2021"; @@ -6796,6 +7715,29 @@ rec { }; resolvedDefaultFeatures = [ "blocking" ]; }; + "rfc6979" = rec { + crateName = "rfc6979"; + version = "0.4.0"; + edition = "2021"; + sha256 = "1chw95jgcfrysyzsq6a10b1j5qb7bagkx8h0wda4lv25in02mpgq"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "hmac"; + packageId = "hmac"; + usesDefaultFeatures = false; + features = [ "reset" ]; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + } + ]; + + }; "ring" = rec { crateName = "ring"; version = "0.17.14"; @@ -6857,6 +7799,119 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "dev_urandom_fallback" ]; }; + "rsa" = rec { + crateName = "rsa"; + version = "0.9.9"; + edition = "2021"; + sha256 = "122wywpd4m3v183sj7gzykqi5qkvgfzy8hynv5wq4dfha1n3g820"; + authors = [ + "RustCrypto Developers" + "dignifiedquire " + ]; + dependencies = [ + { + name = "const-oid"; + packageId = "const-oid"; + usesDefaultFeatures = false; + } + { + name = "digest"; + packageId = "digest"; + usesDefaultFeatures = false; + features = [ "alloc" "oid" ]; + } + { + name = "num-bigint-dig"; + packageId = "num-bigint-dig"; + rename = "num-bigint"; + usesDefaultFeatures = false; + features = [ "i128" "prime" "zeroize" ]; + } + { + name = "num-integer"; + packageId = "num-integer"; + usesDefaultFeatures = false; + } + { + name = "num-traits"; + packageId = "num-traits"; + usesDefaultFeatures = false; + features = [ "libm" ]; + } + { + name = "pkcs1"; + packageId = "pkcs1"; + usesDefaultFeatures = false; + features = [ "alloc" "pkcs8" ]; + } + { + name = "pkcs8"; + packageId = "pkcs8"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + usesDefaultFeatures = false; + } + { + name = "sha2"; + packageId = "sha2"; + optional = true; + usesDefaultFeatures = false; + features = [ "oid" ]; + } + { + name = "signature"; + packageId = "signature"; + usesDefaultFeatures = false; + features = [ "alloc" "digest" "rand_core" ]; + } + { + name = "spki"; + packageId = "spki"; + usesDefaultFeatures = false; + features = [ "alloc" ]; + } + { + name = "subtle"; + packageId = "subtle"; + usesDefaultFeatures = false; + } + { + name = "zeroize"; + packageId = "zeroize"; + features = [ "alloc" ]; + } + ]; + devDependencies = [ + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + usesDefaultFeatures = false; + } + { + name = "sha2"; + packageId = "sha2"; + usesDefaultFeatures = false; + features = [ "oid" ]; + } + ]; + features = { + "default" = [ "std" "pem" "u64_digit" ]; + "getrandom" = [ "rand_core/getrandom" ]; + "nightly" = [ "num-bigint/nightly" ]; + "pem" = [ "pkcs1/pem" "pkcs8/pem" ]; + "pkcs5" = [ "pkcs8/encryption" ]; + "serde" = [ "dep:serde" "num-bigint/serde" ]; + "sha1" = [ "dep:sha1" ]; + "sha2" = [ "dep:sha2" ]; + "std" = [ "digest/std" "pkcs1/std" "pkcs8/std" "rand_core/std" "signature/std" ]; + "u64_digit" = [ "num-bigint/u64_digit" ]; + }; + resolvedDefaultFeatures = [ "default" "pem" "sha2" "std" "u64_digit" ]; + }; "rstest" = rec { crateName = "rstest"; version = "0.26.1"; @@ -7295,6 +8350,66 @@ rec { "default" = [ "use_std" ]; }; }; + "sec1" = rec { + crateName = "sec1"; + version = "0.7.3"; + edition = "2021"; + sha256 = "1p273j8c87pid6a1iyyc7vxbvifrw55wbxgr0dh3l8vnbxb7msfk"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "base16ct"; + packageId = "base16ct"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "der"; + packageId = "der"; + optional = true; + features = [ "oid" ]; + } + { + name = "generic-array"; + packageId = "generic-array"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "pkcs8"; + packageId = "pkcs8"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "subtle"; + packageId = "subtle"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "zeroize"; + packageId = "zeroize"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "alloc" = [ "der?/alloc" "pkcs8?/alloc" "zeroize?/alloc" ]; + "default" = [ "der" "point" ]; + "der" = [ "dep:der" "zeroize" ]; + "pem" = [ "alloc" "der/pem" "pkcs8/pem" ]; + "pkcs8" = [ "dep:pkcs8" ]; + "point" = [ "dep:base16ct" "dep:generic-array" ]; + "serde" = [ "dep:serdect" ]; + "std" = [ "alloc" "der?/std" ]; + "subtle" = [ "dep:subtle" ]; + "zeroize" = [ "dep:zeroize" "der?/zeroize" ]; + }; + resolvedDefaultFeatures = [ "alloc" "default" "der" "pem" "pkcs8" "point" "std" "subtle" "zeroize" ]; + }; "secrecy" = rec { crateName = "secrecy"; version = "0.10.3"; @@ -7694,6 +8809,45 @@ rec { ]; }; + "sha1" = rec { + crateName = "sha1"; + version = "0.10.6"; + edition = "2018"; + sha256 = "1fnnxlfg08xhkmwf2ahv634as30l1i3xhlhkvxflmasi5nd85gz3"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "cpufeatures"; + packageId = "cpufeatures"; + target = { target, features }: (("aarch64" == target."arch" or null) || ("x86" == target."arch" or null) || ("x86_64" == target."arch" or null)); + } + { + name = "digest"; + packageId = "digest"; + } + ]; + devDependencies = [ + { + name = "digest"; + packageId = "digest"; + features = [ "dev" ]; + } + ]; + features = { + "asm" = [ "sha1-asm" ]; + "default" = [ "std" ]; + "oid" = [ "digest/oid" ]; + "sha1-asm" = [ "dep:sha1-asm" ]; + "std" = [ "digest/std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "sha2" = rec { crateName = "sha2"; version = "0.10.9"; @@ -7732,6 +8886,7 @@ rec { "sha2-asm" = [ "dep:sha2-asm" ]; "std" = [ "digest/std" ]; }; + resolvedDefaultFeatures = [ "default" "oid" "std" ]; }; "sharded-slab" = rec { crateName = "sharded-slab"; @@ -7788,6 +8943,36 @@ rec { ]; }; + "signature" = rec { + crateName = "signature"; + version = "2.2.0"; + edition = "2021"; + sha256 = "1pi9hd5vqfr3q3k49k37z06p7gs5si0in32qia4mmr1dancr6m3p"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "digest"; + packageId = "digest"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "rand_core"; + packageId = "rand_core 0.6.4"; + optional = true; + usesDefaultFeatures = false; + } + ]; + features = { + "derive" = [ "dep:derive" ]; + "digest" = [ "dep:digest" ]; + "rand_core" = [ "dep:rand_core" ]; + "std" = [ "alloc" "rand_core?/std" ]; + }; + resolvedDefaultFeatures = [ "alloc" "digest" "rand_core" "std" ]; + }; "simd-adler32" = rec { crateName = "simd-adler32"; version = "0.3.7"; @@ -7989,6 +9174,63 @@ rec { }; resolvedDefaultFeatures = [ "all" ]; }; + "spin" = rec { + crateName = "spin"; + version = "0.9.8"; + edition = "2015"; + sha256 = "0rvam5r0p3a6qhc18scqpvpgb3ckzyqxpgdfyjnghh8ja7byi039"; + authors = [ + "Mathijs van de Nes " + "John Ericson " + "Joshua Barretto " + ]; + features = { + "barrier" = [ "mutex" ]; + "default" = [ "lock_api" "mutex" "spin_mutex" "rwlock" "once" "lazy" "barrier" ]; + "fair_mutex" = [ "mutex" ]; + "lazy" = [ "once" ]; + "lock_api" = [ "lock_api_crate" ]; + "lock_api_crate" = [ "dep:lock_api_crate" ]; + "portable-atomic" = [ "dep:portable-atomic" ]; + "portable_atomic" = [ "portable-atomic" ]; + "spin_mutex" = [ "mutex" ]; + "ticket_mutex" = [ "mutex" ]; + "use_ticket_mutex" = [ "mutex" "ticket_mutex" ]; + }; + resolvedDefaultFeatures = [ "once" ]; + }; + "spki" = rec { + crateName = "spki"; + version = "0.7.3"; + edition = "2021"; + sha256 = "17fj8k5fmx4w9mp27l970clrh5qa7r5sjdvbsln987xhb34dc7nr"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "base64ct"; + packageId = "base64ct"; + optional = true; + usesDefaultFeatures = false; + } + { + name = "der"; + packageId = "der"; + features = [ "oid" ]; + } + ]; + features = { + "alloc" = [ "base64ct?/alloc" "der/alloc" ]; + "arbitrary" = [ "std" "dep:arbitrary" "der/arbitrary" ]; + "base64" = [ "dep:base64ct" ]; + "fingerprint" = [ "sha2" ]; + "pem" = [ "alloc" "der/pem" ]; + "sha2" = [ "dep:sha2" ]; + "std" = [ "der/std" "alloc" ]; + }; + resolvedDefaultFeatures = [ "alloc" "pem" "std" ]; + }; "stable_deref_trait" = rec { crateName = "stable_deref_trait"; version = "1.2.1"; @@ -8049,68 +9291,171 @@ rec { packageId = "indoc"; } { - name = "product-config"; - packageId = "product-config"; + name = "product-config"; + packageId = "product-config"; + } + { + name = "rand"; + packageId = "rand 0.9.2"; + } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + } + { + name = "snafu"; + packageId = "snafu 0.8.9"; + } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "telemetry" "versioned" "webhook" ]; + } + { + name = "strum"; + packageId = "strum"; + features = [ "derive" ]; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "full" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + ]; + buildDependencies = [ + { + name = "built"; + packageId = "built"; + features = [ "chrono" "git2" ]; + } + ]; + devDependencies = [ + { + name = "rstest"; + packageId = "rstest"; + } + { + name = "serde_yaml"; + packageId = "serde_yaml"; + } + ]; + + }; + "stackable-certs" = rec { + crateName = "stackable-certs"; + version = "0.4.0"; + edition = "2024"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; + }; + libName = "stackable_certs"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "const-oid"; + packageId = "const-oid"; + features = [ "db" ]; + } + { + name = "ecdsa"; + packageId = "ecdsa"; + features = [ "digest" "pem" ]; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi"; + usesDefaultFeatures = false; + features = [ "schemars" "v1_34" ]; + } + { + name = "kube"; + packageId = "kube"; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + } + { + name = "p256"; + packageId = "p256"; + features = [ "ecdsa" ]; } { name = "rand"; - packageId = "rand"; + packageId = "rand 0.9.2"; } { - name = "serde"; - packageId = "serde"; - features = [ "derive" ]; + name = "rand_core"; + packageId = "rand_core 0.6.4"; } { - name = "serde_json"; - packageId = "serde_json"; + name = "rsa"; + packageId = "rsa"; + features = [ "sha2" ]; } { - name = "serde_yaml"; - packageId = "serde_yaml"; + name = "sha2"; + packageId = "sha2"; + features = [ "oid" ]; } { - name = "snafu"; - packageId = "snafu 0.8.9"; + name = "signature"; + packageId = "signature"; } { - name = "stackable-operator"; - packageId = "stackable-operator"; - features = [ "telemetry" "versioned" ]; + name = "snafu"; + packageId = "snafu 0.8.9"; } { - name = "strum"; - packageId = "strum"; - features = [ "derive" ]; + name = "stackable-shared"; + packageId = "stackable-shared"; } { name = "tokio"; packageId = "tokio"; - features = [ "full" ]; + features = [ "macros" "rt-multi-thread" "fs" ]; } { - name = "tracing"; - packageId = "tracing"; + name = "tokio-rustls"; + packageId = "tokio-rustls"; + optional = true; + usesDefaultFeatures = false; + features = [ "ring" "logging" "tls12" ]; } - ]; - buildDependencies = [ { - name = "built"; - packageId = "built"; - features = [ "chrono" "git2" ]; + name = "tracing"; + packageId = "tracing"; } - ]; - devDependencies = [ { - name = "rstest"; - packageId = "rstest"; + name = "x509-cert"; + packageId = "x509-cert"; + features = [ "builder" ]; } { - name = "serde_yaml"; - packageId = "serde_yaml"; + name = "zeroize"; + packageId = "zeroize"; } ]; - + features = { + "rustls" = [ "dep:tokio-rustls" ]; + }; + resolvedDefaultFeatures = [ "default" "rustls" ]; }; "stackable-operator" = rec { crateName = "stackable-operator"; @@ -8241,6 +9586,11 @@ rec { packageId = "stackable-versioned"; optional = true; } + { + name = "stackable-webhook"; + packageId = "stackable-webhook"; + optional = true; + } { name = "strum"; packageId = "strum"; @@ -8279,7 +9629,7 @@ rec { "versioned" = [ "dep:stackable-versioned" ]; "webhook" = [ "dep:stackable-webhook" ]; }; - resolvedDefaultFeatures = [ "clap" "default" "telemetry" "versioned" ]; + resolvedDefaultFeatures = [ "clap" "default" "telemetry" "versioned" "webhook" ]; }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; @@ -8615,6 +9965,125 @@ rec { } ]; + }; + "stackable-webhook" = rec { + crateName = "stackable-webhook"; + version = "0.7.1"; + edition = "2024"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; + }; + libName = "stackable_webhook"; + authors = [ + "Stackable GmbH " + ]; + dependencies = [ + { + name = "arc-swap"; + packageId = "arc-swap"; + } + { + name = "axum"; + packageId = "axum"; + features = [ "http2" ]; + } + { + name = "futures-util"; + packageId = "futures-util"; + } + { + name = "hyper"; + packageId = "hyper"; + features = [ "full" ]; + } + { + name = "hyper-util"; + packageId = "hyper-util"; + } + { + name = "k8s-openapi"; + packageId = "k8s-openapi"; + usesDefaultFeatures = false; + features = [ "schemars" "v1_34" ]; + } + { + name = "kube"; + packageId = "kube"; + usesDefaultFeatures = false; + features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + } + { + name = "opentelemetry"; + packageId = "opentelemetry"; + } + { + name = "opentelemetry-semantic-conventions"; + packageId = "opentelemetry-semantic-conventions"; + } + { + name = "rand"; + packageId = "rand 0.9.2"; + } + { + name = "serde_json"; + packageId = "serde_json"; + } + { + name = "snafu"; + packageId = "snafu 0.8.9"; + } + { + name = "stackable-certs"; + packageId = "stackable-certs"; + features = [ "rustls" ]; + } + { + name = "stackable-shared"; + packageId = "stackable-shared"; + } + { + name = "stackable-telemetry"; + packageId = "stackable-telemetry"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "macros" "rt-multi-thread" "fs" ]; + } + { + name = "tokio-rustls"; + packageId = "tokio-rustls"; + usesDefaultFeatures = false; + features = [ "ring" "logging" "tls12" ]; + } + { + name = "tower"; + packageId = "tower"; + features = [ "util" ]; + } + { + name = "tower-http"; + packageId = "tower-http"; + features = [ "trace" ]; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-opentelemetry"; + packageId = "tracing-opentelemetry"; + } + { + name = "x509-cert"; + packageId = "x509-cert"; + features = [ "builder" ]; + } + ]; + }; "strsim" = rec { crateName = "strsim"; @@ -8692,6 +10161,7 @@ rec { features = { "default" = [ "std" "i128" ]; }; + resolvedDefaultFeatures = [ "i128" ]; }; "syn 1.0.109" = rec { crateName = "syn"; @@ -9066,6 +10536,68 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "zerovec" ]; }; + "tls_codec" = rec { + crateName = "tls_codec"; + version = "0.4.2"; + edition = "2021"; + sha256 = "0sxzj0pdinn7fsc8aihqgfylsqi7z9jca0aqy3b8kfz28l9f1qhd"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "tls_codec_derive"; + packageId = "tls_codec_derive"; + optional = true; + } + { + name = "zeroize"; + packageId = "zeroize"; + usesDefaultFeatures = false; + features = [ "alloc" "zeroize_derive" ]; + } + ]; + features = { + "arbitrary" = [ "std" "dep:arbitrary" ]; + "conditional_deserialization" = [ "derive" "tls_codec_derive/conditional_deserialization" ]; + "default" = [ "std" ]; + "derive" = [ "tls_codec_derive" ]; + "serde" = [ "std" "dep:serde" ]; + "std" = [ "tls_codec_derive?/std" ]; + "tls_codec_derive" = [ "dep:tls_codec_derive" ]; + }; + resolvedDefaultFeatures = [ "derive" "std" "tls_codec_derive" ]; + }; + "tls_codec_derive" = rec { + crateName = "tls_codec_derive"; + version = "0.4.2"; + edition = "2021"; + sha256 = "1gglj5cxkpv7i3jazffksrfy5h5242kdvsqawjm2yh1915lpcbid"; + procMacro = true; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.108"; + features = [ "parsing" ]; + } + ]; + features = { + "conditional_deserialization" = [ "syn/full" ]; + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "default" "std" ]; + }; "tokio" = rec { crateName = "tokio"; version = "1.48.0"; @@ -9244,7 +10776,7 @@ rec { "tls12" = [ "rustls/tls12" ]; "zlib" = [ "rustls/zlib" ]; }; - resolvedDefaultFeatures = [ "logging" "tls12" ]; + resolvedDefaultFeatures = [ "logging" "ring" "tls12" ]; }; "tokio-stream" = rec { crateName = "tokio-stream"; @@ -12477,6 +14009,62 @@ rec { "either" = [ "dep:either" ]; }; }; + "x509-cert" = rec { + crateName = "x509-cert"; + version = "0.2.5"; + edition = "2021"; + sha256 = "155f42vm6m7phn8w7s2wmk9vli3ws45dqpk5z3jilw0a04syj08k"; + libName = "x509_cert"; + authors = [ + "RustCrypto Developers" + ]; + dependencies = [ + { + name = "const-oid"; + packageId = "const-oid"; + features = [ "db" ]; + } + { + name = "der"; + packageId = "der"; + features = [ "alloc" "derive" "flagset" "oid" ]; + } + { + name = "sha1"; + packageId = "sha1"; + optional = true; + } + { + name = "signature"; + packageId = "signature"; + optional = true; + features = [ "rand_core" ]; + } + { + name = "spki"; + packageId = "spki"; + features = [ "alloc" ]; + } + { + name = "tls_codec"; + packageId = "tls_codec"; + optional = true; + usesDefaultFeatures = false; + features = [ "derive" ]; + } + ]; + features = { + "arbitrary" = [ "dep:arbitrary" "std" "der/arbitrary" "spki/arbitrary" ]; + "builder" = [ "std" "sha1/default" "signature" ]; + "default" = [ "pem" "std" ]; + "pem" = [ "der/pem" "spki/pem" ]; + "sct" = [ "dep:tls_codec" ]; + "sha1" = [ "dep:sha1" ]; + "signature" = [ "dep:signature" ]; + "std" = [ "const-oid/std" "der/std" "spki/std" "tls_codec?/std" ]; + }; + resolvedDefaultFeatures = [ "builder" "default" "pem" "sha1" "signature" "std" ]; + }; "xml" = rec { crateName = "xml"; version = "1.0.1"; @@ -12693,6 +14281,13 @@ rec { authors = [ "The RustCrypto Project Developers" ]; + dependencies = [ + { + name = "zeroize_derive"; + packageId = "zeroize_derive"; + optional = true; + } + ]; features = { "default" = [ "alloc" ]; "derive" = [ "zeroize_derive" ]; @@ -12700,7 +14295,33 @@ rec { "std" = [ "alloc" ]; "zeroize_derive" = [ "dep:zeroize_derive" ]; }; - resolvedDefaultFeatures = [ "alloc" "default" ]; + resolvedDefaultFeatures = [ "alloc" "default" "zeroize_derive" ]; + }; + "zeroize_derive" = rec { + crateName = "zeroize_derive"; + version = "1.4.2"; + edition = "2021"; + sha256 = "0sczjlqjdmrp3wn62g7mw6p438c9j4jgp2f9zamd56991mdycdnf"; + procMacro = true; + authors = [ + "The RustCrypto Project Developers" + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.108"; + features = [ "full" "extra-traits" "visit" ]; + } + ]; + }; "zerotrie" = rec { crateName = "zerotrie"; diff --git a/Cargo.toml b/Cargo.toml index be57cf89..52f71311 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned", "webhook"], tag = "stackable-operator-0.100.3" } anyhow = "1.0" base64 = "0.22" diff --git a/crate-hashes.json b/crate-hashes.json index 299e3d15..ac30e3aa 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,11 +1,13 @@ { "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-certs@0.4.0": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-webhook@0.7.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 76c505e3..18f1c3c7 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -16,6 +16,3190 @@ spec: singular: airflowcluster scope: Namespaced versions: + - additionalPrinterColumns: [] + name: v1alpha2 + schema: + openAPIV3Schema: + description: Auto-generated derived type for AirflowClusterSpec via `CustomResource` + properties: + spec: + description: |- + An Airflow cluster stacklet. This resource is managed by the Stackable operator for Apache Airflow. + Find more information on how to use it and the resources that the operator generates in the + [operator documentation](https://docs.stackable.tech/home/nightly/airflow/). + + The CRD contains three roles: webserver, scheduler and worker/celeryExecutor. + You can use either the celeryExecutor or the kubernetesExecutor. + oneOf: + - required: + - celeryExecutors + - required: + - kubernetesExecutors + properties: + celeryExecutors: + description: |- + The celery executor. + Deployed with an explicit number of replicas. + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + clusterConfig: + description: |- + Configuration that applies to all roles and role groups. + This includes settings for authentication, git sync, service exposition and volumes, among other things. + properties: + authentication: + default: [] + items: + properties: + authenticationClass: + description: Name of the [AuthenticationClass](https://docs.stackable.tech/home/nightly/concepts/authentication) used to authenticate users + type: string + oidc: + description: This field contains OIDC-specific configuration. It is only required in case OIDC is used. + nullable: true + properties: + clientCredentialsSecret: + description: |- + A reference to the OIDC client credentials secret. The secret contains + the client id and secret. + type: string + extraScopes: + default: [] + description: An optional list of extra scopes which get merged with the scopes defined in the AuthenticationClass + items: + type: string + type: array + required: + - clientCredentialsSecret + type: object + syncRolesAt: + default: Registration + description: |- + If we should replace ALL the user's roles each login, or only on registration. + Gets mapped to `AUTH_ROLES_SYNC_AT_LOGIN` + enum: + - Registration + - Login + type: string + userRegistration: + default: true + description: |- + Allow users who are not already in the FAB DB. + Gets mapped to `AUTH_USER_REGISTRATION` + type: boolean + userRegistrationRole: + default: Public + description: |- + This role will be given in addition to any AUTH_ROLES_MAPPING. + Gets mapped to `AUTH_USER_REGISTRATION_ROLE` + type: string + required: + - authenticationClass + type: object + type: array + authorization: + description: |- + Authorization options. + Learn more in the [Airflow authorization usage guide](https://docs.stackable.tech/home/nightly/airflow/usage-guide/security#_authorization). + nullable: true + properties: + opa: + description: |- + Configure the OPA stacklet [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + and the name of the Rego package containing your authorization rules. + Consult the [OPA authorization documentation](https://docs.stackable.tech/home/nightly/concepts/opa) + to learn how to deploy Rego authorization rules with OPA. + nullable: true + properties: + cache: + default: + entryTimeToLive: 30s + maxEntries: 10000 + description: Least Recently Used (LRU) cache with per-entry time-to-live (TTL) value. + properties: + entryTimeToLive: + default: 30s + description: Time to live per entry + type: string + maxEntries: + default: 10000 + description: |- + Maximum number of entries in the cache; If this threshold is reached then the least + recently used item is removed. + format: uint32 + minimum: 0.0 + type: integer + type: object + configMapName: + description: |- + The [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) + for the OPA stacklet that should be used for authorization requests. + type: string + package: + description: The name of the Rego package containing the Rego rules for the product. + nullable: true + type: string + required: + - configMapName + type: object + type: object + credentialsSecret: + description: |- + The name of the Secret object containing the admin user credentials and database connection details. + Read the + [getting started guide first steps](https://docs.stackable.tech/home/nightly/airflow/getting_started/first_steps) + to find out more. + type: string + dagsGitSync: + default: [] + description: |- + The `gitSync` settings allow configuring DAGs to mount via `git-sync`. + Learn more in the + [mounting DAGs documentation](https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_via_git_sync). + items: + properties: + branch: + default: main + description: |- + The branch to clone; defaults to `main`. + + Since git-sync v4.x.x this field is mapped to the flag `--ref`. + type: string + credentials: + description: An optional secret used for git access. + nullable: true + oneOf: + - required: + - basicAuthSecretName + - required: + - sshPrivateKeySecretName + properties: + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. + + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + type: object + depth: + default: 1 + description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. + format: uint32 + minimum: 0.0 + type: integer + gitFolder: + default: / + description: |- + Location in the Git repository containing the resource; defaults to the root folder. + + It can optionally start with `/`, however, no trailing slash is recommended. + An empty string (``) or slash (`/`) corresponds to the root folder in Git. + type: string + gitSyncConf: + additionalProperties: + type: string + default: {} + description: |- + A map of optional configuration settings that are listed in the git-sync [documentation]. + + Also read the git-sync [example] in our documentation. These settings are not verified. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example + type: object + repo: + description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' + format: uri + type: string + wait: + default: 20s + description: |- + The synchronization interval, e.g. `20s` or `5m`; defaults to `20s`. + + Since git-sync v4.x.x this field is mapped to the flag `--period`. + type: string + required: + - repo + type: object + type: array + databaseInitialization: + default: + enabled: true + description: Settings related to the database initialization routines (which are always executed by default). + properties: + enabled: + default: true + description: |- + Whether to execute the database initialization routines (a combination of database initialization, upgrade and migration depending on the Airflow version). Defaults to true to be backwards-compatible. + WARNING: setting this to false is *unsupported* as subsequent updates to the Airflow cluster may result in broken behaviour due to inconsistent metadata! + Do not change the default unless you know what you are doing! + type: boolean + type: object + exposeConfig: + default: false + description: for internal use only - not for production use. + type: boolean + loadExamples: + default: false + description: |- + Whether to load example DAGs or not; defaults to false. The examples are used in the + [getting started guide](https://docs.stackable.tech/home/nightly/airflow/getting_started/). + type: boolean + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + nullable: true + type: string + volumeMounts: + default: [] + description: Additional volumes to mount. Use together with `volumes` to define volumes. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + volumes: + default: [] + description: Additional volumes to define. Use together with `volumeMounts` to mount the volumes. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - credentialsSecret + type: object + clusterOperation: + default: + reconciliationPaused: false + stopped: false + description: |- + [Cluster operations](https://docs.stackable.tech/home/nightly/concepts/operations/cluster_operations) + properties, allow stopping the product instance as well as pausing reconciliation. + properties: + reconciliationPaused: + default: false + description: |- + Flag to stop cluster reconciliation by the operator. This means that all changes in the + custom resource spec are ignored until this flag is set to false or removed. The operator + will however still watch the deployed resources at the time and update the custom resource + status field. + If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over + `stopped` and stop the reconciliation immediately. + type: boolean + stopped: + default: false + description: |- + Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, + ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 + and therefore stopped and removed. + If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation + and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. + type: boolean + type: object + dagProcessors: + description: The `dagProcessors` role runs the DAG processor routine for DAG preparation. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Overwrite the docker image. + Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: Name of the docker repo, e.g. `oci.stackable.tech/sdp` + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + If not specified, the operator will use its own version, e.g. `23.4.1`. + When using a nightly operator or a pr version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + kubernetesExecutors: + description: With the Kuberentes executor, executor Pods are created on demand. + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + schedulers: + description: |- + The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database. + Jobs are scheduled on the workers/executors. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + triggerers: + description: The `triggerers` role runs the triggerer process for use with deferrable DAG operators. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + webservers: + description: The `webservers` role provides the main UI for user interaction. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + type: string + podDisruptionBudget: + default: + enabled: true + maxUnavailable: null + description: |- + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + additionalProperties: + additionalProperties: + type: string + type: object + default: {} + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + required: + - clusterConfig + - image + type: object + status: + nullable: true + properties: + conditions: + default: [] + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + nullable: true + type: string + message: + description: A human readable message indicating details about the transition. + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - 'True' + - 'False' + - Unknown + type: string + type: + description: Type of deployment condition. + enum: + - Available + - Degraded + - Progressing + - ReconciliationPaused + - Stopped + type: string + required: + - status + - type + type: object + type: array + type: object + required: + - spec + title: AirflowCluster + type: object + served: true + storage: true + subresources: + status: {} - additionalPrinterColumns: [] name: v1alpha1 schema: @@ -3197,6 +6381,6 @@ spec: title: AirflowCluster type: object served: true - storage: true + storage: false subresources: status: {} diff --git a/deploy/helm/airflow-operator/templates/deployment.yaml b/deploy/helm/airflow-operator/templates/deployment.yaml index 4601af6f..fe17287d 100644 --- a/deploy/helm/airflow-operator/templates/deployment.yaml +++ b/deploy/helm/airflow-operator/templates/deployment.yaml @@ -4,6 +4,9 @@ kind: Deployment metadata: name: {{ include "operator.fullname" . }}-deployment labels: + {{- if .Values.maintenance.customResourceDefinitions.maintain }} + webhook.stackable.tech/conversion: enabled + {{- end }} {{- include "operator.labels" . | nindent 4 }} spec: replicas: 1 @@ -79,6 +82,7 @@ spec: value: {{ .Values.kubernetesClusterDomain | quote }} {{- end }} {{- include "telemetry.envVars" . | nindent 12 }} + {{- include "maintenance.envVars" . | nindent 12 }} volumes: - name: config-spec configMap: diff --git a/deploy/helm/airflow-operator/templates/roles.yaml b/deploy/helm/airflow-operator/templates/roles.yaml index 4c2246e5..70e5641c 100644 --- a/deploy/helm/airflow-operator/templates/roles.yaml +++ b/deploy/helm/airflow-operator/templates/roles.yaml @@ -91,6 +91,12 @@ rules: - customresourcedefinitions verbs: - get + # Required to maintain the CRD. The operator needs to do this, as it needs to enter e.g. it's + # generated certificate in the conversion webhook. + {{- if .Values.maintenance.customResourceDefinitions.maintain }} + - create + - patch + {{- end }} - apiGroups: - listeners.stackable.tech resources: @@ -111,6 +117,9 @@ rules: - list - patch - watch + {{- if .Values.maintenance.customResourceDefinitions.maintain }} + - create + {{- end }} - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/deploy/helm/airflow-operator/templates/service.yaml b/deploy/helm/airflow-operator/templates/service.yaml index 0cb05696..8dcef00f 100644 --- a/deploy/helm/airflow-operator/templates/service.yaml +++ b/deploy/helm/airflow-operator/templates/service.yaml @@ -1,4 +1,4 @@ - +{{- if .Values.maintenance.customResourceDefinitions.maintain }} --- apiVersion: v1 kind: Service @@ -11,9 +11,11 @@ metadata: {{- include "operator.labels" . | nindent 4 }} spec: selector: + webhook.stackable.tech/conversion: enabled {{- include "operator.selectorLabels" . | nindent 6 }} ports: - name: conversion-webhook protocol: TCP port: 8443 targetPort: 8443 +{{- end }} diff --git a/deploy/helm/airflow-operator/values.yaml b/deploy/helm/airflow-operator/values.yaml index 37a353d5..89b8a10d 100644 --- a/deploy/helm/airflow-operator/values.yaml +++ b/deploy/helm/airflow-operator/values.yaml @@ -46,6 +46,15 @@ affinity: {} # See the https://docs.stackable.tech/home/stable/guides/kubernetes-cluster-domain guide for details. # kubernetesClusterDomain: my-cluster.local +maintenance: + endOfSupportCheck: + enabled: true + # mode: offline + # interval: 24h + + customResourceDefinitions: + maintain: true + # See all available options and detailed explanations about the concept here: # https://docs.stackable.tech/home/stable/concepts/telemetry/ telemetry: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 432c2648..c7389e0a 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -1,4 +1,4 @@ -//! Ensures that `Pod`s are configured and running for each [`v1alpha1::AirflowCluster`] +//! Ensures that `Pod`s are configured and running for each [`v1alpha2::AirflowCluster`] use std::{ collections::{BTreeMap, BTreeSet, HashMap}, io::Write, @@ -97,7 +97,7 @@ use crate::{ FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY, create_random_secret, }, - v1alpha1, + v1alpha2, }, env_vars::{self, build_airflow_template_envs}, operations::{ @@ -137,19 +137,19 @@ pub enum Error { #[snafu(display("failed to apply Service for {rolegroup}"))] ApplyRoleGroupService { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply ConfigMap for {rolegroup}"))] ApplyRoleGroupConfig { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to apply StatefulSet for {rolegroup}"))] ApplyRoleGroupStatefulSet { source: stackable_operator::cluster_resources::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("invalid product config"))] @@ -200,13 +200,13 @@ pub enum Error { #[snafu(display("failed to build config file for {rolegroup}"))] BuildRoleGroupConfigFile { source: FlaskAppConfigWriterError, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to build ConfigMap for {rolegroup}"))] BuildRoleGroupConfig { source: stackable_operator::builder::configmap::Error, - rolegroup: RoleGroupRef, + rolegroup: RoleGroupRef, }, #[snafu(display("failed to resolve and merge config for role and role group"))] @@ -365,7 +365,7 @@ impl ReconcilerError for Error { } pub async fn reconcile_airflow( - airflow: Arc>, + airflow: Arc>, ctx: Arc, ) -> Result { tracing::info!("Starting reconcile"); @@ -682,7 +682,7 @@ pub async fn reconcile_airflow( #[allow(clippy::too_many_arguments)] async fn build_executor_template( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, common_config: &CommonConfiguration, resolved_product_image: &ResolvedProductImage, authentication_config: &AirflowClientAuthenticationDetailsResolved, @@ -750,9 +750,9 @@ async fn build_executor_template( /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, resolved_product_image: &ResolvedProductImage, - rolegroup: &RoleGroupRef, + rolegroup: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, @@ -863,9 +863,9 @@ fn build_rolegroup_config_map( } fn build_rolegroup_metadata( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, resolved_product_image: &&ResolvedProductImage, - rolegroup: &&RoleGroupRef, + rolegroup: &&RoleGroupRef, prometheus_label: Label, name: String, ) -> Result { @@ -888,8 +888,8 @@ fn build_rolegroup_metadata( } pub fn build_group_listener( - airflow: &v1alpha1::AirflowCluster, - object_labels: ObjectLabels, + airflow: &v1alpha2::AirflowCluster, + object_labels: ObjectLabels, listener_class: String, listener_group_name: String, ) -> Result { @@ -924,10 +924,10 @@ fn listener_ports() -> Vec { /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. #[allow(clippy::too_many_arguments)] fn build_server_rolegroup_statefulset( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, resolved_product_image: &ResolvedProductImage, airflow_role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, rolegroup_config: &HashMap>, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, @@ -1247,14 +1247,14 @@ fn build_logging_container( #[allow(clippy::too_many_arguments)] fn build_executor_template_config_map( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, resolved_product_image: &ResolvedProductImage, authentication_config: &AirflowClientAuthenticationDetailsResolved, sa_name: &str, merged_executor_config: &ExecutorConfig, env_overrides: &HashMap, pod_overrides: &PodTemplateSpec, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, ) -> Result { let mut pb = PodBuilder::new(); @@ -1386,7 +1386,7 @@ fn build_executor_template_config_map( } pub fn error_policy( - _obj: Arc>, + _obj: Arc>, error: &Error, _ctx: Arc, ) -> Action { diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index 53f303b9..74cc2ee7 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -55,7 +55,7 @@ mod tests { role_utils::RoleGroupRef, }; - use crate::crd::{AirflowExecutor, AirflowRole, v1alpha1}; + use crate::crd::{AirflowExecutor, AirflowRole, v1alpha2}; #[rstest] #[case(AirflowRole::Worker)] @@ -63,7 +63,7 @@ mod tests { #[case(AirflowRole::Webserver)] fn test_affinity_defaults(#[case] role: AirflowRole) { let cluster = " - apiVersion: airflow.stackable.tech/v1alpha1 + apiVersion: airflow.stackable.tech/v1alpha2 kind: AirflowCluster metadata: name: airflow @@ -87,7 +87,7 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: v1alpha1::AirflowCluster = + let airflow: v1alpha2::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let rolegroup_ref = RoleGroupRef { @@ -157,7 +157,7 @@ mod tests { #[test] fn test_executor_affinity_defaults() { let cluster = " - apiVersion: airflow.stackable.tech/v1alpha1 + apiVersion: airflow.stackable.tech/v1alpha2 kind: AirflowCluster metadata: name: airflow @@ -178,7 +178,7 @@ mod tests { "; let deserializer = serde_yaml::Deserializer::from_str(cluster); - let airflow: v1alpha1::AirflowCluster = + let airflow: v1alpha2::AirflowCluster = serde_yaml::with::singleton_map_recursive::deserialize(deserializer).unwrap(); let expected: StackableAffinity = StackableAffinity { diff --git a/rust/operator-binary/src/crd/authorization.rs b/rust/operator-binary/src/crd/authorization.rs index 62e77323..d71ae370 100644 --- a/rust/operator-binary/src/crd/authorization.rs +++ b/rust/operator-binary/src/crd/authorization.rs @@ -1,6 +1,6 @@ use stackable_operator::{client::Client, commons::opa::OpaApiVersion, shared::time::Duration}; -use crate::crd::{AirflowAuthorization, AirflowOpaConfig, v1alpha1}; +use crate::crd::{AirflowAuthorization, AirflowOpaConfig, v1alpha2}; pub struct AirflowAuthorizationResolved { pub opa: Option, @@ -9,7 +9,7 @@ pub struct AirflowAuthorizationResolved { impl AirflowAuthorizationResolved { pub async fn from_authorization_config( client: &Client, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, authorization: &Option, ) -> Result { let opa = if let Some(AirflowAuthorization { @@ -33,7 +33,7 @@ pub struct OpaConfigResolved { impl OpaConfigResolved { pub async fn from_opa_config( client: &Client, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, airflow_opa_config: &AirflowOpaConfig, ) -> Result { let connection_string = airflow_opa_config diff --git a/rust/operator-binary/src/crd/internal_secret.rs b/rust/operator-binary/src/crd/internal_secret.rs index bf2eeb0a..b6d6a678 100644 --- a/rust/operator-binary/src/crd/internal_secret.rs +++ b/rust/operator-binary/src/crd/internal_secret.rs @@ -9,7 +9,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::{airflow_controller::AIRFLOW_CONTROLLER_NAME, crd::v1alpha1}; +use crate::{airflow_controller::AIRFLOW_CONTROLLER_NAME, crd::v1alpha2}; // Used for env-vars: AIRFLOW__WEBSERVER__SECRET_KEY, AIRFLOW__API__SECRET_KEY // N.B. AIRFLOW__WEBSERVER__SECRET_KEY is deprecated as of 3.0.2. @@ -63,7 +63,7 @@ pub async fn create_random_secret( secret_name: &str, secret_key: &str, secret_byte_size: usize, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, client: &Client, ) -> Result<()> { let mut internal_secret = BTreeMap::new(); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 41639513..2944a27f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -19,7 +19,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - crd::git_sync, + crd::git_sync::{self, credentials_secret_to_basic_auth, credentials_to_secret}, deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, @@ -64,6 +64,7 @@ pub mod authorization; pub mod internal_secret; pub const APP_NAME: &str = "airflow"; +pub const FIELD_MANAGER: &str = "airflow-operator"; pub const OPERATOR_NAME: &str = "airflow.stackable.tech"; pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const STACKABLE_LOG_DIR: &str = "/stackable/log"; @@ -171,6 +172,7 @@ impl FlaskAppConfigOptions for AirflowConfigOptions { #[versioned( version(name = "v1alpha1"), + version(name = "v1alpha2"), crates( kube_core = "stackable_operator::kube::core", kube_client = "stackable_operator::kube::client", @@ -204,7 +206,7 @@ pub mod versioned { /// Configuration that applies to all roles and role groups. /// This includes settings for authentication, git sync, service exposition and volumes, among other things. - pub cluster_config: v1alpha1::AirflowClusterConfig, + pub cluster_config: v1alpha2::AirflowClusterConfig, // no doc string - See ClusterOperation struct #[serde(default)] @@ -252,6 +254,12 @@ pub mod versioned { /// Learn more in the /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). #[serde(default)] + #[versioned(changed( + since = "v1alpha2", + from_type = "Vec", + upgrade_with = gitsync_v1_to_v2, + downgrade_with = gitsync_v2_to_v1 + ))] pub dags_git_sync: Vec, /// for internal use only - not for production use. @@ -297,6 +305,40 @@ pub mod versioned { } } +pub fn gitsync_v1_to_v2( + input: Vec, +) -> Vec { + input + .iter() + .map(|g| git_sync::v1alpha2::GitSync { + credentials: credentials_secret_to_basic_auth(g.credentials_secret.clone()), + repo: g.repo.clone(), + branch: g.branch.clone(), + git_folder: g.git_folder.clone(), + depth: g.depth, + wait: g.wait, + git_sync_conf: g.git_sync_conf.clone(), + }) + .collect() +} + +pub fn gitsync_v2_to_v1( + input: Vec, +) -> Vec { + input + .iter() + .map(|g| git_sync::v1alpha1::GitSync { + credentials_secret: credentials_to_secret(g.credentials.clone()), + repo: g.repo.clone(), + branch: g.branch.clone(), + git_folder: g.git_folder.clone(), + depth: g.depth, + wait: g.wait, + git_sync_conf: g.git_sync_conf.clone(), + }) + .collect() +} + #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct DatabaseInitializationConfig { @@ -339,7 +381,7 @@ pub struct AirflowClusterStatus { pub conditions: Vec, } -impl HasStatusCondition for v1alpha1::AirflowCluster { +impl HasStatusCondition for v1alpha2::AirflowCluster { fn conditions(&self) -> Vec { match &self.status { Some(status) => status.conditions.clone(), @@ -348,7 +390,7 @@ impl HasStatusCondition for v1alpha1::AirflowCluster { } } -impl v1alpha1::AirflowCluster { +impl v1alpha2::AirflowCluster { /// The name of the group-listener provided for a specific role. /// Webservers will use this group listener so that only one load balancer /// is needed for that role. @@ -404,7 +446,7 @@ impl v1alpha1::AirflowCluster { pub fn merged_config( &self, role: &AirflowRole, - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, ) -> Result { // Initialize the result with all default values as baseline let conf_defaults = AirflowConfig::default_config(&self.name_any(), role); @@ -586,7 +628,7 @@ impl AirflowRole { /// if authentication is enabled. pub fn get_commands( &self, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, auth_config: &AirflowClientAuthenticationDetailsResolved, resolved_product_image: &ResolvedProductImage, ) -> Vec { @@ -778,7 +820,7 @@ impl AirflowRole { roles } - pub fn listener_class_name(&self, airflow: &v1alpha1::AirflowCluster) -> Option { + pub fn listener_class_name(&self, airflow: &v1alpha2::AirflowCluster) -> Option { match self { Self::Webserver => airflow .spec @@ -791,7 +833,7 @@ impl AirflowRole { pub fn role_config( &self, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, ) -> Result, Error> { let role = self.to_string(); let roles = AirflowRole::roles(); @@ -973,7 +1015,7 @@ impl AirflowConfig { } impl Configuration for AirflowConfigFragment { - type Configurable = v1alpha1::AirflowCluster; + type Configurable = v1alpha2::AirflowCluster; fn compute_env( &self, @@ -1090,12 +1132,12 @@ pub fn build_recommended_labels<'a, T>( mod tests { use stackable_operator::commons::product_image_selection::ResolvedProductImage; - use crate::v1alpha1::AirflowCluster; + use crate::v1alpha2::AirflowCluster; #[test] fn test_cluster_config() { let cluster = " - apiVersion: airflow.stackable.tech/v1alpha1 + apiVersion: airflow.stackable.tech/v1alpha2 kind: AirflowCluster metadata: name: airflow diff --git a/rust/operator-binary/src/env_vars.rs b/rust/operator-binary/src/env_vars.rs index d2a75921..1ae60bd6 100644 --- a/rust/operator-binary/src/env_vars.rs +++ b/rust/operator-binary/src/env_vars.rs @@ -24,7 +24,7 @@ use crate::{ internal_secret::{ FERNET_KEY_SECRET_KEY, INTERNAL_SECRET_SECRET_KEY, JWT_SECRET_SECRET_KEY, }, - v1alpha1, + v1alpha2, }, util::{env_var_from_secret, role_service_name}, }; @@ -74,7 +74,7 @@ pub enum Error { /// used which is defined in [`build_airflow_template_envs`]). #[allow(clippy::too_many_arguments)] pub fn build_airflow_statefulset_envs( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, airflow_role: &AirflowRole, rolegroup_config: &HashMap>, executor: &AirflowExecutor, @@ -369,7 +369,7 @@ fn static_envs( /// Return environment variables to be applied to the configuration map used in conjunction with /// the `kubernetesExecutor` worker. pub fn build_airflow_template_envs( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, env_overrides: &HashMap, config: &ExecutorConfig, git_sync_resources: &git_sync::v1alpha2::GitSyncResources, @@ -464,7 +464,7 @@ pub fn build_airflow_template_envs( } fn add_version_specific_env_vars( - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, airflow_role: &AirflowRole, resolved_product_image: &ResolvedProductImage, env: &mut BTreeMap, @@ -622,7 +622,7 @@ fn authorization_env_vars( env } -fn execution_server_env_vars(airflow: &v1alpha1::AirflowCluster) -> BTreeMap { +fn execution_server_env_vars(airflow: &v1alpha2::AirflowCluster) -> BTreeMap { let mut env: BTreeMap = BTreeMap::new(); if let Some(name) = airflow.metadata.name.as_ref() { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b4f4d9ee..42f0acb1 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -4,8 +4,9 @@ use std::sync::Arc; +use anyhow::anyhow; use clap::Parser; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt}; use stackable_operator::{ YamlSchema, cli::{Command, RunArguments}, @@ -32,7 +33,8 @@ use stackable_operator::{ use crate::{ airflow_controller::AIRFLOW_FULL_CONTROLLER_NAME, - crd::{AirflowCluster, AirflowClusterVersion, OPERATOR_NAME, v1alpha1}, + crd::{AirflowCluster, AirflowClusterVersion, OPERATOR_NAME, v1alpha2}, + webhooks::conversion::create_webhook_and_maintainer, }; mod airflow_controller; @@ -44,6 +46,7 @@ mod operations; mod product_logging; mod service; mod util; +mod webhooks; mod built_info { include!(concat!(env!("OUT_DIR"), "/built.rs")); @@ -62,13 +65,13 @@ async fn main() -> anyhow::Result<()> { match opts.cmd { Command::Crd => { - AirflowCluster::merged_crd(AirflowClusterVersion::V1Alpha1)? + AirflowCluster::merged_crd(AirflowClusterVersion::V1Alpha2)? .print_yaml_schema(built_info::PKG_VERSION, SerializeOptions::default())?; } Command::Run(RunArguments { product_config, watch_namespace, - operator_environment: _, + operator_environment, maintenance, common, }) => { @@ -114,7 +117,7 @@ async fn main() -> anyhow::Result<()> { )); let airflow_controller = Controller::new( - watch_namespace.get_api::>(&client), + watch_namespace.get_api::>(&client), watcher::Config::default(), ); @@ -139,7 +142,7 @@ async fn main() -> anyhow::Result<()> { .state() .into_iter() .filter( - move |airflow: &Arc>| { + move |airflow: &Arc>| { references_authentication_class(airflow, &authentication_class) }, ) @@ -184,7 +187,28 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); - futures::try_join!(airflow_controller, eos_checker)?; + let (conversion_webhook, crd_maintainer, _initial_reconcile_rx) = + create_webhook_and_maintainer( + &operator_environment, + maintenance.disable_crd_maintenance, + client.as_kube_client(), + ) + .await?; + + let conversion_webhook = conversion_webhook + .run() + .map_err(|err| anyhow!(err).context("failed to run conversion webhook")); + + let crd_maintainer = crd_maintainer + .run() + .map_err(|err| anyhow!(err).context("failed to run CRD maintainer")); + + futures::try_join!( + airflow_controller, + conversion_webhook, + crd_maintainer, + eos_checker + )?; } } @@ -192,7 +216,7 @@ async fn main() -> anyhow::Result<()> { } fn references_authentication_class( - airflow: &DeserializeGuard, + airflow: &DeserializeGuard, authentication_class: &DeserializeGuard, ) -> bool { let Ok(airflow) = &airflow.0 else { @@ -209,7 +233,7 @@ fn references_authentication_class( } fn references_config_map( - airflow: &DeserializeGuard, + airflow: &DeserializeGuard, config_map: &DeserializeGuard, ) -> bool { let Ok(airflow) = &airflow.0 else { diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index 8b471082..d70d9d61 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -6,7 +6,7 @@ use stackable_operator::{ use crate::{ airflow_controller::AIRFLOW_CONTROLLER_NAME, - crd::{APP_NAME, AirflowExecutor, AirflowRole, OPERATOR_NAME, v1alpha1}, + crd::{APP_NAME, AirflowExecutor, AirflowRole, OPERATOR_NAME, v1alpha2}, }; #[derive(Snafu, Debug)] @@ -25,7 +25,7 @@ pub enum Error { pub async fn add_pdbs( pdb: &PdbConfig, - airflow: &v1alpha1::AirflowCluster, + airflow: &v1alpha2::AirflowCluster, role: &AirflowRole, client: &Client, cluster_resources: &mut ClusterResources, diff --git a/rust/operator-binary/src/service.rs b/rust/operator-binary/src/service.rs index ff313e7a..71347956 100644 --- a/rust/operator-binary/src/service.rs +++ b/rust/operator-binary/src/service.rs @@ -8,7 +8,7 @@ use stackable_operator::{ role_utils::RoleGroupRef, }; -use crate::crd::{HTTP_PORT, HTTP_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha1}; +use crate::crd::{HTTP_PORT, HTTP_PORT_NAME, METRICS_PORT, METRICS_PORT_NAME, v1alpha2}; pub const METRICS_SERVICE_SUFFIX: &str = "metrics"; pub const HEADLESS_SERVICE_SUFFIX: &str = "headless"; @@ -34,9 +34,9 @@ pub enum Error { /// The rolegroup headless [`Service`] is a service that allows direct access to the instances of a certain rolegroup /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. pub fn build_rolegroup_headless_service( - airflow: &v1alpha1::AirflowCluster, - rolegroup_ref: &RoleGroupRef, - object_labels: ObjectLabels, + airflow: &v1alpha2::AirflowCluster, + rolegroup_ref: &RoleGroupRef, + object_labels: ObjectLabels, selector: BTreeMap, ) -> Result { let ports = headless_service_ports(); @@ -71,9 +71,9 @@ pub fn build_rolegroup_headless_service( /// The rolegroup metrics [`Service`] is a service that exposes metrics and a prometheus scraping label. pub fn build_rolegroup_metrics_service( - airflow: &v1alpha1::AirflowCluster, - rolegroup_ref: &RoleGroupRef, - object_labels: ObjectLabels, + airflow: &v1alpha2::AirflowCluster, + rolegroup_ref: &RoleGroupRef, + object_labels: ObjectLabels, selector: BTreeMap, ) -> Result { let ports = metrics_service_ports(); @@ -107,7 +107,7 @@ pub fn build_rolegroup_metrics_service( } pub fn stateful_set_service_name( - rolegroup_ref: &RoleGroupRef, + rolegroup_ref: &RoleGroupRef, ) -> Option { Some(rolegroup_headless_service_name( &rolegroup_ref.object_name(), diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs new file mode 100644 index 00000000..84bbeb6c --- /dev/null +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -0,0 +1,53 @@ +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + cli::OperatorEnvironmentOptions, + kube::{Client, core::crd::MergeError}, + webhook::{ + maintainer::CustomResourceDefinitionMaintainer, + servers::{ConversionWebhookError, ConversionWebhookServer}, + }, +}; +use tokio::sync::oneshot; + +use crate::crd::{AirflowCluster, AirflowClusterVersion, FIELD_MANAGER}; + +/// Contains errors which can be encountered when creating the conversion webhook server and the +/// CRD maintainer. +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("failed to merge CRD"))] + MergeCrd { source: MergeError }, + + #[snafu(display("failed to create conversion webhook server"))] + CreateConversionWebhook { source: ConversionWebhookError }, +} + +/// Creates and returns a [`ConversionWebhookServer`] and a [`CustomResourceDefinitionMaintainer`]. +pub async fn create_webhook_and_maintainer<'a>( + operator_environment: &'a OperatorEnvironmentOptions, + disable_crd_maintenance: bool, + client: Client, +) -> Result< + ( + ConversionWebhookServer, + CustomResourceDefinitionMaintainer<'a>, + oneshot::Receiver<()>, + ), + Error, +> { + let crds_and_handlers = [( + AirflowCluster::merged_crd(AirflowClusterVersion::V1Alpha2).context(MergeCrdSnafu)?, + AirflowCluster::try_convert as fn(_) -> _, + )]; + + ConversionWebhookServer::with_maintainer( + crds_and_handlers, + &operator_environment.operator_service_name, + &operator_environment.operator_namespace, + FIELD_MANAGER, + disable_crd_maintenance, + client, + ) + .await + .context(CreateConversionWebhookSnafu) +} diff --git a/rust/operator-binary/src/webhooks/mod.rs b/rust/operator-binary/src/webhooks/mod.rs new file mode 100644 index 00000000..11eb2fc0 --- /dev/null +++ b/rust/operator-binary/src/webhooks/mod.rs @@ -0,0 +1 @@ +pub mod conversion; diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 7b2b888b..d6cd1bef 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -82,8 +82,9 @@ spec: {% endif %} {% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags - credentials: - basicAuthSecretName: git-credentials + credentialsSecret: git-credentials + #credentials: + # basicAuthSecretName: git-credentials {% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) From 80ccc2d881886c7fcf2d159cdcc394306e3032c7 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 22 Dec 2025 10:33:20 +0100 Subject: [PATCH 10/15] remove version prefix from airflow config --- deploy/helm/airflow-operator/crds/crds.yaml | 28 ++------------------- rust/operator-binary/src/crd/mod.rs | 2 +- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 18f1c3c7..ceb51459 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -3851,34 +3851,10 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentials: + credentialsSecret: description: An optional secret used for git access. nullable: true - oneOf: - - required: - - basicAuthSecretName - - required: - - sshPrivateKeySecretName - properties: - basicAuthSecretName: - description: |- - The name of the Secret used to access the repository via Basic Authentication if it is not public. - - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - sshPrivateKeySecretName: - description: |- - The name of the Secret used for SSH access to the repository. - - The referenced Secret must include two fields: `key` and `knownHosts`. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - type: object + type: string depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 2944a27f..251593ef 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -206,7 +206,7 @@ pub mod versioned { /// Configuration that applies to all roles and role groups. /// This includes settings for authentication, git sync, service exposition and volumes, among other things. - pub cluster_config: v1alpha2::AirflowClusterConfig, + pub cluster_config: AirflowClusterConfig, // no doc string - See ClusterOperation struct #[serde(default)] From c3f5a0ff56336d19cda0dbd485b0bd395e0a801f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 22 Dec 2025 13:28:16 +0100 Subject: [PATCH 11/15] use hint instead of explicit functions --- .../airflow-operator/templates/service.yaml | 6 +-- rust/operator-binary/src/crd/mod.rs | 54 ++++--------------- 2 files changed, 12 insertions(+), 48 deletions(-) diff --git a/deploy/helm/airflow-operator/templates/service.yaml b/deploy/helm/airflow-operator/templates/service.yaml index 8dcef00f..ac11bbcc 100644 --- a/deploy/helm/airflow-operator/templates/service.yaml +++ b/deploy/helm/airflow-operator/templates/service.yaml @@ -1,4 +1,3 @@ -{{- if .Values.maintenance.customResourceDefinitions.maintain }} --- apiVersion: v1 kind: Service @@ -11,11 +10,12 @@ metadata: {{- include "operator.labels" . | nindent 4 }} spec: selector: + {{- if .Values.maintenance.customResourceDefinitions.maintain }} webhook.stackable.tech/conversion: enabled - {{- include "operator.selectorLabels" . | nindent 6 }} + {{- end }} + {{- include "operator.selectorLabels" . | nindent 4 }} ports: - name: conversion-webhook protocol: TCP port: 8443 targetPort: 8443 -{{- end }} diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 251593ef..4a8d4bd3 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -19,7 +19,7 @@ use stackable_operator::{ fragment::{self, Fragment, ValidationError}, merge::Merge, }, - crd::git_sync::{self, credentials_secret_to_basic_auth, credentials_to_secret}, + crd::git_sync::{self}, deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, @@ -53,7 +53,7 @@ use crate::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetails, AirflowClientAuthenticationDetailsResolved, }, - v1alpha1::WebserverRoleConfig, + v1alpha2::WebserverRoleConfig, }, util::role_service_name, }; @@ -214,7 +214,7 @@ pub mod versioned { /// The `webservers` role provides the main UI for user interaction. #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, + pub webservers: Option>, /// The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database. /// Jobs are scheduled on the workers/executors. @@ -254,12 +254,10 @@ pub mod versioned { /// Learn more in the /// [mounting DAGs documentation](DOCS_BASE_URL_PLACEHOLDER/airflow/usage-guide/mounting-dags#_via_git_sync). #[serde(default)] - #[versioned(changed( - since = "v1alpha2", - from_type = "Vec", - upgrade_with = gitsync_v1_to_v2, - downgrade_with = gitsync_v2_to_v1 - ))] + #[versioned( + changed(since = "v1alpha2", from_type = "Vec"), + hint(vec) + )] pub dags_git_sync: Vec, /// for internal use only - not for production use. @@ -305,40 +303,6 @@ pub mod versioned { } } -pub fn gitsync_v1_to_v2( - input: Vec, -) -> Vec { - input - .iter() - .map(|g| git_sync::v1alpha2::GitSync { - credentials: credentials_secret_to_basic_auth(g.credentials_secret.clone()), - repo: g.repo.clone(), - branch: g.branch.clone(), - git_folder: g.git_folder.clone(), - depth: g.depth, - wait: g.wait, - git_sync_conf: g.git_sync_conf.clone(), - }) - .collect() -} - -pub fn gitsync_v2_to_v1( - input: Vec, -) -> Vec { - input - .iter() - .map(|g| git_sync::v1alpha1::GitSync { - credentials_secret: credentials_to_secret(g.credentials.clone()), - repo: g.repo.clone(), - branch: g.branch.clone(), - git_folder: g.git_folder.clone(), - depth: g.depth, - wait: g.wait, - git_sync_conf: g.git_sync_conf.clone(), - }) - .collect() -} - #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct DatabaseInitializationConfig { @@ -361,9 +325,9 @@ pub fn default_db_init() -> bool { true } -impl Default for v1alpha1::WebserverRoleConfig { +impl Default for v1alpha2::WebserverRoleConfig { fn default() -> Self { - v1alpha1::WebserverRoleConfig { + v1alpha2::WebserverRoleConfig { listener_class: webserver_default_listener_class(), common: Default::default(), } From 7b18fd35bc1ec79dce0f34d0da89c93d2e399834 Mon Sep 17 00:00:00 2001 From: Techassi Date: Mon, 22 Dec 2025 15:44:22 +0100 Subject: [PATCH 12/15] chore: Add missing label to Deployment --- deploy/helm/airflow-operator/templates/deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/helm/airflow-operator/templates/deployment.yaml b/deploy/helm/airflow-operator/templates/deployment.yaml index fe17287d..2ccad49f 100644 --- a/deploy/helm/airflow-operator/templates/deployment.yaml +++ b/deploy/helm/airflow-operator/templates/deployment.yaml @@ -24,6 +24,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: + {{- if .Values.maintenance.customResourceDefinitions.maintain }} + webhook.stackable.tech/conversion: enabled + {{- end }} {{- include "operator.selectorLabels" . | nindent 8 }} spec: {{- with .Values.image.pullSecrets }} From 1768854295d68c26bcf64b98660326c765b53e14 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 22 Dec 2025 17:05:40 +0100 Subject: [PATCH 13/15] restore test to v1alpha1 --- .../mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index d6cd1bef..bebf7a2a 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -56,7 +56,7 @@ data: test.txt: | some test text here --- -apiVersion: airflow.stackable.tech/v1alpha1 +apiVersion: airflow.stackable.tech/v1alpha2 kind: AirflowCluster metadata: name: airflow @@ -82,9 +82,8 @@ spec: {% endif %} {% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags - credentialsSecret: git-credentials - #credentials: - # basicAuthSecretName: git-credentials + credentials: + basicAuthSecretName: git-credentials {% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) From bb541b82e7e0460a1304c2dda890f227c3dd7820 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 23 Dec 2025 10:19:33 +0100 Subject: [PATCH 14/15] added versioning integration test --- .../kuttl/versioning/00-patch-ns.yaml.j2 | 9 ++ .../templates/kuttl/versioning/03-assert.yaml | 14 +++ .../versioning/03-install-postgresql.yaml | 11 +++ .../kuttl/versioning/05-assert.yaml.j2 | 10 +++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 ++ .../kuttl/versioning/10-clusterrole.yaml | 27 ++++++ .../kuttl/versioning/30-assert.yaml.j2 | 22 +++++ .../30-install-airflow-cluster.yaml.j2 | 89 +++++++++++++++++++ .../templates/kuttl/versioning/40-assert.yaml | 14 +++ .../versioning/40-install-airflow-python.yaml | 23 +++++ .../kuttl/versioning/50-assert.yaml.j2 | 12 +++ .../kuttl/versioning/50-health-check.yaml | 7 ++ .../templates/kuttl/versioning/60-assert.yaml | 8 ++ .../versioning/60-install-metrics-script.yaml | 8 ++ .../kuttl/versioning/70-assert.yaml.j2 | 25 ++++++ .../templates/kuttl/versioning/dag_metrics.py | 88 ++++++++++++++++++ .../helm-bitnami-postgresql-values.yaml.j2 | 37 ++++++++ tests/test-definition.yaml | 4 + 18 files changed, 417 insertions(+) create mode 100644 tests/templates/kuttl/versioning/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/03-assert.yaml create mode 100644 tests/templates/kuttl/versioning/03-install-postgresql.yaml create mode 100644 tests/templates/kuttl/versioning/05-assert.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/05-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/10-clusterrole.yaml create mode 100644 tests/templates/kuttl/versioning/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/40-assert.yaml create mode 100644 tests/templates/kuttl/versioning/40-install-airflow-python.yaml create mode 100644 tests/templates/kuttl/versioning/50-assert.yaml.j2 create mode 100644 tests/templates/kuttl/versioning/50-health-check.yaml create mode 100644 tests/templates/kuttl/versioning/60-assert.yaml create mode 100644 tests/templates/kuttl/versioning/60-install-metrics-script.yaml create mode 100644 tests/templates/kuttl/versioning/70-assert.yaml.j2 create mode 100755 tests/templates/kuttl/versioning/dag_metrics.py create mode 100644 tests/templates/kuttl/versioning/helm-bitnami-postgresql-values.yaml.j2 diff --git a/tests/templates/kuttl/versioning/00-patch-ns.yaml.j2 b/tests/templates/kuttl/versioning/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/versioning/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/versioning/03-assert.yaml b/tests/templates/kuttl/versioning/03-assert.yaml new file mode 100644 index 00000000..319e927a --- /dev/null +++ b/tests/templates/kuttl/versioning/03-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/versioning/03-install-postgresql.yaml b/tests/templates/kuttl/versioning/03-install-postgresql.yaml new file mode 100644 index 00000000..dc25ba20 --- /dev/null +++ b/tests/templates/kuttl/versioning/03-install-postgresql.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 16.4.2 + -f helm-bitnami-postgresql-values.yaml + oci://registry-1.docker.io/bitnamicharts/postgresql + timeout: 600 diff --git a/tests/templates/kuttl/versioning/05-assert.yaml.j2 b/tests/templates/kuttl/versioning/05-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/versioning/05-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/versioning/05-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/versioning/05-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/versioning/05-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/versioning/10-clusterrole.yaml b/tests/templates/kuttl/versioning/10-clusterrole.yaml new file mode 100644 index 00000000..bcc19e30 --- /dev/null +++ b/tests/templates/kuttl/versioning/10-clusterrole.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: airflow-spark-clusterrole +rules: + - apiGroups: + - spark.stackable.tech + resources: + - sparkapplications + verbs: + - create + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: airflow-spark-clusterrole-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: airflow-spark-clusterrole +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts diff --git a/tests/templates/kuttl/versioning/30-assert.yaml.j2 b/tests/templates/kuttl/versioning/30-assert.yaml.j2 new file mode 100644 index 00000000..e0d31cb8 --- /dev/null +++ b/tests/templates/kuttl/versioning/30-assert.yaml.j2 @@ -0,0 +1,22 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-cluster +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-scheduler-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 new file mode 100644 index 00000000..fe9d5bc2 --- /dev/null +++ b/tests/templates/kuttl/versioning/30-install-airflow-cluster.yaml.j2 @@ -0,0 +1,89 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-airflow +timeout: 480 +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-credentials +type: Opaque +data: + # This is a fine-grained access token for the owner of the repo (stackable-airflow/dags) which has read only access + # to *only* this repo. Contact github users @razvan or @adwk67 for details. + # This token doesn't expire. + user: c3RhY2thYmxlLWFpcmZsb3c= + password: Z2l0aHViX3BhdF8xMUJLUURCRVkwSk1EWlNVQk1RYTdoX0c2OGlhbWtpRkpFV1RMTTF0ajFwbHFTVFNyZ3p3dHZneXI5b2tubGRXaGpVRDZITFRFV0JJcm9yT0dXCg== +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + image: +{% if test_scenario['values']['airflow-latest'].find(",") > 0 %} + custom: "{{ test_scenario['values']['airflow-latest'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['airflow-latest'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + credentialsSecret: test-airflow-credentials + dagsGitSync: + - repo: https://github.com/stackable-airflow/dags + # v1alpha1 field which should be converted to an enum + # N.B. only works for the existing v1 field + credentialsSecret: git-credentials + gitFolder: "mount-dags-gitsync/dags_airflow3" + webservers: + roleConfig: + listenerClass: external-unstable + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + envOverrides: &envOverrides + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: '{"conn_type": "kubernetes", "extra": {"extra__kubernetes__in_cluster": true}}' + replicas: 1 + kubernetesExecutors: + envOverrides: *envOverrides + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + schedulers: + config: + gracefulShutdownTimeout: 10s + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + envOverrides: *envOverrides + replicas: 1 + dagProcessors: + config: + gracefulShutdownTimeout: 10s + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + envOverrides: *envOverrides + replicas: 1 diff --git a/tests/templates/kuttl/versioning/40-assert.yaml b/tests/templates/kuttl/versioning/40-assert.yaml new file mode 100644 index 00000000..6edaa3c3 --- /dev/null +++ b/tests/templates/kuttl/versioning/40-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-python +timeout: 240 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-airflow-python +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/versioning/40-install-airflow-python.yaml b/tests/templates/kuttl/versioning/40-install-airflow-python.yaml new file mode 100644 index 00000000..c3f865a0 --- /dev/null +++ b/tests/templates/kuttl/versioning/40-install-airflow-python.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-airflow-python + labels: + app: test-airflow-python +spec: + replicas: 1 + selector: + matchLabels: + app: test-airflow-python + template: + metadata: + labels: + app: test-airflow-python + spec: + containers: + - name: test-airflow-python + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + imagePullPolicy: IfNotPresent + stdin: true + tty: true diff --git a/tests/templates/kuttl/versioning/50-assert.yaml.j2 b/tests/templates/kuttl/versioning/50-assert.yaml.j2 new file mode 100644 index 00000000..b85052aa --- /dev/null +++ b/tests/templates/kuttl/versioning/50-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-webserver-health-check +timeout: 480 +commands: +{% if test_scenario['values']['airflow-latest'].find(",") > 0 %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'].split(',')[0] }}" +{% else %} + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/health.py --airflow-version "{{ test_scenario['values']['airflow-latest'] }}" +{% endif %} diff --git a/tests/templates/kuttl/versioning/50-health-check.yaml b/tests/templates/kuttl/versioning/50-health-check.yaml new file mode 100644 index 00000000..5d3b329f --- /dev/null +++ b/tests/templates/kuttl/versioning/50-health-check.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 480 +commands: + - script: kubectl cp -n $NAMESPACE ../../../../templates/kuttl/commons/health.py test-airflow-python-0:/tmp + timeout: 240 diff --git a/tests/templates/kuttl/versioning/60-assert.yaml b/tests/templates/kuttl/versioning/60-assert.yaml new file mode 100644 index 00000000..8e7d5b22 --- /dev/null +++ b/tests/templates/kuttl/versioning/60-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: metrics +timeout: 480 +commands: + - script: kubectl exec -n $NAMESPACE test-airflow-python-0 -- python /tmp/dag_metrics.py diff --git a/tests/templates/kuttl/versioning/60-install-metrics-script.yaml b/tests/templates/kuttl/versioning/60-install-metrics-script.yaml new file mode 100644 index 00000000..b5b604b6 --- /dev/null +++ b/tests/templates/kuttl/versioning/60-install-metrics-script.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: metrics +commands: + - script: kubectl cp -n $NAMESPACE dag_metrics.py test-airflow-python-0:/tmp + timeout: 240 diff --git a/tests/templates/kuttl/versioning/70-assert.yaml.j2 b/tests/templates/kuttl/versioning/70-assert.yaml.j2 new file mode 100644 index 00000000..7227e06f --- /dev/null +++ b/tests/templates/kuttl/versioning/70-assert.yaml.j2 @@ -0,0 +1,25 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-log-endpoint +timeout: 240 +commands: +{% if test_scenario['values']['executor'] == 'celery' %} + - script: | + set -eu + + # Log-Endpoint Test: + # This is executed from the Webserver as JWT keys must be present. + # A small server is started on each worker that serves the logs on its + # 8793 port for the Webserver: we don't use the token as that is an + # internal implementation, but check that the endpoint is reachable, + # indicated by a 403. + CURL_RESPONSE=$( + kubectl -n $NAMESPACE exec airflow-webserver-default-0 -- sh -c 'CODE=$(curl -s -o /dev/null -w "%{http_code}" http://airflow-worker-default-headless:8793 2>/dev/null || true);echo "$CODE"' + ) + + # Log-Endpoint Test Assertion: + echo "The HTTP Code is $CURL_RESPONSE (an internal JWT token is needed for full access)" + [ "$CURL_RESPONSE" -eq 403 ] +{% endif %} diff --git a/tests/templates/kuttl/versioning/dag_metrics.py b/tests/templates/kuttl/versioning/dag_metrics.py new file mode 100755 index 00000000..d0dae8cf --- /dev/null +++ b/tests/templates/kuttl/versioning/dag_metrics.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +import requests +import time +import sys +import logging +from datetime import datetime, timezone + + +def assert_metric(role, metric): + metric_response = requests.get( + f"http://airflow-{role}-default-metrics:9102/metrics" + ) + assert metric_response.status_code == 200, ( + f"Metrics could not be retrieved from the {role}." + ) + return metric in metric_response.text + + +now = datetime.now(timezone.utc) +ts = now.strftime("%Y-%m-%dT%H:%M:%S.%f") + now.strftime("%z") + +# Trigger a DAG run to create metrics +dag_id = "sparkapp_dag" +dag_data = {"logical_date": f"{ts}"} + +print(f"DAG-Data: {dag_data}") + +# allow a few moments for the DAGs to be registered to all roles +time.sleep(10) + +rest_url = "http://airflow-webserver:8080/api/v2" +token_url = "http://airflow-webserver:8080/auth/token" + +data = {"username": "airflow", "password": "airflow"} + +headers = {"Content-Type": "application/json"} + +response = requests.post(token_url, headers=headers, json=data) + +if response.status_code == 200 or response.status_code == 201: + token_data = response.json() + access_token = token_data["access_token"] + print(f"Access Token: {access_token}") +else: + print(f"Failed to obtain access token: {response.status_code} - {response.text}") + sys.exit(1) + +headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", +} + +# activate DAG +response = requests.patch( + f"{rest_url}/dags/{dag_id}", headers=headers, json={"is_paused": False} +) +# trigger DAG +response = requests.post( + f"{rest_url}/dags/{dag_id}/dagRuns", headers=headers, json=dag_data +) + +# Wait for the metrics to be consumed by the statsd-exporter +time.sleep(5) +# Test the DAG in a loop. Each time we call the script a new job will be started: we can avoid +# or minimize this by looping over the check instead. +iterations = 9 +loop = 0 +while True: + try: + logging.info(f"Response code: {response.status_code}") + assert response.status_code == 200, "DAG run could not be triggered." + # Worker is not deployed with the kubernetes executor so retrieve success metric from scheduler + # (disable line-break flake checks) + if (assert_metric("scheduler", "airflow_scheduler_heartbeat")) and ( + assert_metric( + "scheduler", "airflow_dagrun_duration_success_sparkapp_dag_count" + ) + ): # noqa: W503, W504 + break + time.sleep(10) + loop += 1 + if loop == iterations: + logging.error("Still waiting for metrics. Exiting to start a new loop....") + # force re-try of script + sys.exit(1) + except AssertionError as error: + logging.warning(f"Encountered: {error}") diff --git a/tests/templates/kuttl/versioning/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/versioning/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..80c50924 --- /dev/null +++ b/tests/templates/kuttl/versioning/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,37 @@ +--- +global: + security: + allowInsecureImages: true + +image: + repository: bitnamilegacy/postgresql + +volumePermissions: + enabled: false + image: + repository: bitnamilegacy/os-shell + securityContext: + runAsUser: auto + +metrics: + image: + repository: bitnamilegacy/postgres-exporter + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + +shmVolume: + chmod: + enabled: false + +auth: + username: airflow + password: airflow + database: airflow diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 367ddeae..91d93a0b 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -105,6 +105,10 @@ tests: - airflow-latest - openshift - executor + - name: versioning + dimensions: + - airflow-latest + - openshift suites: - name: nightly # Run nightly with the latest airflow From 0140d273aee50f9a1cce0e261e3c19e1670d6378 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 23 Dec 2025 16:28:22 +0100 Subject: [PATCH 15/15] updated to operator-rs 101.1 --- Cargo.lock | 76 +++++--- Cargo.nix | 164 ++++++++++++++---- Cargo.toml | 2 +- crate-hashes.json | 4 +- .../operator-binary/src/airflow_controller.rs | 4 +- rust/operator-binary/src/main.rs | 30 ++-- rust/operator-binary/src/operations/pdb.rs | 2 +- .../src/webhooks/conversion.rs | 51 +++--- 8 files changed, 227 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a8096ec..e40bbeea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,9 +410,9 @@ dependencies = [ [[package]] name = "convert_case" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baaaa0ecca5b51987b9423ccdc971514dd8b0bb7b4060b983d3664dad3f1f89f" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" dependencies = [ "unicode-segmentation", ] @@ -494,8 +494,18 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -512,13 +522,37 @@ dependencies = [ "syn 2.0.108", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.108", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", + "quote", + "syn 2.0.108", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", "quote", "syn 2.0.108", ] @@ -1471,9 +1505,9 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ - "darling", + "darling 0.23.0", "regex", "snafu 0.8.9", ] @@ -1548,7 +1582,7 @@ name = "kube-derive" version = "2.0.1" source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "serde", @@ -2818,7 +2852,7 @@ dependencies = [ [[package]] name = "stackable-certs" version = "0.4.0" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "const-oid", "ecdsa", @@ -2841,8 +2875,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.100.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +version = "0.101.1" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "chrono", "clap", @@ -2881,9 +2915,9 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.108", @@ -2892,7 +2926,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "chrono", "k8s-openapi", @@ -2909,7 +2943,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "axum", "clap", @@ -2933,7 +2967,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "schemars", "serde", @@ -2946,10 +2980,10 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "convert_case", - "darling", + "darling 0.23.0", "indoc", "itertools", "k8s-openapi", @@ -2962,10 +2996,11 @@ dependencies = [ [[package]] name = "stackable-webhook" -version = "0.7.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#8b94f6a154f54eca2df9bba839e510ed45b99233" +version = "0.8.0" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0" dependencies = [ "arc-swap", + "async-trait", "axum", "futures-util", "hyper", @@ -2975,6 +3010,7 @@ dependencies = [ "opentelemetry", "opentelemetry-semantic-conventions", "rand 0.9.2", + "serde", "serde_json", "snafu 0.8.9", "stackable-certs", diff --git a/Cargo.nix b/Cargo.nix index c81df7e1..cbc2f42e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1286,9 +1286,9 @@ rec { }; "convert_case" = rec { crateName = "convert_case"; - version = "0.8.0"; + version = "0.10.0"; edition = "2021"; - sha256 = "17zqy79xlr1n7nc0n1mlnw5qpp8l2nbxrk13jixrhlavrbna1ams"; + sha256 = "1fff1x78mp2c233g68my0ag0zrmjdbym8bfyahjbfy4cxza5hd33"; authors = [ "rutrum " ]; @@ -1298,10 +1298,7 @@ rec { packageId = "unicode-segmentation"; } ]; - features = { - "rand" = [ "dep:rand" ]; - "random" = [ "rand" ]; - }; + }; "core-foundation" = rec { crateName = "core-foundation"; @@ -1511,7 +1508,7 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "darling" = rec { + "darling 0.21.3" = rec { crateName = "darling"; version = "0.21.3"; edition = "2021"; @@ -1522,11 +1519,11 @@ rec { dependencies = [ { name = "darling_core"; - packageId = "darling_core"; + packageId = "darling_core 0.21.3"; } { name = "darling_macro"; - packageId = "darling_macro"; + packageId = "darling_macro 0.21.3"; } ]; features = { @@ -1537,7 +1534,33 @@ rec { }; resolvedDefaultFeatures = [ "default" "suggestions" ]; }; - "darling_core" = rec { + "darling 0.23.0" = rec { + crateName = "darling"; + version = "0.23.0"; + edition = "2021"; + sha256 = "179fj6p6ajw4dnkrik51wjhifxwy02x5zhligyymcb905zd17bi5"; + authors = [ + "Ted Driggs " + ]; + dependencies = [ + { + name = "darling_core"; + packageId = "darling_core 0.23.0"; + } + { + name = "darling_macro"; + packageId = "darling_macro 0.23.0"; + } + ]; + features = { + "default" = [ "suggestions" ]; + "diagnostics" = [ "darling_core/diagnostics" ]; + "serde" = [ "darling_core/serde" ]; + "suggestions" = [ "darling_core/suggestions" ]; + }; + resolvedDefaultFeatures = [ "default" "suggestions" ]; + }; + "darling_core 0.21.3" = rec { crateName = "darling_core"; version = "0.21.3"; edition = "2021"; @@ -1580,7 +1603,46 @@ rec { }; resolvedDefaultFeatures = [ "strsim" "suggestions" ]; }; - "darling_macro" = rec { + "darling_core 0.23.0" = rec { + crateName = "darling_core"; + version = "0.23.0"; + edition = "2021"; + sha256 = "1c033vrks38vpw8kwgd5w088dsr511kfz55n9db56prkgh7sarcq"; + authors = [ + "Ted Driggs " + ]; + dependencies = [ + { + name = "ident_case"; + packageId = "ident_case"; + } + { + name = "proc-macro2"; + packageId = "proc-macro2"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "strsim"; + packageId = "strsim"; + optional = true; + } + { + name = "syn"; + packageId = "syn 2.0.108"; + features = [ "full" "extra-traits" ]; + } + ]; + features = { + "serde" = [ "dep:serde" ]; + "strsim" = [ "dep:strsim" ]; + "suggestions" = [ "strsim" ]; + }; + resolvedDefaultFeatures = [ "strsim" "suggestions" ]; + }; + "darling_macro 0.21.3" = rec { crateName = "darling_macro"; version = "0.21.3"; edition = "2021"; @@ -1592,7 +1654,32 @@ rec { dependencies = [ { name = "darling_core"; - packageId = "darling_core"; + packageId = "darling_core 0.21.3"; + } + { + name = "quote"; + packageId = "quote"; + } + { + name = "syn"; + packageId = "syn 2.0.108"; + } + ]; + + }; + "darling_macro 0.23.0" = rec { + crateName = "darling_macro"; + version = "0.23.0"; + edition = "2021"; + sha256 = "13fvzji9xyp304mgq720z5l0xgm54qj68jibwscagkynggn88fdc"; + procMacro = true; + authors = [ + "Ted Driggs " + ]; + dependencies = [ + { + name = "darling_core"; + packageId = "darling_core 0.23.0"; } { name = "quote"; @@ -4669,7 +4756,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "k8s_version"; @@ -4679,7 +4766,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling"; + packageId = "darling 0.23.0"; optional = true; } { @@ -4773,7 +4860,7 @@ rec { "webpki-roots" = [ "kube-client/webpki-roots" "client" ]; "ws" = [ "kube-client/ws" "kube-core/ws" ]; }; - resolvedDefaultFeatures = [ "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "ring" "runtime" "rustls-tls" ]; + resolvedDefaultFeatures = [ "admission" "client" "config" "derive" "jsonpatch" "kube-client" "kube-derive" "kube-runtime" "ring" "runtime" "rustls-tls" ]; }; "kube-client" = rec { crateName = "kube-client"; @@ -5098,7 +5185,7 @@ rec { "schema" = [ "schemars" ]; "schemars" = [ "dep:schemars" ]; }; - resolvedDefaultFeatures = [ "json-patch" "jsonpatch" "schema" "schemars" ]; + resolvedDefaultFeatures = [ "admission" "json-patch" "jsonpatch" "schema" "schemars" ]; }; "kube-derive" = rec { crateName = "kube-derive"; @@ -5120,7 +5207,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling"; + packageId = "darling 0.21.3"; } { name = "proc-macro2"; @@ -9361,7 +9448,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_certs"; @@ -9389,7 +9476,7 @@ rec { name = "kube"; packageId = "kube"; usesDefaultFeatures = false; - features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; } { name = "p256"; @@ -9459,12 +9546,12 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.100.3"; + version = "0.101.1"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_operator"; @@ -9530,7 +9617,7 @@ rec { name = "kube"; packageId = "kube"; usesDefaultFeatures = false; - features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; } { name = "product-config"; @@ -9638,7 +9725,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; procMacro = true; @@ -9649,7 +9736,7 @@ rec { dependencies = [ { name = "darling"; - packageId = "darling"; + packageId = "darling 0.23.0"; } { name = "proc-macro2"; @@ -9673,7 +9760,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_shared"; @@ -9697,7 +9784,7 @@ rec { name = "kube"; packageId = "kube"; usesDefaultFeatures = false; - features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; } { name = "schemars"; @@ -9755,7 +9842,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_telemetry"; @@ -9865,7 +9952,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_versioned"; @@ -9909,7 +9996,7 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; procMacro = true; @@ -9924,7 +10011,7 @@ rec { } { name = "darling"; - packageId = "darling"; + packageId = "darling 0.23.0"; } { name = "indoc"; @@ -9949,7 +10036,7 @@ rec { name = "kube"; packageId = "kube"; usesDefaultFeatures = false; - features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; } { name = "proc-macro2"; @@ -9968,12 +10055,12 @@ rec { }; "stackable-webhook" = rec { crateName = "stackable-webhook"; - version = "0.7.1"; + version = "0.8.0"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "8b94f6a154f54eca2df9bba839e510ed45b99233"; + rev = "dbe2250ee16d8f73b0bd3fa8f7e3954ca528b1e0"; sha256 = "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba"; }; libName = "stackable_webhook"; @@ -9985,6 +10072,10 @@ rec { name = "arc-swap"; packageId = "arc-swap"; } + { + name = "async-trait"; + packageId = "async-trait"; + } { name = "axum"; packageId = "axum"; @@ -10013,7 +10104,7 @@ rec { name = "kube"; packageId = "kube"; usesDefaultFeatures = false; - features = [ "client" "jsonpatch" "runtime" "derive" "rustls-tls" "ring" ]; + features = [ "client" "jsonpatch" "runtime" "derive" "admission" "rustls-tls" "ring" ]; } { name = "opentelemetry"; @@ -10027,6 +10118,11 @@ rec { name = "rand"; packageId = "rand 0.9.2"; } + { + name = "serde"; + packageId = "serde"; + features = [ "derive" ]; + } { name = "serde_json"; packageId = "serde_json"; diff --git a/Cargo.toml b/Cargo.toml index 52f71311..47824741 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned", "webhook"], tag = "stackable-operator-0.100.3" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned", "webhook"], tag = "stackable-operator-0.101.1" } anyhow = "1.0" base64 = "0.22" diff --git a/crate-hashes.json b/crate-hashes.json index ac30e3aa..18443c49 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -2,12 +2,12 @@ "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-certs@0.4.0": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.101.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-webhook@0.7.1": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-webhook@0.8.0": "0bfvi39gwq1qf72xj3xaailynh6apjkv2qn8f1pi7r6zwzsg56ba", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index c7389e0a..31c1b58c 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -436,7 +436,7 @@ pub async fn reconcile_airflow( AIRFLOW_CONTROLLER_NAME, &airflow.object_ref(&()), ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), - airflow.spec.object_overrides.clone(), + &airflow.spec.object_overrides, ) .context(CreateClusterResourcesSnafu)?; @@ -687,7 +687,7 @@ async fn build_executor_template( resolved_product_image: &ResolvedProductImage, authentication_config: &AirflowClientAuthenticationDetailsResolved, authorization_config: &AirflowAuthorizationResolved, - cluster_resources: &mut ClusterResources, + cluster_resources: &mut ClusterResources<'_>, client: &stackable_operator::client::Client, rbac_sa: &stackable_operator::k8s_openapi::api::core::v1::ServiceAccount, ) -> Result<(), Error> { diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 42f0acb1..2b3ee726 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -34,7 +34,7 @@ use stackable_operator::{ use crate::{ airflow_controller::AIRFLOW_FULL_CONTROLLER_NAME, crd::{AirflowCluster, AirflowClusterVersion, OPERATOR_NAME, v1alpha2}, - webhooks::conversion::create_webhook_and_maintainer, + webhooks::conversion::create_webhook_server, }; mod airflow_controller; @@ -187,28 +187,18 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); - let (conversion_webhook, crd_maintainer, _initial_reconcile_rx) = - create_webhook_and_maintainer( - &operator_environment, - maintenance.disable_crd_maintenance, - client.as_kube_client(), - ) - .await?; + let webhook_server = create_webhook_server( + &operator_environment, + maintenance.disable_crd_maintenance, + client.as_kube_client(), + ) + .await?; - let conversion_webhook = conversion_webhook + let webhook_server = webhook_server .run() - .map_err(|err| anyhow!(err).context("failed to run conversion webhook")); + .map_err(|err| anyhow!(err).context("failed to run webhook server")); - let crd_maintainer = crd_maintainer - .run() - .map_err(|err| anyhow!(err).context("failed to run CRD maintainer")); - - futures::try_join!( - airflow_controller, - conversion_webhook, - crd_maintainer, - eos_checker - )?; + futures::try_join!(airflow_controller, webhook_server, eos_checker)?; } } diff --git a/rust/operator-binary/src/operations/pdb.rs b/rust/operator-binary/src/operations/pdb.rs index d70d9d61..9f010800 100644 --- a/rust/operator-binary/src/operations/pdb.rs +++ b/rust/operator-binary/src/operations/pdb.rs @@ -28,7 +28,7 @@ pub async fn add_pdbs( airflow: &v1alpha2::AirflowCluster, role: &AirflowRole, client: &Client, - cluster_resources: &mut ClusterResources, + cluster_resources: &mut ClusterResources<'_>, ) -> Result<(), Error> { if !pdb.enabled { return Ok(()); diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 84bbeb6c..f7b5f857 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -3,11 +3,10 @@ use stackable_operator::{ cli::OperatorEnvironmentOptions, kube::{Client, core::crd::MergeError}, webhook::{ - maintainer::CustomResourceDefinitionMaintainer, - servers::{ConversionWebhookError, ConversionWebhookServer}, + WebhookServer, WebhookServerError, WebhookServerOptions, + webhooks::{ConversionWebhook, ConversionWebhookOptions}, }, }; -use tokio::sync::oneshot; use crate::crd::{AirflowCluster, AirflowClusterVersion, FIELD_MANAGER}; @@ -19,35 +18,35 @@ pub enum Error { MergeCrd { source: MergeError }, #[snafu(display("failed to create conversion webhook server"))] - CreateConversionWebhook { source: ConversionWebhookError }, + CreateWebhook { source: WebhookServerError }, } -/// Creates and returns a [`ConversionWebhookServer`] and a [`CustomResourceDefinitionMaintainer`]. -pub async fn create_webhook_and_maintainer<'a>( - operator_environment: &'a OperatorEnvironmentOptions, +/// Creates and returns a [`WebhookServer`]. +pub async fn create_webhook_server( + operator_environment: &OperatorEnvironmentOptions, disable_crd_maintenance: bool, client: Client, -) -> Result< - ( - ConversionWebhookServer, - CustomResourceDefinitionMaintainer<'a>, - oneshot::Receiver<()>, - ), - Error, -> { - let crds_and_handlers = [( +) -> Result { + let crds_and_handlers = vec![( AirflowCluster::merged_crd(AirflowClusterVersion::V1Alpha2).context(MergeCrdSnafu)?, - AirflowCluster::try_convert as fn(_) -> _, + AirflowCluster::try_convert, )]; - ConversionWebhookServer::with_maintainer( - crds_and_handlers, - &operator_environment.operator_service_name, - &operator_environment.operator_namespace, - FIELD_MANAGER, + let conversion_webhook_options = ConversionWebhookOptions { disable_crd_maintenance, - client, - ) - .await - .context(CreateConversionWebhookSnafu) + field_manager: FIELD_MANAGER.to_owned(), + }; + + let (conversion_webhook, _initial_reconcile_rx) = + ConversionWebhook::new(crds_and_handlers, client, conversion_webhook_options); + + let webhook_server_options = WebhookServerOptions { + socket_addr: WebhookServer::DEFAULT_SOCKET_ADDRESS, + webhook_namespace: operator_environment.operator_namespace.to_owned(), + webhook_service_name: operator_environment.operator_service_name.to_owned(), + }; + + WebhookServer::new(vec![Box::new(conversion_webhook)], webhook_server_options) + .await + .context(CreateWebhookSnafu) }