|
| 1 | +--- |
| 2 | +apiVersion: airflow.stackable.tech/v1alpha1 |
| 3 | +kind: AirflowCluster |
| 4 | +metadata: |
| 5 | + name: airflow |
| 6 | +spec: |
| 7 | + image: |
| 8 | + # Currently does not work with the kubernetes executor S3 logging |
| 9 | + # (and its still marked experimental as of release 25.7). See: |
| 10 | + # https://github.com/apache/airflow/issues/50583 |
| 11 | + # https://github.com/apache/airflow/issues/52501 |
| 12 | + # productVersion: 3.0.1 |
| 13 | + productVersion: 2.10.5 |
| 14 | + clusterConfig: |
| 15 | + loadExamples: false |
| 16 | + exposeConfig: false |
| 17 | + credentialsSecret: airflow-credentials |
| 18 | + # NOTE: This cannot be parameterized via stackablectl since it is synced via ArgoCD. |
| 19 | + # |
| 20 | + # In terms of forking the repository, this should not be a problem since the DAGs are the same. |
| 21 | + # You can still change this locally and point to the fork in case of any changes to DAGs. |
| 22 | + # |
| 23 | + # TODO(@maltesander): In a Stackable release, this branch should point to the release-YY.MM branch! |
| 24 | + dagsGitSync: |
| 25 | + - repo: https://github.com/stackabletech/demos/ |
| 26 | + branch: main |
| 27 | + gitFolder: "demos/argo-cd-git-ops/dags" |
| 28 | + depth: 2 |
| 29 | + volumes: |
| 30 | + - name: minio-tls |
| 31 | + ephemeral: |
| 32 | + volumeClaimTemplate: |
| 33 | + metadata: |
| 34 | + annotations: |
| 35 | + secrets.stackable.tech/class: tls |
| 36 | + secrets.stackable.tech/scope: pod |
| 37 | + spec: |
| 38 | + accessModes: |
| 39 | + - ReadWriteOnce |
| 40 | + resources: |
| 41 | + requests: |
| 42 | + storage: "1" |
| 43 | + storageClassName: secrets.stackable.tech |
| 44 | + volumeMounts: |
| 45 | + - name: minio-tls |
| 46 | + mountPath: /stackable/minio-tls |
| 47 | + webservers: |
| 48 | + roleConfig: |
| 49 | + listenerClass: external-unstable |
| 50 | + envOverrides: &envOverrides |
| 51 | + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" |
| 52 | + # Via sealed secrets and pod overrides, just kept for reference here |
| 53 | + # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" |
| 54 | + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" |
| 55 | + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" |
| 56 | + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ |
| 57 | + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio |
| 58 | + # 20 seconds to reload the DAGS folder (default: 300) |
| 59 | + AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "20" |
| 60 | + podOverrides: &podOverrides |
| 61 | + spec: |
| 62 | + containers: |
| 63 | + - name: airflow |
| 64 | + env: |
| 65 | + - name: AIRFLOW_CONN_MINIO |
| 66 | + valueFrom: |
| 67 | + secretKeyRef: |
| 68 | + name: airflow-minio-connection |
| 69 | + key: airflow-minio-connection |
| 70 | + config: |
| 71 | + gracefulShutdownTimeout: 30s |
| 72 | + resources: |
| 73 | + cpu: |
| 74 | + min: 400m |
| 75 | + max: "1" |
| 76 | + memory: |
| 77 | + limit: 2Gi |
| 78 | + roleGroups: |
| 79 | + default: |
| 80 | + # NOTE: In order to properly show ArgoCD in action - syncing changes via Git - do the following: |
| 81 | + # - Fork this repository |
| 82 | + # - Use the demo parameters `customGitUrl` and `customGitBranch` to point the |
| 83 | + # the repository to your fork and branch |
| 84 | + # - Change the replicas here and push that to the provided `customGitBranch` in your fork |
| 85 | + replicas: 1 |
| 86 | + kubernetesExecutors: |
| 87 | + envOverrides: *envOverrides |
| 88 | + podOverrides: |
| 89 | + spec: |
| 90 | + containers: |
| 91 | + - name: base |
| 92 | + env: |
| 93 | + - name: AIRFLOW_CONN_MINIO |
| 94 | + valueFrom: |
| 95 | + secretKeyRef: |
| 96 | + name: airflow-minio-connection |
| 97 | + key: airflow-minio-connection |
| 98 | + schedulers: |
| 99 | + envOverrides: *envOverrides |
| 100 | + podOverrides: *podOverrides |
| 101 | + config: |
| 102 | + gracefulShutdownTimeout: 30s |
| 103 | + resources: |
| 104 | + cpu: |
| 105 | + min: 400m |
| 106 | + max: "1" |
| 107 | + memory: |
| 108 | + limit: 1Gi |
| 109 | + roleGroups: |
| 110 | + default: |
| 111 | + replicas: 1 |
0 commit comments