diff --git a/.scripts/update_refs.sh b/.scripts/update_refs.sh index 0581cd79..d1144a41 100755 --- a/.scripts/update_refs.sh +++ b/.scripts/update_refs.sh @@ -65,6 +65,17 @@ if [[ "$CURRENT_BRANCH" == release-* ]]; then | prepend "\t" maybe_commit "chore(release): $MESSAGE" + # Replace 0.0.0-dev on lines marked `# release-ref: version`. These are + # versions that don't follow the `stackable0.0.0-dev` image tag pattern above, + # e.g. the operator version Argo CD installs in the argo-cd-git-ops stack. + MESSAGE="Update marked release versions to ${STACKABLE_RELEASE}.0" + echo "$MESSAGE" + find demos stacks -type f \ + -exec grep --color=always -lE '0\.0\.0-dev\s+# release-ref: version' {} \; \ + -exec sed -i -E "s#0\.0\.0-dev(\s+\# release-ref: version)#${STACKABLE_RELEASE}.0\1#" {} \; \ + | prepend "\t" + maybe_commit "chore(release): $MESSAGE" + # Look for remaining references echo "Checking files with older stackable release references which will be assumed to be intentional." grep --color=always -ronE "stackable24\.3(\.[0-9]+)" | prepend "\t" @@ -73,7 +84,7 @@ else >&2 echo "WARNING: doesn't look like a release branch. Will not update stackableRelease versions in stacks and image references." fi -MESSAGE="Replace githubusercontent references main->${CURRENT_BRANCH}" +MESSAGE="Replace main branch references with ${CURRENT_BRANCH}" echo "$MESSAGE" # Search for githubusercontent urls and replace the branch reference with a placeholder variable # This is done just in case the branch has special regex characters (like `/`). @@ -83,6 +94,14 @@ find demos stacks -type f \ -exec sed -i -E 's#(stackabletech/demos)/(refs/heads/)?main/#\1/\${UPDATE_BRANCH_REF}/#' {} \; \ | prepend "\t" +# Same for git refs that are not githubusercontent URLs, e.g. branches Argo CD +# or git-sync pull from at runtime. These lines are marked `# release-ref: branch`. +# shellcheck disable=SC2016 # We intentionally don't want to expand the variable. +find demos stacks -type f \ + -exec grep --color=always -lE 'main\s+# release-ref: branch' {} \; \ + -exec sed -i -E 's#main(\s+\# release-ref: branch)#${UPDATE_BRANCH_REF}\1#' {} \; \ +| prepend "\t" + # Now, for all modified files, we can use envsubst export UPDATE_BRANCH_REF="$CURRENT_BRANCH" for MODIFIED_FILE in $(git diff --name-only); do @@ -91,3 +110,12 @@ for MODIFIED_FILE in $(git diff --name-only); do mv "$MODIFIED_FILE.replacements" "$MODIFIED_FILE" done maybe_commit "chore(release): $MESSAGE" + +# A marked line that still points at main or 0.0.0-dev would leave a release +# depending on main at runtime, so treat it as an error. +if [[ "$CURRENT_BRANCH" == release-* ]]; then + if grep -rnE '(main|0\.0\.0-dev)\s+# release-ref:' demos stacks; then + >&2 echo "The lines above are marked as release references but were not updated. Exiting." + exit 3 + fi +fi diff --git a/demos/argo-cd-git-ops/README.md b/demos/argo-cd-git-ops/README.md index 49a6c7ba..b7acc7c6 100644 --- a/demos/argo-cd-git-ops/README.md +++ b/demos/argo-cd-git-ops/README.md @@ -12,6 +12,8 @@ It provides the Sealed Secrets controller and a generated certificate that is au ```sh kubeseal -n stackable-airflow --format=yaml < airflow-admin-credentials.yaml > sealed-airflow-admin-credentials.yaml kubeseal -n stackable-airflow --format=yaml < airflow-postgresql-credentials.yaml > sealed-airflow-postgresql-credentials.yaml +kubeseal -n stackable-airflow --format=yaml < airflow-s3-connection.yaml > sealed-airflow-s3-connection.yaml +kubeseal -n garage --format=yaml < s3-credentials.yaml > sealed-s3-credentials.yaml ``` For "offline" usage you can first fetch the CA from the `sealed-secrets-controller` and use that for encryption when the demo / stack is not available. @@ -34,7 +36,7 @@ Kubernetes rejects them as invalid label values, which breaks unsealing — labe ## Plaintext secrets used in this demo -The sealed secrets under `manifests/airflow/` were created from the following plaintext manifests. +The sealed secrets under `manifests/airflow/` and `manifests/garage/` were created from the following plaintext manifests. These are demo-only credentials: the Sealed Secrets controller key for this stack is committed to this repository (`stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml`), so the sealed secrets provide no confidentiality here — sealing only demonstrates the GitOps workflow. Never use this pattern for real credentials. @@ -69,4 +71,41 @@ stringData: password: airflow ``` +`s3-credentials.yaml` (namespace `garage`). Garage requires the `GK<24 hex characters>` key id format and a 64 hex character secret, so these are not free form values: + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials +type: Opaque +stringData: + accessKey: GK31c0ffee31c0ffee31c0ffee + secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef +``` + +`airflow-s3-connection.yaml` (must use the key pair from `s3-credentials.yaml`): + +```yaml +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-s3-connection +type: Opaque +stringData: + airflow-s3-connection: aws://GK31c0ffee31c0ffee31c0ffee:deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef@/?endpoint_url=https%3A%2F%2Fgarage.garage.svc.cluster.local%3A9000 +``` + TODO: We should probably create a script that seals all secrets and puts them in the respective stack / demo folders + +## git-sync only fetches the DAG folder + +A full clone of this repository needs several times the default memory of the git-sync containers. +git-sync therefore does a sparse partial clone of `dags/` only: `gitSyncConf` in `manifests/airflow/airflow.yaml`, with the sparse checkout file in `manifests/airflow/git-sync-sparse-checkout.yaml`. + +- If the DAGs move, update both `gitFolder` in `airflow.yaml` and the sparse checkout file. +- git documents sparse checkout as experimental, so a newer git in the Airflow image may change how it behaves. + A breaking change fails silently: git-sync reports no errors, but the DAG folder is missing and Airflow shows no DAGs. + The fallback is a full clone, described in the comment above `gitSyncConf` in `airflow.yaml`: drop the sparse checkout configuration and give each git-sync container 512Mi of memory. diff --git a/demos/argo-cd-git-ops/applications/minio.yaml b/demos/argo-cd-git-ops/applications/garage.yaml similarity index 82% rename from demos/argo-cd-git-ops/applications/minio.yaml rename to demos/argo-cd-git-ops/applications/garage.yaml index b7ee2863..71f1fd81 100644 --- a/demos/argo-cd-git-ops/applications/minio.yaml +++ b/demos/argo-cd-git-ops/applications/garage.yaml @@ -2,19 +2,19 @@ apiVersion: argoproj.io/v1alpha1 kind: Application metadata: - name: minio + name: garage # Finalizer needed so resources created by the Application are deleted when the Application gets removed finalizers: - resources-finalizer.argocd.argoproj.io spec: - project: minio + project: garage destination: server: https://kubernetes.default.svc - namespace: minio + namespace: garage source: repoURL: "{{ customGitUrl }}" targetRevision: "{{ customGitBranch }}" - path: demos/argo-cd-git-ops/manifests/minio/ + path: demos/argo-cd-git-ops/manifests/garage/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml index b9a2476e..3617dd15 100644 --- a/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml @@ -24,20 +24,50 @@ spec: # In terms of forking the repository, this should not be a problem since the DAGs are the same. # You can still change this locally and point to the fork in case of any changes to DAGs. # - # TODO(@maltesander): In a Stackable release, this branch should point to the release-YY.MM branch! + # The `release-ref` marker lets .scripts/update_refs.sh point this branch at + # the release branch when one is cut. dagsGitSync: - repo: https://github.com/stackabletech/demos/ - branch: main + branch: main # release-ref: branch gitFolder: "demos/argo-cd-git-ops/dags" - depth: 2 + depth: 1 + # Fetch and check out only the DAG folder (see git-sync-sparse-checkout.yaml). + # A full clone of this repository needs several times the git-sync + # containers' default memory, a sparse partial clone needs a fraction of it. + # + # git documents sparse checkout as experimental, so its behaviour may + # change with the git version in a future Airflow image. If it breaks, it + # does so silently: git-sync succeeds, but the DAG folder is missing and + # Airflow shows no DAGs. The fallback is a full clone: remove gitSyncConf, + # the git-sync-sparse volume and its mount, and git-sync-sparse-checkout.yaml, + # and give the git-sync containers (git-sync-0 and git-sync-0-init, in the + # podOverrides of every role including kubernetesExecutors) 512Mi of memory: + # + # - name: git-sync-0 + # resources: + # limits: + # memory: "512Mi" + # requests: + # memory: "512Mi" + gitSyncConf: + --filter: blob:none + # See the mounted ConfigMap: git-sync-sparse-checkout + --sparse-checkout-file: /stackable/git-sync-sparse/sparse-checkout volumes: - - name: minio-tls + # Mounted into the git-sync containers as well, which is how they find + # the sparse checkout file. + - name: git-sync-sparse + configMap: + name: git-sync-sparse-checkout + # Only the CA is needed, to verify the Garage S3 endpoint. + - name: s3-tls ephemeral: volumeClaimTemplate: metadata: annotations: secrets.stackable.tech/class: tls secrets.stackable.tech/scope: pod + secrets.stackable.tech/provision-parts: public spec: accessModes: - ReadWriteOnce @@ -46,19 +76,24 @@ spec: storage: "1" storageClassName: secrets.stackable.tech volumeMounts: - - name: minio-tls - mountPath: /stackable/minio-tls + - name: git-sync-sparse + mountPath: /stackable/git-sync-sparse + - name: s3-tls + mountPath: /stackable/s3-tls webservers: roleConfig: listenerClass: external-stable envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets and pod overrides, just kept for reference here - # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" - AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" + # AIRFLOW_CONN_S3: "aws://GK31c0ffee31c0ffee31c0ffee:@/?endpoint_url=https%3A%2F%2Fgarage.garage.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/s3-tls/ca.crt" + # boto3 has no region here otherwise, and Garage rejects requests signed + # for a different region than the one it is configured with. + AWS_DEFAULT_REGION: region-1 AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ - AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: s3 # 20 seconds to reload the DAGS folder (default: 300) AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: "20" podOverrides: &podOverrides @@ -66,24 +101,11 @@ spec: containers: - name: airflow env: - - name: AIRFLOW_CONN_MINIO + - name: AIRFLOW_CONN_S3 valueFrom: secretKeyRef: - name: airflow-minio-connection - key: airflow-minio-connection - - name: git-sync-0 - resources: - limits: - memory: "128Mi" - requests: - memory: "128Mi" - initContainers: - - name: git-sync-0-init - resources: - limits: - memory: "128Mi" - requests: - memory: "128Mi" + name: airflow-s3-connection + key: airflow-s3-connection config: gracefulShutdownTimeout: 30s resources: @@ -107,18 +129,11 @@ spec: containers: - name: base env: - - name: AIRFLOW_CONN_MINIO + - name: AIRFLOW_CONN_S3 valueFrom: secretKeyRef: - name: airflow-minio-connection - key: airflow-minio-connection - initContainers: - - name: git-sync-0-init - resources: - limits: - memory: "128Mi" - requests: - memory: "128Mi" + name: airflow-s3-connection + key: airflow-s3-connection schedulers: envOverrides: *envOverrides podOverrides: *podOverrides @@ -133,3 +148,11 @@ spec: roleGroups: default: replicas: 1 + # Without this role the DAG processor runs in the scheduler container, and the + # two together exceed the scheduler's memory limit. + dagProcessors: + envOverrides: *envOverrides + podOverrides: *podOverrides + roleGroups: + default: + replicas: 1 diff --git a/demos/argo-cd-git-ops/manifests/airflow/git-sync-sparse-checkout.yaml b/demos/argo-cd-git-ops/manifests/airflow/git-sync-sparse-checkout.yaml new file mode 100644 index 00000000..b46704b1 --- /dev/null +++ b/demos/argo-cd-git-ops/manifests/airflow/git-sync-sparse-checkout.yaml @@ -0,0 +1,26 @@ +--- +# Limits what git-sync checks out of this repository to the DAG folder. Used +# together with `--filter: blob:none` in airflow.yaml, so git-sync also only +# downloads the files it checks out. +# +# The git-sync image enables sparse checkout in cone mode, which requires every +# parent directory to be listed in this form. A single `/path/to/dags/` line +# checks out nothing but the top-level files. +# +# If the DAGs are moved, update both this file and `gitFolder` in airflow.yaml. +apiVersion: v1 +kind: ConfigMap +metadata: + name: git-sync-sparse-checkout + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +data: + sparse-checkout: | + /* + !/*/ + /demos/ + !/demos/*/ + /demos/argo-cd-git-ops/ + !/demos/argo-cd-git-ops/*/ + /demos/argo-cd-git-ops/dags/ diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml deleted file mode 100644 index 40bc62b2..00000000 --- a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml +++ /dev/null @@ -1,19 +0,0 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - creationTimestamp: null - name: airflow-minio-connection - namespace: stackable-airflow -spec: - encryptedData: - airflow-minio-connection: AgAHQL1NIeZiWH1opQYqHINixw0muWaNsYKMhtgJqqdVMLeKIU/sckZ7Z6aokaEYXGVQRe+Tp+D9AaYaF5CdKeMQKQ3MmhxXYZh0EuAcNRyvz4u4K696KEIa4XpaT5k6oRbIwRUGCc4hiRVApmMOgGYl4iEwVQ/1Twd5ffEAohMsySgZQBn17zB+hKC3u2AcIrZ1btHvBrVhC4pil/kyytBwjTkMq1iAkF8lwt7gatLukD9ldLaVdYTe21R+nP/UT7DwHRnBlI1dzkfrMdplFud2XIEmSrFF7ihzc5r+Lsm2vsnjaV5OgzDjIPI3RRA7g8/06Lebpru/E9h6HB5RIu59ga72g02QIHh5HffGUCnbEsLTDDbBcuAQB2jSodotfsofVKOhZChOIJDchvohhrQ86wqlkQyjfNMq2Gh46Nc7ZjomlpL/p0afmak34GFoPYMCD7IJ2DafEsmKA8dYfjTI1kVzLhYuoeDB3f82tu8xL9cXCEUsw7Y7EosZjeITeLcV1oMm+jeHzNHGaLmCI9k1sY5bL6xjbG59sOvuEgI302tS9ua592u9uqOOKKjE4eZM7CAmEjgMmDPbeteXGnZcOqfj8FWmlp0f4gm9s+68KH5PDtkKdtBckNjDeLj3QhMC2gstXnPJR3Hnm/bFQ4i/6XpnkuOFCvQT/Baar5scw4vhJ0tN8TGeMd651YXU7N2MEB+2WtiO0StbzhcWIoiGmvN+SRLawPZ87Sqlt26/bs5+jnuEv4Fy1oXhxbZwuf/YsZjEaKFI+HbFVvwM0K2d00/z9DYAB44oY2PmrZwsk0KqP2hkuumq - template: - metadata: - creationTimestamp: null - name: airflow-minio-connection - namespace: stackable-airflow - labels: - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops - type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-s3-connection.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-s3-connection.yaml new file mode 100644 index 00000000..ca4e8ca0 --- /dev/null +++ b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-s3-connection.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-s3-connection + namespace: stackable-airflow +spec: + encryptedData: + airflow-s3-connection: AgCiOpa8CqtPyxatxCRHubdWYJGlrSPpstbD8wSgDkZFqFSaw+5bpKRFs0+CAnbq5ziCW4j3mSd/YMhaamqSCkQ/VWDH24VBSzlX/5tBcWNk93mMAtlZkBn2z1Uv8wMwX4sbdgdPnF9BzaelF24CBUpHTzBsFzPegBwRwAD4fqRA2jFueTSGfpIct+zLgUlAlUxn3UySSowE/12SCY+kkM/JLmF5jNLHwP54EIc8eBw0qVk9LF9zRrrhUeK9K7DZTmZO4wdaGBrQJgOteycMaMP0vRVvxK1+/MLURX5nVpvjKGWQdN8qtLA/5xezH2IUzhZgyn6iXJukvll+63RG1jnpDJAmYJtqktNz9fc/Y6lpn612a821zzS6DQS8dwqLbARhDROuAMcTiP3L2XAR48vR6ZJQoxPkGxkUWfUMOPFKIC5JeYQ5WQcQf74BRPYh5n497jif6PTabFPTCoIngodNGUd7dN0C4Si8XpUMXu8tMWFhzv4UeMaOw218oqGXuu2RTqYyJ2lsGM3poNBIX3GDSLOTBB9tblhmENX3yFs1SBj7g7j4Z8trFPj1ylM2UQsizI/rh+8165sGRJlHvHr7DwdHihLzLVUdKsPfwjiAWP0OU4ao/kzuzEI0gerpxltAp2Yix9o8KfzDj+xOG9GTDclhGc13DhTZMHjLSHZ1YCaR68JtCqceHGdG5OIjTVbHbfDU9VNLeabuQM7Ph2Q4tApAj+In5GFjArK4BK/2qaxoPY5+kc04DR1vWq/Ku3OUUMwBLvioFsWbtGRchs3LZceQTBHHiRNch4cct2oAn2AtQbvTjiFJC32zxfGpxV2Xx+NVuKs8OxM42UECa604zxs/0oOPKYoZBC6qbeQQS/uPh0ECJPfuKXU5QtYL6EIqteVDbPKX0HyQQK+D6TW0Y6aJ2iE= + template: + metadata: + creationTimestamp: null + name: airflow-s3-connection + namespace: stackable-airflow + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops + type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/garage/garage.yaml b/demos/argo-cd-git-ops/manifests/garage/garage.yaml new file mode 100644 index 00000000..abde204a --- /dev/null +++ b/demos/argo-cd-git-ops/manifests/garage/garage.yaml @@ -0,0 +1,316 @@ +--- +# Garage provides the S3 storage Airflow writes its task logs to. +# +# Derived from stacks/_templates/garage.yaml. Argo CD syncs this file raw from +# git, so it must not contain any Jinja2 expressions: every value that is a +# parameter in the template is a literal here. +# +# Garage terminates no TLS on any of its endpoints, so an nginx sidecar does it +# and forwards to Garage on loopback. +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +data: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ + garage.toml: | + metadata_dir = "/var/lib/garage/meta" + data_dir = "/var/lib/garage/data" + db_engine = "lmdb" + + # Single node, no redundancy. This is a demo, not a production deployment. + replication_factor = 1 + + # Throwaway value, this cluster is never joined by another node. + rpc_secret = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + rpc_bind_addr = "[::]:3901" + rpc_public_addr = "127.0.0.1:3901" + + [s3_api] + # Garage rejects requests signed for a different region, so this must match + # the region every client signs with: AWS_DEFAULT_REGION in + # manifests/airflow/airflow.yaml and RCLONE_S3_REGION in the Job below. + s3_region = "region-1" + api_bind_addr = "[::]:3900" + + [admin] + api_bind_addr = "[::]:3903" + # Garage terminates no TLS, so nginx does it and forwards to Garage on + # loopback. Mounted over /etc/nginx/nginx.conf, hence the full file. + nginx.conf: | + events {} + http { + server { + listen 9000 ssl; + ssl_certificate /stackable/tls/tls.crt; + ssl_certificate_key /stackable/tls/tls.key; + + # Don't buffer or size-limit object uploads. + client_max_body_size 0; + + location / { + proxy_pass http://127.0.0.1:3900; + # Must be the original Host, it is part of the SigV4 signature. + proxy_set_header Host $http_host; + proxy_http_version 1.1; + proxy_buffering off; + proxy_request_buffering off; + } + } + } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: garage + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +spec: + # NodePort so the S3 API can be reached from outside the cluster, e.g. to + # inspect the buckets with `aws s3 ls` or `rclone lsf`. + type: NodePort + selector: + app: garage + ports: + - name: https + port: 9000 + targetPort: https +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: garage + labels: + app: garage + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +spec: + replicas: 1 + strategy: + # The PVC is ReadWriteOnce, so the old Pod has to release it before the new + # one can claim it. + type: Recreate + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops + spec: + containers: + - name: garage + image: oci.stackable.tech/stackable/dxflrs/garage:v2.4.1 + imagePullPolicy: IfNotPresent + # The image has no entrypoint. --single-node creates the cluster + # layout, --default-access-key creates the access key from the + # GARAGE_DEFAULT_* variables below. That key is allowed to create + # buckets, which is how the garage-init-buckets Job below creates + # the bucket the demo needs. + command: + - /garage + - server + - --single-node + - --default-access-key + env: + # Garage requires the GK<24 hex> key id format and a 64 hex char + # secret, so this is not a free form password. The values live in + # the sealed s3-credentials Secret (sealed-s3-credentials.yaml). + - name: GARAGE_DEFAULT_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: GARAGE_DEFAULT_SECRET_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey + ports: + - name: s3 + containerPort: 3900 + - name: rpc + containerPort: 3901 + - name: admin + containerPort: 3903 + readinessProbe: + httpGet: + path: /health + port: admin + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: data + mountPath: /var/lib/garage + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + - name: nginx + image: docker.io/library/nginx:1.29-alpine + imagePullPolicy: IfNotPresent + ports: + - name: https + containerPort: 9000 + volumeMounts: + - name: config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: config + configMap: + name: garage + - name: data + persistentVolumeClaim: + claimName: garage + # Request a TLS certificate from the secret-operator for nginx to serve + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=garage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech +--- +# Creates the bucket Airflow writes its task logs to. The access key created by +# --default-access-key is allowed to create buckets, so this needs no admin +# credentials, only the ordinary S3 key. `rclone mkdir` is idempotent, so +# re-running the Job is harmless. +apiVersion: batch/v1 +kind: Job +metadata: + name: garage-init-buckets + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops +spec: + template: + metadata: + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops + spec: + containers: + - name: init-buckets + image: docker.io/rclone/rclone:1.75.1 + # The image entrypoint is rclone itself, so override it for the loop. + command: + - /bin/sh + - -euo + - pipefail + - -c + - | + # Wait for Garage to accept requests. The Deployment readiness + # probe covers Garage itself, but not the nginx sidecar in front. + # Bounded, so a broken endpoint fails the Job instead of hanging. + attempts=0 + until rclone lsd :s3: >/dev/null 2>&1; do + attempts=$((attempts + 1)) + if [ "$attempts" -gt 150 ]; then + echo "Garage did not become available within 5 minutes" >&2 + rclone lsd :s3: # run once more to surface the actual error + exit 1 + fi + echo "Waiting for Garage to become available..." + sleep 2 + done + + # Airflow writes its task logs to s3://demo/airflow-task-logs/ + # (see manifests/airflow/airflow.yaml). + echo "Creating bucket 'demo'" + rclone mkdir ":s3:demo" + + rclone lsd :s3: + env: + # No config file, everything is configured through the environment. + - name: RCLONE_CONFIG + value: /dev/null + - name: RCLONE_S3_PROVIDER + value: Other + - name: RCLONE_S3_ENDPOINT + value: https://garage.garage.svc.cluster.local:9000 + # Must match s3_region in the Garage config above. + - name: RCLONE_S3_REGION + value: region-1 + # Garage only serves path style, not virtual-hosted style. + - name: RCLONE_S3_FORCE_PATH_STYLE + value: "true" + - name: RCLONE_CA_CERT + value: /stackable/tls/ca.crt + - name: RCLONE_S3_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: s3-credentials + key: accessKey + - name: RCLONE_S3_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: s3-credentials + key: secretKey + volumeMounts: + - name: tls + mountPath: /stackable/tls + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 500m + memory: 256Mi + volumes: + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/provision-parts: public + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + restartPolicy: OnFailure + backoffLimit: 50 diff --git a/demos/argo-cd-git-ops/manifests/garage/sealed-s3-credentials.yaml b/demos/argo-cd-git-ops/manifests/garage/sealed-s3-credentials.yaml new file mode 100644 index 00000000..1ed05928 --- /dev/null +++ b/demos/argo-cd-git-ops/manifests/garage/sealed-s3-credentials.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: s3-credentials + namespace: garage +spec: + encryptedData: + accessKey: AgAPU6JUXVdVAhOrPPDqjcxiThsWJrjhIPyD/KLjRAlregXvxn6QD9QStdYVo2wWSvN2AslHqVS6ede8YO9gJx89o+TnaiWN65AQeBCe1GgV0PoWdY3lusQ8Gp6AlfkFz7gMcKnDR0oD50S4LlsDd6ldT1sXL7sTWgl1lNG/PMml2D0n96cCTInnlFl7BD6fMdvtbkCkWA9ZUUrbLCBptc+6uUjk5MKVx59y+4iEzpn+RYdAeMDfcTMANSD9UDrx/y3rnTyAZl5FEqj6VOXPpR+OFS6V8lFugYEOzqSoBvSlXBjCLk2VP7m7TOJZP6tISnmJ+n2L1TtemMbU7LCgBVsGF1q2jhk1g5mtkU2tsbnEO4LNM0kNuzhfZyp0gYo5aShR+ZRj2PEuv1JiY6ZbxelBFAmx//OJ9dc8kf0I2NMrysKpZWu5OYik73zManSWG72d76WC3YrdI9n20DmvjkOvfyq2jrks9FSO3de6ttQ+gcIrZEGK/mP9X7rY1xO+RVaMUdeljBJ6RKeQrzhxbr1i85deyYuPOdX+z9qghf7C7QgWf47yzO4dWt2kgbF+m+/0UEDxRTZ4/IdhAj+8p3NS9fFPE7T5CcRBAJRJlj2/H40JlYaLLgtXyjp7YcTIE2jRvP/Iuh9HYkjfj5E9G46TgT/lAxwsDn6tzKog8uyg62yabTHEL2W0N/lN5SdYcAYcyCAr8k/x5tCNrLtzPDbteQouMlPVoVqaFg== + secretKey: AgANeUwlarxs4e19S+lsqpXML9EkwI8aosIdE5qeEZtd/vzaW+owZWictWeitLmvMDLPOJInibFXuOsQOWY/+nCEyo9XZN/g0TmH9hi6SE+v1jENobLw0Qnt29G0KgSFb1sqoi69lwXb5OLccLbAyeCdknyr77q4Ofxk1QB0r6Kxl8uaHHBPjMo/7YBOHJTCCTYE4p8Vs9RwcL1np5QxYQ/ePFMfPaXO+r8BXnyjKAuR8kt3LePvaS3VBRZRfjK/Av3lJbT0sYIT7akXM8GNTB7vYM41gDZo7gI2516r8MrE69S2as2ymHgEEeBaynpKgXJyXISjX3Pct8ytNGEx/tsiKeP/yWBfc5CfzU4J+Daa+vcSADz/PgXH7OB9cLeDAsYUo/BnIeSpxex440IylFDBPmDoSut9Ifh2e4FllB9SLvubBhqOpKZLHqNq+/JqCYHqoWXjIJwgFPh0A1caqNLMpXb0wSvNlZWNnecjqz+IXIf9IHxVAYkxHm6XHBmuriiPYrcubc5VO1b8v9NcgLOZrjSyW3ToSrwRWgoe4QYRas7YjaDVfDvSk5leV4+syLoRoN8Qk5pgKUrecNTFaOYOepba4Ma+IV/fLzdDVrhNmmYAlFX/x8vWBfs5DLSfK3i5+FTVavEEymJfWZidl+ZEInrj2h3cVVNG38KqpmJr56QFf2TeEn83P+Nb9bPd441P71UQbHHETbizdbIrx9k+TQ57i5+GfcfsoRfikFTblx1G+3F7RuaSDFDWO0nLR4rfrP1qJhlNyXgf3A754lA8 + template: + metadata: + creationTimestamp: null + name: s3-credentials + namespace: garage + labels: + stackable.tech/vendor: Stackable + stackable.tech/demo: argo-cd-git-ops + type: Opaque diff --git a/demos/argo-cd-git-ops/manifests/minio/minio.yaml b/demos/argo-cd-git-ops/manifests/minio/minio.yaml deleted file mode 100644 index 731d1109..00000000 --- a/demos/argo-cd-git-ops/manifests/minio/minio.yaml +++ /dev/null @@ -1,706 +0,0 @@ ---- -# Source: minio/templates/serviceaccount.yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - name: "minio-sa" - labels: - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops ---- -# Source: minio/templates/configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops -data: - initialize: |- - #!/bin/sh - set -e # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 - LIMIT=29 # Allow 30 attempts - set -e # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) - SECRET=$(cat /config/rootPassword) - set +e # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" - $MC_COMMAND - STATUS=$? - until [ $STATUS = 0 ]; do - ATTEMPTS=$(expr $ATTEMPTS + 1) - echo \"Failed attempts: $ATTEMPTS\" - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 - fi - sleep 2 # 1 second intervals between attempts - $MC_COMMAND - STATUS=$? - done - set -e # reset `e` as active - return 0 - } - - # checkBucketExists ($bucket) - # Check if the bucket exists, by using the exit code of `mc ls` - checkBucketExists() { - BUCKET=$1 - CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) - return $? - } - - # createBucket ($bucket, $policy, $purge) - # Ensure bucket exists, purging if asked to - createBucket() { - BUCKET=$1 - POLICY=$2 - PURGE=$3 - VERSIONING=$4 - OBJECTLOCKING=$5 - - # Purge the bucket, if set & exists - # Since PURGE is user input, check explicitly for `true` - if [ $PURGE = true ]; then - if checkBucketExists $BUCKET; then - echo "Purging bucket '$BUCKET'." - set +e # don't exit if this fails - ${MC} rm -r --force myminio/$BUCKET - set -e # reset `e` as active - else - echo "Bucket '$BUCKET' does not exist, skipping purge." - fi - fi - - # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) - if ! checkBucketExists $BUCKET; then - if [ ! -z $OBJECTLOCKING ]; then - if [ $OBJECTLOCKING = true ]; then - echo "Creating bucket with OBJECTLOCKING '$BUCKET'" - ${MC} mb --with-lock myminio/$BUCKET - elif [ $OBJECTLOCKING = false ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - fi - elif [ -z $OBJECTLOCKING ]; then - echo "Creating bucket '$BUCKET'" - ${MC} mb myminio/$BUCKET - else - echo "Bucket '$BUCKET' already exists." - fi - fi - - # set versioning for bucket if objectlocking is disabled or not set - if [ $OBJECTLOCKING = false ]; then - if [ ! -z $VERSIONING ]; then - if [ $VERSIONING = true ]; then - echo "Enabling versioning for '$BUCKET'" - ${MC} version enable myminio/$BUCKET - elif [ $VERSIONING = false ]; then - echo "Suspending versioning for '$BUCKET'" - ${MC} version suspend myminio/$BUCKET - fi - fi - else - echo "Bucket '$BUCKET' versioning unchanged." - fi - - # At this point, the bucket should exist, skip checking for existence - # Set policy on the bucket - echo "Setting policy of bucket '$BUCKET' to '$POLICY'." - ${MC} anonymous set $POLICY myminio/$BUCKET - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the buckets - createBucket demo "public" false false false - - add-user: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkUserExists () - # Check if the user exists, by using the exit code of `mc admin user info` - checkUserExists() { - CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createUser ($policy) - createUser() { - POLICY=$1 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the user if it does not exist - if ! checkUserExists ; then - echo "Creating user '$USER'" - cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio - else - echo "User '$USER' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - - # set policy for user - if [ ! -z $POLICY -a $POLICY != " " ] ; then - echo "Adding policy '$POLICY' for '$USER'" - set +e ; # policy already attach errors out, allow it. - ${MC} admin policy attach myminio $POLICY --user=$USER - set -e - else - echo "User '$USER' has no policy attached." - fi - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - # Create the users - echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP - echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP - createUser consoleAdmin - - add-policy: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkPolicyExists ($policy) - # Check if the policy exists, by using the exit code of `mc admin policy info` - checkPolicyExists() { - POLICY=$1 - CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) - return $? - } - - # createPolicy($name, $filename) - createPolicy () { - NAME=$1 - FILENAME=$2 - - # Create the name if it does not exist - echo "Checking policy: $NAME (in /config/$FILENAME.json)" - if ! checkPolicyExists $NAME ; then - echo "Creating policy '$NAME'" - else - echo "Policy '$NAME' already exists." - fi - ${MC} admin policy create myminio $NAME /config/$FILENAME.json - - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - add-svcacct: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. - # Special characters for example : ',",<,>,{,} - MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 2 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # checkSvcacctExists () - # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` - checkSvcacctExists() { - CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) - return $? - } - - # createSvcacct ($user) - createSvcacct () { - USER=$1 - FILENAME=$2 - #check accessKey_and_secretKey_tmp file - if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then - echo "credentials file does not exist" - return 1 - fi - if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then - echo "credentials file is invalid" - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - return 1 - fi - SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) - # Create the svcacct if it does not exist - if ! checkSvcacctExists ; then - echo "Creating svcacct '$SVCACCT'" - # Check if policy file is define - if [ -z $FILENAME ]; then - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER - else - ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER - fi - else - echo "Svcacct '$SVCACCT' already exists." - fi - #clean up credentials files. - rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme - - custom-command: |- - #!/bin/sh - set -e ; # Have script exit in the event of a failed command. - MC_CONFIG_DIR="/etc/minio/mc/" - MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" - - # connectToMinio - # Use a check-sleep-check loop to wait for MinIO service to be available - connectToMinio() { - SCHEME=$1 - ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts - set -e ; # fail if we can't read the keys. - ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; - set +e ; # The connections to minio are allowed to fail. - echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; - MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; - $MC_COMMAND ; - STATUS=$? ; - until [ $STATUS = 0 ] - do - ATTEMPTS=`expr $ATTEMPTS + 1` ; - echo \"Failed attempts: $ATTEMPTS\" ; - if [ $ATTEMPTS -gt $LIMIT ]; then - exit 1 ; - fi ; - sleep 2 ; # 1 second intervals between attempts - $MC_COMMAND ; - STATUS=$? ; - done ; - set -e ; # reset `e` as active - return 0 - } - - # runCommand ($@) - # Run custom mc command - runCommand() { - ${MC} "$@" - return $? - } - - # Try connecting to MinIO instance - scheme=https - connectToMinio $scheme ---- -# Source: minio/templates/pvc.yaml -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops -spec: - accessModes: - - "ReadWriteOnce" - resources: - requests: - storage: "10Gi" ---- -# Source: minio/templates/console-service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio-console - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9001 - protocol: TCP - targetPort: 9001 - selector: - app: minio - release: minio ---- -# Source: minio/templates/service.yaml -apiVersion: v1 -kind: Service -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - monitoring: "true" - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops -spec: - type: NodePort - externalTrafficPolicy: "Cluster" - ports: - - name: https - port: 9000 - protocol: TCP - targetPort: 9000 - selector: - app: minio - release: minio ---- -# Source: minio/templates/deployment.yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops -spec: - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 100% - maxUnavailable: 0 - replicas: 1 - selector: - matchLabels: - app: minio - release: minio - template: - metadata: - name: minio - labels: - app: minio - release: minio - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops - annotations: - checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 - checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 - spec: - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - - serviceAccountName: minio-sa - containers: - - name: minio - image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # minio requires the TLS key pair to be specially named - # mkdir -p /etc/minio/certs - cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt - cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key - - /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 - volumeMounts: - - name: minio-user - mountPath: "/tmp/credentials" - readOnly: true - - name: export - mountPath: /export - - mountPath: /etc/minio/original_certs - name: tls - - mountPath: /etc/minio/certs - name: certs - ports: - - name: https - containerPort: 9000 - - name: https-console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: minio - key: rootUser - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: minio - key: rootPassword - - name: MINIO_PROMETHEUS_AUTH_TYPE - value: "public" - resources: - requests: - cpu: 1 - memory: 2Gi - securityContext: - readOnlyRootFilesystem: false - volumes: - - name: export - persistentVolumeClaim: - claimName: minio - - name: minio-user - secret: - secretName: minio - - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs ---- -# Source: minio/templates/post-job.yaml -apiVersion: batch/v1 -kind: Job -metadata: - name: minio-post-job - labels: - app: minio-post-job - chart: minio-5.4.0 - release: minio - heritage: Helm - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation -spec: - template: - metadata: - labels: - app: minio-job - release: minio - stackable.tech/vendor: Stackable - spec: - restartPolicy: OnFailure - volumes: - - name: etc-path - emptyDir: {} - - name: tmp - emptyDir: {} - - name: minio-configuration - projected: - sources: - - configMap: - name: minio - - secret: - name: minio - - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: service=minio - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 - storageClassName: secrets.stackable.tech - name: tls - - emptyDir: - medium: Memory - sizeLimit: 5Mi - name: certs - serviceAccountName: minio-sa - containers: - - name: minio-make-bucket - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/initialize - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi - - name: minio-make-user - image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" - imagePullPolicy: IfNotPresent - command: - - "/bin/sh" - - "-ce" - - | - # Copy the CA cert from the "tls" SecretClass - # mkdir -p /etc/minio/mc/certs/CAs - cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt - - . /config/add-user - env: - - name: MINIO_ENDPOINT - value: minio - - name: MINIO_PORT - value: "9000" - volumeMounts: - - name: etc-path - mountPath: /etc/minio/mc - - name: tmp - mountPath: /tmp - - name: minio-configuration - mountPath: /config - - name: tls - mountPath: /etc/minio/mc/original_certs - - name: certs - mountPath: /etc/minio/mc/certs/CAs - resources: - requests: - memory: 128Mi diff --git a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml deleted file mode 100644 index 02fd6ae0..00000000 --- a/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - creationTimestamp: null - name: minio - namespace: minio -spec: - encryptedData: - rootPassword: AgBSBE4tAAJHV28O13Z+lg4CAzk/snvC+2WAM/ErBUeg4JiQZBIT+9o0MAIwM/IPcT7TpjrIpjx9UUiM5o8b/xjBIkxYBkpr8dNPE57IX+yVyWSxe5CPmBqazxEHU6cdlVT+s2NrydxyEsreSd6gs3+sHRd88AHQUhf3QJiD+hV2UE33jRe/so1if3OG9om5mNa/8EOqXeBnNBTCynaMDDuGyUcCUyzh5MEdaLG1YUu9flYCqlXNEsIkDR+1vgmbX1gcYdGcAIF7C71g8KMnGSt5dtAzxyuDhtDTCaq4+z3FbwEoJ4JQpmiDZzqtR5FDVUD8yS9b01Weg2cSlIEeNq83qN40wn8PkDQ6GEkl8Dou6yBpVklgnNJ+zgr+CxmFjrE6tGJnKQK1eOZmVCW43QFOOBvhyqXE0gpIlTO6rTzwfawHyWT1Wl83qCCcslb+/YylQx4tl/7fdFda5aNRN+1KQrfhkO9kc1zqeaJgO2nI31OpZOxu2U86WF9cQ/acFtOcF4/DDfYOu2dvvzyfzazfj6NIP0P2DjIpqIt7lTZ5t0SYgWB+cf5afsR8Y2TrO4K5B4Ys1Q9RTojqTuL/lyjuhNxYm2b/sIaQbBoJthntNrwaQO/KJ2nWngeRA+lFZTgbWigR6eQaLZXMIA+GioKKLiQxDiLeIvqfVMO6fwQWpeqznBruuQjVcB1aH18wLejD89ze9ztBI9JB - rootUser: AgAydeD6HzjMO94+dVqBLipjRRMLqHBDnAgqvDc+GUwT7dUuenMl2Qjx6D4G9Lk6HIuoqGpW1ZNVs65w+IEIETUGBR23I+E4drp2DFkGamXng+flkCJIvvvnmVoHYSDvO1mGEaqiF109vPOqj8vHtNHDjM89DdVWvdMkLQzM/XWK8pjIY/9g+K+a2te9yM3MzzlQBh7DPw5WdL5Nn97Ip5dUMA9D6fSLNidSqz24TLBbS9cLvRHERy9BLow7MqxyK0017iNH8TnpIzWphE+6iH9qPXGLNVlmeq8Ex4L74x3BIyas+yC2ErXPlvBNb/PG+L/788d4ZHW8JRNnQRhI2u8tvrf/Gg55/9a6hu2DQ+oqgT5jSJTPUE0meHIwAIVtLfxHozHYPD0IIdsgZgj89yfzaM3uIv02TnqAaDU113DVq5r7yZhh/THz9euw82/lPJYpuYwuSOuarddz18mOYWgQrhfBgOINxOr/OsFBl2HAfxnJyE2J4N50o4g9P5idr8+Px8P2bIKmrIbYI/YWVkHuGrslyXJFvdQvo3VJVXp7qGVJLOu3BTujxHc13CfMMDQGWXP3LvUFtDYH2fn1LHl0TlSlzyNzFdRir/b75tNKWRRuyOQ4qUJGSES1sDCmK7C8BA7d4oD0Sn0HZt7n4xp+3+b/8Rmicm/w8Uh1syTaxgW/2KQA9iXmHT8Un6gET71NF9E8mg== - template: - metadata: - creationTimestamp: null - name: minio - namespace: minio - labels: - stackable.tech/vendor: Stackable - stackable.tech/demo: argo-cd-git-ops - type: Opaque diff --git a/demos/argo-cd-git-ops/projects/minio.yaml b/demos/argo-cd-git-ops/projects/garage.yaml similarity index 73% rename from demos/argo-cd-git-ops/projects/minio.yaml rename to demos/argo-cd-git-ops/projects/garage.yaml index 38fee1df..d1defa73 100644 --- a/demos/argo-cd-git-ops/projects/minio.yaml +++ b/demos/argo-cd-git-ops/projects/garage.yaml @@ -2,13 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: - name: minio + name: garage spec: - description: Project to manage Minio + description: Project to manage Garage sourceRepos: - "*" destinations: - - namespace: minio + - namespace: garage server: https://kubernetes.default.svc clusterResourceWhitelist: - group: "*" diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index c23f1916..59da3ad8 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -26,8 +26,7 @@ demos: # products ################################ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/argo-cd-git-ops/applications/airflow.yaml - supportedNamespaces: - - argo-cd + supportedNamespaces: [] resourceRequests: cpu: 15000m memory: 15Gi @@ -38,7 +37,7 @@ demos: default: https://github.com/stackabletech/demos/ - name: customGitBranch description: The Branch in the Git repository (this can be adapted in case of forking the repository) to demonstrate syncing via Argo and Git. - default: main + default: main # release-ref: branch airflow-scheduled-job: description: Activate a simple Airflow DAG to run continuously at a set interval stackableStack: airflow diff --git a/docs/modules/demos/images/argo-cd-git-ops/minio-dag-run-logs.png b/docs/modules/demos/images/argo-cd-git-ops/minio-dag-run-logs.png deleted file mode 100644 index 9f8de472..00000000 Binary files a/docs/modules/demos/images/argo-cd-git-ops/minio-dag-run-logs.png and /dev/null differ diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 1ef8f65c..5f3610bc 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -59,13 +59,13 @@ WARNING: This demo should not be run alongside other demos. This demo can be installed on an existing Kubernetes cluster in two ways: -### 1. Exploration - interact with the ArgoCD, Airflow and Minio web UI +### 1. Exploration - interact with the ArgoCD and Airflow web UI -The "easy" way. Checkout the ArgoCD, Airflow and Minio web UI, start DAGs and explore logs. +The "easy" way. Checkout the ArgoCD and Airflow web UI, start DAGs and explore the logs stored in S3. [source,console] ---- -$ stackablectl demo install argo-cd-git-ops --namespace argo-cd +$ stackablectl demo install argo-cd-git-ops --namespace argo-cd-git-ops ---- ### 2. GitOps - interact with a Git repository to apply changes made in code into the cluster @@ -78,11 +78,13 @@ This is explained in more detail in the section < --parameters customGitBranch= +stackablectl demo install argo-cd-git-ops --namespace argo-cd-git-ops --parameters customGitUrl= --parameters customGitBranch= ---- -NOTE: ArgoCD will be deployed in the `argo-cd` namespace by `stackablectl`. -ArgoCD itself will create other namespaces for the deployed products. +NOTE: ArgoCD will be deployed by default in the `argo-cd-git-ops` namespace by `stackablectl`, but this can be changed with the `--namespace` argument. +The namespaces for the operators (`stackable-operators`) and project components (`stackable-airflow`, `garage`, `sealed-secrets`) are fixed. + +ArgoCD itself will create these other namespaces for the deployed products. == Sealed Secrets @@ -101,7 +103,7 @@ After bootstrapping ArgoCD via `stackablectl`, once the pods are ready, you can [source,console] ---- -kubectl --namespace argo-cd port-forward service/argocd-server 8443:https +kubectl --namespace argo-cd-git-ops port-forward service/argocd-server 8443:https ---- In your browser, go to `https://localhost:8443` and login with username `admin` and password `adminadmin`. @@ -283,32 +285,50 @@ image::argo-cd-git-ops/airflow-dag-overview.png[] === Inspect a DAG run A single DAG run can be selected by clicking on one of the green squares next to `run_every_minute` on the left. -More information is displayed here, and the DAG logs written by the Kubernetes Executor to S3/Minio can be selected in the `Logs` tab. +More information is displayed here, and the DAG logs written by the Kubernetes Executor to S3 (Garage) can be selected in the `Logs` tab. image::argo-cd-git-ops/airflow-dag-run-logs.png[] In the logs, the output of the DAG is printed under a line containing `Output:`, the timestamp of the DAG run. -== Minio +== S3 storage (Garage) Since the Airflow Kubernetes Executor will be deleted after its run, the logs are written to an S3 bucket. When accessing the logs via the Airflow webserver, the logs are fetched from S3 instead of the (already deleted) executor pods. -The Minio / S3 instance can be accessed via port-forward: + +The S3 storage is provided by https://garagehq.deuxfleurs.fr/[Garage], deployed by ArgoCD into the `garage` namespace. +Garage has no web interface, so the bucket is inspected with an S3 client. +Its credentials are synced from Git as a sealed secret and unsealed into the `s3-credentials` Secret: [source,console] ---- -kubectl --namespace minio port-forward service/minio-console 9001:9001 +$ export S3_ENDPOINT="https://$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}'):$(kubectl --namespace garage get service garage -o jsonpath='{.spec.ports[0].nodePort}')" +$ export S3_ACCESS_KEY="$(kubectl --namespace garage get secret s3-credentials -o jsonpath='{.data.accessKey}' | base64 -d)" +$ export S3_SECRET_KEY="$(kubectl --namespace garage get secret s3-credentials -o jsonpath='{.data.secretKey}' | base64 -d)" ---- -Minio then is reachable via `https://localhost:9001` with username `admin` and password `adminadmin`. -After the successful Airflow DAG run, logs should be stored in `demo/airflow-task-logs`. - -NOTE: There will be an initial warning from the browser, stating that the site is insecure due to self-signed certificates. -This can be ignored in this case. +`rclone` needs no installation, it runs straight from a container image. +The region is not optional: Garage rejects any request signed for a different region than `region-1`. +The certificate is issued by the Stackable secret-operator, whose CA your machine does not trust, so certificate verification is skipped. -image::argo-cd-git-ops/minio-dag-run-logs.png[] +[source,console] +---- +$ rclone() { + docker run --rm -i --network host \ + -e RCLONE_S3_PROVIDER=Other \ + -e RCLONE_S3_ENDPOINT="$S3_ENDPOINT" \ + -e RCLONE_S3_REGION=region-1 \ + -e RCLONE_S3_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ + -e RCLONE_S3_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ + -e RCLONE_S3_FORCE_PATH_STYLE=true \ + -e RCLONE_NO_CHECK_CERTIFICATE=true \ + rclone/rclone:1.75.1 "$@" + } +$ rclone lsf --recursive :s3:demo/airflow-task-logs/ +---- -The log files contained in the single folders are the same as the logs shown above in the Airflow web UI. +After a successful Airflow DAG run, its logs are stored under `demo/airflow-task-logs/`. +These are the same logs shown above in the Airflow web UI, and `rclone cat :s3:demo/airflow-task-logs/` prints one of them. [#interact-with-git-repository] == How to interact with ArgoCD, Airflow and the Git repository @@ -335,7 +355,7 @@ After forking and cloning a local copy, the `stackablectl` install command must [source,console] ---- -stackablectl demo install argo-cd-git-ops --namespace argo-cd --parameters customGitUrl= --parameters customGitBranch= +stackablectl demo install argo-cd-git-ops --namespace argo-cd-git-ops --parameters customGitUrl= --parameters customGitBranch= ---- == Making changes to the repository diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index bf7fb417..6dd9ac05 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -26,9 +26,8 @@ spec: # TODO(@maltesander): This should be parameterized with customGitUrl and customGitBranch parameters. # Currently these parameters are defined in the demo-v2.yaml and are # not picked up in the stack - What to do? - # TODO(@maltesander): Must be adapted in case of release! - repoURL: https://github.com/stackabletech/demos/ - targetRevision: main + targetRevision: main # release-ref: branch path: stacks/argo-cd-git-ops/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b70a34b3..ef328abf 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -26,8 +26,7 @@ stacks: # applicationset - operators ################################ - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/argo-cd-git-ops/applicationsets/stackable-operators.yaml - supportedNamespaces: - - argo-cd + supportedNamespaces: [] resourceRequests: memory: 5000Mi cpu: 5000m @@ -35,8 +34,7 @@ stacks: parameters: - name: stackableReleaseVersion description: The Stackable release to be installed via Argo. This replaces the demo `stackableRelease` parameter. - # TODO(@maltesander): Must be adapted in case of release ! - default: 0.0.0-dev + default: 0.0.0-dev # release-ref: version - name: argocdAdminPassword description: Password of the ArgoCD admin user. # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'`