From 7ddd7cceb2604a08b8ace8d8b9c3a4f447452d84 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 16 Nov 2026 09:06:07 +0100 Subject: [PATCH 1/6] chore(stack/nifi-kafka-druid-superset-s3): Use a consistent s3-credentals Secret/SecretClass This makes the s3 partial usable in the associated demo. SecretClasses are cluster-scoped, and so we should really be prefixing them with the demo name or something (in case multiple demos have been deployed). --- stacks/nifi-kafka-druid-superset-s3/druid.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index 76523c4d..f61d189a 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -31,7 +31,7 @@ spec: name: region-1 accessStyle: Path credentials: - secretClass: druid-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -101,7 +101,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: druid-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -111,9 +111,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: druid-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: druid-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: accessKey: GK31c0ffee31c0ffee31c0ffee secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef From 80b5f2e99971d4057b108895ad8a2f2538a502a1 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 16 Nov 2026 09:11:27 +0100 Subject: [PATCH 2/6] fix(stack/nifi-kafka-druid-superset-s3): Disable Druid S3 ACLs They don't work with garage, so Druid never flushes to S3. --- stacks/nifi-kafka-druid-superset-s3/druid.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index f61d189a..e41d7eff 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -44,6 +44,11 @@ spec: configOverrides: &configOverrides runtime.properties: druid.server.http.enforceStrictSNIHostChecking: "false" + # Druid grants the bucket owner full control on every pushed segment, + # which means a GetBucketAcl call. Garage returns no owner in that + # response, so S3DataSegmentPusher dies with a NullPointerException and + # no segment ever reaches deep storage. + druid.storage.disableAcl: "true" roleGroups: default: replicas: 1 From f674f4923750aafe353637fab6f2bf5c16f8b50b Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 16 Nov 2026 09:21:11 +0100 Subject: [PATCH 3/6] docs(demo/nifi-kafka-druid-earthquake-data): Small improvements - Use the same namespace as what stackablectl would default to - Remove noise from the Kafka commands and replace a deprecated command arg, - Correct the record count (it appears to have been run twice when the original docs were written). --- .../nifi-kafka-druid-earthquake-data.adoc | 94 +++++++++++-------- 1 file changed, 56 insertions(+), 38 deletions(-) diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc index e199b0e8..bf767485 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc @@ -10,9 +10,16 @@ Install this demo on an existing Kubernetes cluster: +[NOTE] +==== +The namespace `nifi-kafka-druid-earthquake-data` will be assumed in this guide. +This is the default namespace used by `stackablectl` for this demo, so the `-n` argument can be omitted. +It will be created if it doesn't exist. +==== + [source,console] ---- -$ stackablectl demo install nifi-kafka-druid-earthquake-data +$ stackablectl demo install nifi-kafka-druid-earthquake-data -n nifi-kafka-druid-earthquake-data ---- [#system-requirements] @@ -54,25 +61,25 @@ To list the installed Stackable services run the following command: [source,console] ---- -$ stackablectl stacklet list - -┌───────────┬───────────────┬───────────┬─────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ -│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ -╞═══════════╪═══════════════╪═══════════╪═════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ -│ druid ┆ druid ┆ default ┆ broker-https https://172.19.0.3:32293 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ coordinator-https https://172.19.0.4:31283 ┆ │ -│ ┆ ┆ ┆ router-https https://172.19.0.3:32286 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ kafka ┆ kafka ┆ default ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.4:32321 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.4:30556 ┆ │ -│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.4:31352 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ nifi ┆ nifi ┆ default ┆ node-https https://172.19.0.2:32348 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ superset ┆ superset ┆ default ┆ node-http http://172.19.0.4:30769 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -└───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ +$ stackablectl stacklet list -n nifi-kafka-druid-earthquake-data + +┌───────────┬───────────┬──────────────────────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ +│ PRODUCT │ NAME │ NAMESPACE │ ENDPOINTS │ CONDITIONS │ +╞═══════════╪═══════════╪══════════════════════════════════╪══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ +│ druid ┆ druid ┆ nifi-kafka-druid-earthquake-data ┆ broker-https https://172.19.0.3:32293 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ coordinator-https https://172.19.0.4:31283 ┆ │ +│ ┆ ┆ ┆ router-https https://172.19.0.3:32286 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ kafka ┆ kafka ┆ nifi-kafka-druid-earthquake-data ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.4:32321 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.4:30556 ┆ │ +│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.4:31352 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ nifi ┆ nifi ┆ nifi-kafka-druid-earthquake-data ┆ node-https https://172.19.0.2:32348 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ superset ┆ superset ┆ nifi-kafka-druid-earthquake-data ┆ node-http http://172.19.0.4:30769 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ zookeeper ┆ zookeeper ┆ nifi-kafka-druid-earthquake-data ┆ server-zk zookeeper-server.nifi-kafka-druid-earthquake-data.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ +└───────────┴───────────┴──────────────────────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- include::partial$instance-hint.adoc[] @@ -97,12 +104,12 @@ You can execute a command on the Kafka broker to list the available topics as fo // However, the kcat-prober container has TLS certificates mounted, which can be used by kcat to connect to Kafka. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-topics.sh \ --describe \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties -... Topic: earthquakes TopicId: ND51v_XcQPK4Ilm7A35Pag PartitionCount: 8 ReplicationFactor: 1 Configs: min.insync.replicas=1,segment.bytes=100000000,retention.bytes=900000000 Topic: earthquakes Partition: 0 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: Topic: earthquakes Partition: 1 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: @@ -120,10 +127,11 @@ parameter. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic earthquakes \ --offset earliest \ --partition 0 \ @@ -164,23 +172,24 @@ If you are interested in how many records have been produced to the Kafka topic [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-get-offsets.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties \ --topic earthquakes -... -earthquakes:0:757379 -earthquakes:1:759282 -earthquakes:2:761924 -earthquakes:3:761339 -earthquakes:4:759059 -earthquakes:5:767695 -earthquakes:6:771457 -earthquakes:7:768301 +earthquakes:0:382154 +earthquakes:1:382497 +earthquakes:2:380081 +earthquakes:3:383763 +earthquakes:4:380450 +earthquakes:5:384584 +earthquakes:6:379054 +earthquakes:7:380635 ---- -If you calculate `765,000` records * `8` partitions, you end up with ~ 6,120,000 records. +Summing the offsets of all `8` partitions gives the total number of records produced, ~ 3,050,000 in this case. +The demo ingests the complete earthquake catalogue once, so the offsets stop growing when the import has finished. == NiFi @@ -396,6 +405,15 @@ Opening up a prefix for a specific year shows the file containing the data of th $ rclone lsl :s3:demo/data/earthquakes ---- +`size` totals the segments up, once the whole data set has been ingested: + +[source,console] +---- +$ rclone size :s3:demo/data/earthquakes +Total objects: 73 +Total size: 201.537 MiB (211326832 Byte) +---- + == Summary The demo streamed 10,000 earthquake records/s for a total of ~3 million earthquakes into a Kafka steaming pipeline. From af97de79a1fa5238a2fbb073037c426d544370a6 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 16 Nov 2026 09:22:30 +0100 Subject: [PATCH 4/6] docs(demo/nifi-kafka-druid-water-level-data): Small improvements - Use the same namespace as what stackablectl would default to - Remove noise from the Kafka commands and replace a deprecated command arg, --- .../nifi-kafka-druid-water-level-data.adoc | 86 +++++++++++-------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 84ef8d56..599a0450 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -10,9 +10,16 @@ Install this demo on an existing Kubernetes cluster: +[NOTE] +==== +The namespace `nifi-kafka-druid-water-level-data` will be assumed in this guide. +This is the default namespace used by `stackablectl` for this demo, so the `-n` argument can be omitted. +It will be created if it doesn't exist. +==== + [source,console] ---- -$ stackablectl demo install nifi-kafka-druid-water-level-data +$ stackablectl demo install nifi-kafka-druid-water-level-data -n nifi-kafka-druid-water-level-data ---- [#system-requirements] @@ -59,25 +66,25 @@ To list the installed Stackable services run the following command: [source,console] ---- -$ stackablectl stacklet list +$ stackablectl stacklet list -n nifi-kafka-druid-water-level-data -┌───────────┬───────────────┬───────────┬─────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ -│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ -╞═══════════╪═══════════════╪═══════════╪═════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ -│ druid ┆ druid ┆ default ┆ broker-https https://172.19.0.3:32165 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ coordinator-https https://172.19.0.6:31287 ┆ │ -│ ┆ ┆ ┆ router-https https://172.19.0.3:31965 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ kafka ┆ kafka ┆ default ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.3:31041 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.3:31503 ┆ │ -│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.3:30793 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ nifi ┆ nifi ┆ default ┆ node-https https://172.19.0.6:32038 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ superset ┆ superset ┆ default ┆ node-http http://172.19.0.3:30435 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -└───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ +┌───────────┬───────────┬───────────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ +│ PRODUCT │ NAME │ NAMESPACE │ ENDPOINTS │ CONDITIONS │ +╞═══════════╪═══════════╪═══════════════════════════════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ +│ druid ┆ druid ┆ nifi-kafka-druid-water-level-data ┆ broker-https https://172.19.0.3:32165 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ coordinator-https https://172.19.0.6:31287 ┆ │ +│ ┆ ┆ ┆ router-https https://172.19.0.3:31965 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ kafka ┆ kafka ┆ nifi-kafka-druid-water-level-data ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.3:31041 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.3:31503 ┆ │ +│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.3:30793 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ nifi ┆ nifi ┆ nifi-kafka-druid-water-level-data ┆ node-https https://172.19.0.6:32038 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ superset ┆ superset ┆ nifi-kafka-druid-water-level-data ┆ node-http http://172.19.0.3:30435 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ zookeeper ┆ zookeeper ┆ nifi-kafka-druid-water-level-data ┆ server-zk zookeeper-server.nifi-kafka-druid-water-level-data.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ +└───────────┴───────────┴───────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -102,12 +109,12 @@ You can execute a command on the Kafka broker to list the available topics as fo [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-topics.sh \ --describe \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties -... Topic: measurements TopicId: w9qYb3GaTvCMZj4G8pkPPQ PartitionCount: 8 ReplicationFactor: 1 Configs: min.insync.replicas=1,segment.bytes=100000000,retention.bytes=900000000 Topic: measurements Partition: 0 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: Topic: measurements Partition: 1 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: @@ -138,10 +145,11 @@ print via the `--max-messages` parameter. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic stations \ --offset earliest \ --partition 0 \ @@ -184,10 +192,11 @@ Below is an example of the output of two records: [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic measurements \ --offset earliest \ --partition 0 \ @@ -255,12 +264,12 @@ It prints the current offset for each partition, which tells you how many record [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-get-offsets.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties \ --topic measurements -... measurements:0:1366665 measurements:1:1364930 measurements:2:1395607 @@ -278,15 +287,15 @@ To inspect the last produced records, use the following command. Here, we consum [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic measurements \ --offset latest \ --partition 0 \ --max-messages 3 -... {"timestamp":"2025-10-21T11:00:00+02:00","value":369.54,"station_uuid":"5cdc6555-87d7-4fcd-834d-cbbe24c9d08b"} {"timestamp":"2025-10-21T11:15:00+02:00","value":369.54,"station_uuid":"5cdc6555-87d7-4fcd-834d-cbbe24c9d08b"} {"timestamp":"2025-10-21T11:00:00+02:00","value":8.0,"station_uuid":"7deedc21-2878-40cc-ab47-f6da0d9002f1"} @@ -604,6 +613,13 @@ Opening up a prefix for a specific day shows the file containing that day's data $ rclone lsl :s3:demo/data/measurements ---- +`size` totals the segments up: + +[source,console] +---- +$ rclone size :s3:demo/data/measurements +---- + == Summary The demo puts station records into the Kafka stream pipeline topic `station`. From cd7391697679dac3bf9e0aca4406f9a5aea30eca Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Mon, 16 Nov 2026 09:31:30 +0100 Subject: [PATCH 5/6] chore: Add one-liner helper script to see the demos used by each stack. --- .scripts/stacks_and_their_demos.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 .scripts/stacks_and_their_demos.sh diff --git a/.scripts/stacks_and_their_demos.sh b/.scripts/stacks_and_their_demos.sh new file mode 100755 index 00000000..af7ccc2f --- /dev/null +++ b/.scripts/stacks_and_their_demos.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +cat demos/demos-v2.yaml | yq -y '.demos | to_entries | group_by(.value.stackableStack) | sort_by(.[0].value.stackableStack) | map({(.[0].value.stackableStack): map(.key)}) | add' From 4dac46f29502d3275cb60fdf7f3660eb98cd65a8 Mon Sep 17 00:00:00 2001 From: Nick <10092581+NickLarsenNZ@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:01:19 +0100 Subject: [PATCH 6/6] Apply suggestion from @NickLarsenNZ --- stacks/nifi-kafka-druid-superset-s3/druid.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index e41d7eff..650314e1 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -31,7 +31,7 @@ spec: name: region-1 accessStyle: Path credentials: - secretClass: s3-credentials + secretClass: s3-credentials tls: verification: server: