diff --git a/.scripts/stacks_and_their_demos.sh b/.scripts/stacks_and_their_demos.sh new file mode 100755 index 00000000..af7ccc2f --- /dev/null +++ b/.scripts/stacks_and_their_demos.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env sh +cat demos/demos-v2.yaml | yq -y '.demos | to_entries | group_by(.value.stackableStack) | sort_by(.[0].value.stackableStack) | map({(.[0].value.stackableStack): map(.key)}) | add' diff --git a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc index e199b0e8..bf767485 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-earthquake-data.adoc @@ -10,9 +10,16 @@ Install this demo on an existing Kubernetes cluster: +[NOTE] +==== +The namespace `nifi-kafka-druid-earthquake-data` will be assumed in this guide. +This is the default namespace used by `stackablectl` for this demo, so the `-n` argument can be omitted. +It will be created if it doesn't exist. +==== + [source,console] ---- -$ stackablectl demo install nifi-kafka-druid-earthquake-data +$ stackablectl demo install nifi-kafka-druid-earthquake-data -n nifi-kafka-druid-earthquake-data ---- [#system-requirements] @@ -54,25 +61,25 @@ To list the installed Stackable services run the following command: [source,console] ---- -$ stackablectl stacklet list - -┌───────────┬───────────────┬───────────┬─────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ -│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ -╞═══════════╪═══════════════╪═══════════╪═════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ -│ druid ┆ druid ┆ default ┆ broker-https https://172.19.0.3:32293 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ coordinator-https https://172.19.0.4:31283 ┆ │ -│ ┆ ┆ ┆ router-https https://172.19.0.3:32286 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ kafka ┆ kafka ┆ default ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.4:32321 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.4:30556 ┆ │ -│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.4:31352 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ nifi ┆ nifi ┆ default ┆ node-https https://172.19.0.2:32348 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ superset ┆ superset ┆ default ┆ node-http http://172.19.0.4:30769 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -└───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ +$ stackablectl stacklet list -n nifi-kafka-druid-earthquake-data + +┌───────────┬───────────┬──────────────────────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ +│ PRODUCT │ NAME │ NAMESPACE │ ENDPOINTS │ CONDITIONS │ +╞═══════════╪═══════════╪══════════════════════════════════╪══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ +│ druid ┆ druid ┆ nifi-kafka-druid-earthquake-data ┆ broker-https https://172.19.0.3:32293 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ coordinator-https https://172.19.0.4:31283 ┆ │ +│ ┆ ┆ ┆ router-https https://172.19.0.3:32286 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ kafka ┆ kafka ┆ nifi-kafka-druid-earthquake-data ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.4:32321 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.4:30556 ┆ │ +│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.4:31352 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ nifi ┆ nifi ┆ nifi-kafka-druid-earthquake-data ┆ node-https https://172.19.0.2:32348 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ superset ┆ superset ┆ nifi-kafka-druid-earthquake-data ┆ node-http http://172.19.0.4:30769 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ zookeeper ┆ zookeeper ┆ nifi-kafka-druid-earthquake-data ┆ server-zk zookeeper-server.nifi-kafka-druid-earthquake-data.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ +└───────────┴───────────┴──────────────────────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- include::partial$instance-hint.adoc[] @@ -97,12 +104,12 @@ You can execute a command on the Kafka broker to list the available topics as fo // However, the kcat-prober container has TLS certificates mounted, which can be used by kcat to connect to Kafka. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-topics.sh \ --describe \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties -... Topic: earthquakes TopicId: ND51v_XcQPK4Ilm7A35Pag PartitionCount: 8 ReplicationFactor: 1 Configs: min.insync.replicas=1,segment.bytes=100000000,retention.bytes=900000000 Topic: earthquakes Partition: 0 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: Topic: earthquakes Partition: 1 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: @@ -120,10 +127,11 @@ parameter. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic earthquakes \ --offset earliest \ --partition 0 \ @@ -164,23 +172,24 @@ If you are interested in how many records have been produced to the Kafka topic [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-earthquake-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-get-offsets.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-earthquake-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties \ --topic earthquakes -... -earthquakes:0:757379 -earthquakes:1:759282 -earthquakes:2:761924 -earthquakes:3:761339 -earthquakes:4:759059 -earthquakes:5:767695 -earthquakes:6:771457 -earthquakes:7:768301 +earthquakes:0:382154 +earthquakes:1:382497 +earthquakes:2:380081 +earthquakes:3:383763 +earthquakes:4:380450 +earthquakes:5:384584 +earthquakes:6:379054 +earthquakes:7:380635 ---- -If you calculate `765,000` records * `8` partitions, you end up with ~ 6,120,000 records. +Summing the offsets of all `8` partitions gives the total number of records produced, ~ 3,050,000 in this case. +The demo ingests the complete earthquake catalogue once, so the offsets stop growing when the import has finished. == NiFi @@ -396,6 +405,15 @@ Opening up a prefix for a specific year shows the file containing the data of th $ rclone lsl :s3:demo/data/earthquakes ---- +`size` totals the segments up, once the whole data set has been ingested: + +[source,console] +---- +$ rclone size :s3:demo/data/earthquakes +Total objects: 73 +Total size: 201.537 MiB (211326832 Byte) +---- + == Summary The demo streamed 10,000 earthquake records/s for a total of ~3 million earthquakes into a Kafka steaming pipeline. diff --git a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc index 84ef8d56..599a0450 100644 --- a/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc +++ b/docs/modules/demos/pages/nifi-kafka-druid-water-level-data.adoc @@ -10,9 +10,16 @@ Install this demo on an existing Kubernetes cluster: +[NOTE] +==== +The namespace `nifi-kafka-druid-water-level-data` will be assumed in this guide. +This is the default namespace used by `stackablectl` for this demo, so the `-n` argument can be omitted. +It will be created if it doesn't exist. +==== + [source,console] ---- -$ stackablectl demo install nifi-kafka-druid-water-level-data +$ stackablectl demo install nifi-kafka-druid-water-level-data -n nifi-kafka-druid-water-level-data ---- [#system-requirements] @@ -59,25 +66,25 @@ To list the installed Stackable services run the following command: [source,console] ---- -$ stackablectl stacklet list +$ stackablectl stacklet list -n nifi-kafka-druid-water-level-data -┌───────────┬───────────────┬───────────┬─────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ -│ PRODUCT ┆ NAME ┆ NAMESPACE ┆ ENDPOINTS ┆ CONDITIONS │ -╞═══════════╪═══════════════╪═══════════╪═════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ -│ druid ┆ druid ┆ default ┆ broker-https https://172.19.0.3:32165 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ coordinator-https https://172.19.0.6:31287 ┆ │ -│ ┆ ┆ ┆ router-https https://172.19.0.3:31965 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ kafka ┆ kafka ┆ default ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.3:31041 ┆ Available, Reconciling, Running │ -│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.3:31503 ┆ │ -│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.3:30793 ┆ │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ nifi ┆ nifi ┆ default ┆ node-https https://172.19.0.6:32038 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ superset ┆ superset ┆ default ┆ node-http http://172.19.0.3:30435 ┆ Available, Reconciling, Running │ -├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ zookeeper ┆ zookeeper ┆ default ┆ server-zk zookeeper-server.default.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ -└───────────┴───────────────┴───────────┴─────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ +┌───────────┬───────────┬───────────────────────────────────┬───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─────────────────────────────────┐ +│ PRODUCT │ NAME │ NAMESPACE │ ENDPOINTS │ CONDITIONS │ +╞═══════════╪═══════════╪═══════════════════════════════════╪═══════════════════════════════════════════════════════════════════════════════════════════════════════════════════════╪═════════════════════════════════╡ +│ druid ┆ druid ┆ nifi-kafka-druid-water-level-data ┆ broker-https https://172.19.0.3:32165 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ coordinator-https https://172.19.0.6:31287 ┆ │ +│ ┆ ┆ ┆ router-https https://172.19.0.3:31965 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ kafka ┆ kafka ┆ nifi-kafka-druid-water-level-data ┆ broker-default-0-listener-broker-kafka-tls 172.19.0.3:31041 ┆ Available, Reconciling, Running │ +│ ┆ ┆ ┆ broker-default-0-listener-broker-metrics 172.19.0.3:31503 ┆ │ +│ ┆ ┆ ┆ broker-default-bootstrap-kafka-tls 172.19.0.3:30793 ┆ │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ nifi ┆ nifi ┆ nifi-kafka-druid-water-level-data ┆ node-https https://172.19.0.6:32038 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ superset ┆ superset ┆ nifi-kafka-druid-water-level-data ┆ node-http http://172.19.0.3:30435 ┆ Available, Reconciling, Running │ +├╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ +│ zookeeper ┆ zookeeper ┆ nifi-kafka-druid-water-level-data ┆ server-zk zookeeper-server.nifi-kafka-druid-water-level-data.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ +└───────────┴───────────┴───────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -102,12 +109,12 @@ You can execute a command on the Kafka broker to list the available topics as fo [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-topics.sh \ --describe \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties -... Topic: measurements TopicId: w9qYb3GaTvCMZj4G8pkPPQ PartitionCount: 8 ReplicationFactor: 1 Configs: min.insync.replicas=1,segment.bytes=100000000,retention.bytes=900000000 Topic: measurements Partition: 0 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: Topic: measurements Partition: 1 Leader: 1243966388 Replicas: 1243966388 Isr: 1243966388 Elr: LastKnownElr: @@ -138,10 +145,11 @@ print via the `--max-messages` parameter. [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic stations \ --offset earliest \ --partition 0 \ @@ -184,10 +192,11 @@ Below is an example of the output of two records: [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic measurements \ --offset earliest \ --partition 0 \ @@ -255,12 +264,12 @@ It prints the current offset for each partition, which tells you how many record [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-get-offsets.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ --command-config /stackable/config/client.properties \ --topic measurements -... measurements:0:1366665 measurements:1:1364930 measurements:2:1395607 @@ -278,15 +287,15 @@ To inspect the last produced records, use the following command. Here, we consum [source,console] ---- -$ kubectl exec kafka-broker-default-0 -c kafka -- \ +$ kubectl exec -n nifi-kafka-druid-water-level-data kafka-broker-default-0 -c kafka -- \ +env KAFKA_LOG4J_OPTS=-Dlog4j2.configurationFile=/stackable/kafka/config/tools-log4j2.yaml \ /stackable/kafka/bin/kafka-console-consumer.sh \ ---bootstrap-server kafka-broker-default-headless.default.svc.cluster.local:9093 \ ---consumer.config /stackable/config/client.properties \ +--bootstrap-server kafka-broker-default-headless.nifi-kafka-druid-water-level-data.svc.cluster.local:9093 \ +--command-config /stackable/config/client.properties \ --topic measurements \ --offset latest \ --partition 0 \ --max-messages 3 -... {"timestamp":"2025-10-21T11:00:00+02:00","value":369.54,"station_uuid":"5cdc6555-87d7-4fcd-834d-cbbe24c9d08b"} {"timestamp":"2025-10-21T11:15:00+02:00","value":369.54,"station_uuid":"5cdc6555-87d7-4fcd-834d-cbbe24c9d08b"} {"timestamp":"2025-10-21T11:00:00+02:00","value":8.0,"station_uuid":"7deedc21-2878-40cc-ab47-f6da0d9002f1"} @@ -604,6 +613,13 @@ Opening up a prefix for a specific day shows the file containing that day's data $ rclone lsl :s3:demo/data/measurements ---- +`size` totals the segments up: + +[source,console] +---- +$ rclone size :s3:demo/data/measurements +---- + == Summary The demo puts station records into the Kafka stream pipeline topic `station`. diff --git a/stacks/nifi-kafka-druid-superset-s3/druid.yaml b/stacks/nifi-kafka-druid-superset-s3/druid.yaml index 76523c4d..650314e1 100644 --- a/stacks/nifi-kafka-druid-superset-s3/druid.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/druid.yaml @@ -31,7 +31,7 @@ spec: name: region-1 accessStyle: Path credentials: - secretClass: druid-s3-credentials + secretClass: s3-credentials tls: verification: server: @@ -44,6 +44,11 @@ spec: configOverrides: &configOverrides runtime.properties: druid.server.http.enforceStrictSNIHostChecking: "false" + # Druid grants the bucket owner full control on every pushed segment, + # which means a GetBucketAcl call. Garage returns no owner in that + # response, so S3DataSegmentPusher dies with a NullPointerException and + # no segment ever reaches deep storage. + druid.storage.disableAcl: "true" roleGroups: default: replicas: 1 @@ -101,7 +106,7 @@ spec: apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass metadata: - name: druid-s3-credentials + name: s3-credentials spec: backend: k8sSearch: @@ -111,9 +116,9 @@ spec: apiVersion: v1 kind: Secret metadata: - name: druid-s3-credentials + name: s3-credentials labels: - secrets.stackable.tech/class: druid-s3-credentials + secrets.stackable.tech/class: s3-credentials stringData: accessKey: GK31c0ffee31c0ffee31c0ffee secretKey: deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef