diff --git a/docs/modules/demos/images/logging/logs.png b/docs/modules/demos/images/logging/logs.png index 729a9e30..d38c7736 100644 Binary files a/docs/modules/demos/images/logging/logs.png and b/docs/modules/demos/images/logging/logs.png differ diff --git a/docs/modules/demos/pages/logging.adoc b/docs/modules/demos/pages/logging.adoc index e496e39e..a0fcae28 100644 --- a/docs/modules/demos/pages/logging.adoc +++ b/docs/modules/demos/pages/logging.adoc @@ -2,6 +2,7 @@ :description: Deploy a logging stack with OpenSearch, Vector, and Zookeeper for log data analysis using OpenSearch Dashboards in Kubernetes. :k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu +:opensearch-data-streams: https://docs.opensearch.org/latest/im-plugin/data-streams/ Install this demo on an existing Kubernetes cluster: @@ -39,7 +40,9 @@ This demo will ** *OpenSearch*: A data store and search engine. This demo uses it to store and index the log data. ** *OpenSearch Dashboards*: A visualization and user interface. This demo uses it to make the log data easily accessible to the user. -* Create a view in OpenSearch Dashboards to conveniently browse the log data. +* Create an index template and an Index State Management (ISM) policy in OpenSearch so that the log data are stored in + a data stream which is rolled over and deleted automatically. +* Create an index pattern and a view in OpenSearch Dashboards to conveniently browse the log data. You can see the deployed products and their relationship in the following diagram: @@ -61,8 +64,8 @@ $ stackablectl stacklet list -n logging ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ │ zookeeper ┆ simple-zk ┆ logging ┆ server-zk simple-zk-server.logging.svc.cluster.local:2282 ┆ Available, Reconciling, Running │ ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┤ -│ opensearch-dashboards ┆ opensearch-dashboards ┆ logging ┆ http http://172.18.0.2:30206 ┆ │ -│ ┆ ┆ ┆ metrics 172.18.0.2:31173 ┆ │ +│ opensearch-dashboards ┆ opensearch-dashboards ┆ logging ┆ http http://10.104.213.49:31655 ┆ │ +│ ┆ ┆ ┆ metrics 10.104.213.49:30865 ┆ │ └───────────────────────┴───────────────────────┴───────────┴────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────┘ ---- @@ -70,19 +73,36 @@ include::partial$instance-hint.adoc[] == Inspect the log data -You can have a look at the log data within the OpenSearch Dashboards web interface by running -`stackablectl stacklet list` and opening the URL in the opensearch-dashboard entry's info column. In this case, it is -http://172.18.0.2:30595. +You can have a look at the log data within the OpenSearch Dashboards web interface by running `stackablectl stacklet list -n logging` and opening the HTTP URL in the opensearch-dashboards entry's ENDPOINTS column. +In this case, it is http://10.104.213.49:31655. image::logging/login.png[] Log in with the username `admin` and password `adminadmin`. -NOTE: On first login, you will be presented with some options. Feel free to bypass them to get to the logs. +[NOTE] +==== +On first login, you will be presented with some options. +Feel free to bypass them to get to the logs. +==== -Click _Discover_ in the menu on the left to view the recent logs. +OpenSearch Dashboards opens the view _Logs_ which shows the most interesting fields of the recent log data. If you do not see anything, increase the search window to greater than _Last 15 minutes_. +[NOTE] +==== +If you click _Discover_ in the menu on the left, then a new search is started instead. +Open the view _Logs_ again with _Open_ in the upper right corner. +==== + image::logging/logs.png[] -From here you can inspect the logs. You can select fields on the left side to limit the view to interesting information. +From here you can inspect the logs. +You can select fields on the left side to limit the view to interesting information. + +[#data-stream] +== Log data stream + +Vector writes all log data into the single {opensearch-data-streams}[data stream] `logs-generic-default`. +The data stream is created on the first write because the index template `logs` matches its name and contains a `data_stream` section. +The ISM policy `logs` rolls the backing indices of the data stream over once a day and deletes them after a week. diff --git a/stacks/_templates/opensearch-dashboards.yaml b/stacks/_templates/opensearch-dashboards.yaml index 8554b495..663d0f49 100644 --- a/stacks/_templates/opensearch-dashboards.yaml +++ b/stacks/_templates/opensearch-dashboards.yaml @@ -50,5 +50,3 @@ options: # Use the ServiceAccount of OpenSearch because its permissions are already configured to work on # OpenShift. name: opensearch-serviceaccount -config: - opensearch_security.multitenancy.enabled: false diff --git a/stacks/_templates/truststore.yaml b/stacks/_templates/truststore.yaml new file mode 100644 index 00000000..2d55c6d4 --- /dev/null +++ b/stacks/_templates/truststore.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: secrets.stackable.tech/v1alpha1 +kind: TrustStore +metadata: + name: truststore-pem +spec: + secretClassName: tls + format: tls-pem + targetKind: ConfigMap diff --git a/stacks/_templates/vector-aggregator.yaml b/stacks/_templates/vector-aggregator.yaml index 0a432cc7..827dcf29 100644 --- a/stacks/_templates/vector-aggregator.yaml +++ b/stacks/_templates/vector-aggregator.yaml @@ -40,7 +40,8 @@ options: - vector endpoints: - $OPENSEARCH_HOSTS - mode: bulk + # Write to data streams instead of plain indices. + mode: data_stream # The auto-detection of the API version does not work in Vector # 0.41.1 for OpenSearch, so the version must be set explicitly # (see https://github.com/vectordotdev/vector/issues/17690). @@ -53,26 +54,73 @@ options: auth: strategy: basic user: admin - password: "{{ openSearchAdminPassword }}" + password: $OPEN_SEARCH_ADMIN_PASSWORD + serviceAccount: + create: false + # Use the ServiceAccount of OpenSearch because its permissions are already configured to work + # on OpenShift. + name: opensearch-serviceaccount + initContainers: + # Wait for the index template which makes OpenSearch store the log data in data streams. + # If Vector wrote log events before this index template existed, then OpenSearch would + # create plain indices instead of data streams. + - name: wait-for-index-template + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + env: + - name: OPENSEARCH_HOSTS + valueFrom: + configMapKeyRef: + name: opensearch + key: OPENSEARCH_HOSTS + - name: OPEN_SEARCH_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-user + key: password + volumeMounts: + - name: tls + mountPath: /certs + command: + - bash + - -euo + - pipefail + - -c + - | + #!/bin/bash + # The loop is bounded so that the pod ends up in CrashLoopBackOff if the job + # `setup-opensearch` failed permanently, instead of dropping all log events silently. + for _ in $(seq 120); do + if curl \ + --fail \ + --silent \ + --show-error \ + --cacert /certs/ca.crt \ + --user "admin:${OPEN_SEARCH_ADMIN_PASSWORD}" \ + --output /dev/null \ + "${OPENSEARCH_HOSTS}/_index_template/logs"; then + exit 0 + fi + + echo 'Waiting for the index template "logs" to be created...' + sleep 5 + done + + echo 'The index template "logs" was not created within 10 minutes.' + echo 'Check the logs of the job "setup-opensearch".' + exit 1 + env: + - name: OPEN_SEARCH_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-user + key: password envFrom: - configMapRef: name: opensearch extraVolumeMounts: - # Use the certificate generated by the secret-operator - name: tls mountPath: /certs extraVolumes: - name: tls - ephemeral: - volumeClaimTemplate: - metadata: - annotations: - secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: pod - spec: - storageClassName: secrets.stackable.tech - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1 + configMap: + name: truststore-pem diff --git a/stacks/airflow/kafka.yaml b/stacks/airflow/kafka.yaml index a7c4e1b2..bf7a077e 100644 --- a/stacks/airflow/kafka.yaml +++ b/stacks/airflow/kafka.yaml @@ -1,13 +1,4 @@ --- -apiVersion: secrets.stackable.tech/v1alpha1 -kind: TrustStore -metadata: - name: truststore-pem -spec: - secretClassName: tls - format: tls-pem - targetKind: ConfigMap ---- apiVersion: kafka.stackable.tech/v1alpha1 kind: KafkaCluster metadata: diff --git a/stacks/logging/setup-opensearch-dashboards.yaml b/stacks/logging/setup-opensearch-dashboards.yaml index edec00fa..97136dd6 100644 --- a/stacks/logging/setup-opensearch-dashboards.yaml +++ b/stacks/logging/setup-opensearch-dashboards.yaml @@ -6,6 +6,9 @@ metadata: spec: template: spec: + # Use the ServiceAccount of OpenSearch because its permissions are already configured to work + # on OpenShift. + serviceAccountName: opensearch-serviceaccount containers: - name: setup-opensearch-dashboards image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev @@ -15,6 +18,9 @@ spec: secretKeyRef: name: opensearch-user key: password + volumeMounts: + - name: saved-objects + mountPath: /tmp/saved-objects command: - bash - -euo @@ -22,38 +28,72 @@ spec: - -c - | #!/bin/bash - curl \ + # OpenSearch Dashboards answers requests before all plugins are initialized. The + # import below then fails with the status code 415 (Unsupported Media Type) which + # curl does not retry, so wait for the status to become green first. + for _ in $(seq 120); do + response=$(curl \ + --silent \ + --show-error \ + --user "admin:${OPEN_SEARCH_ADMIN_PASSWORD}" \ + "http://opensearch-dashboards:5601/api/status") || true + state=$(jq -r '.status.overall.state' <<< "$response" 2> /dev/null) || true + + if [[ "$state" == "green" ]]; then + break + fi + + echo "Waiting for OpenSearch Dashboards to be ready: ${response:0:200}" + sleep 5 + done + + [[ "$state" == "green" ]] + + response=$(curl \ + --silent \ + --show-error \ --retry-connrefused \ --retry 120 \ --retry-delay 5 \ - --header "Content-Type:application/json" \ + --user "admin:${OPEN_SEARCH_ADMIN_PASSWORD}" \ --header "osd-xsrf:true" \ - --data '{ - "attributes": { - "title": "vector-*", - "timeFieldName": "timestamp" - } - }' \ - "http://admin:${OPEN_SEARCH_ADMIN_PASSWORD}@opensearch-dashboards:5601/api/saved_objects/index-pattern/vector-*" + --form file=@/tmp/saved-objects/saved-objects.ndjson \ + "http://opensearch-dashboards:5601/api/saved_objects/_import?overwrite=true") + + echo "$response" + + # The endpoint returns the status code 200 even if objects could not be imported. + [[ "$(jq -r '.success' <<< "$response")" == "true" ]] + + # Select the index pattern in Discover by default and open the view `Logs` when + # OpenSearch Dashboards is opened. Clicking `Discover` in the menu still starts a new + # search, because OpenSearch Dashboards has no setting for a default saved search. curl \ - --header "Content-Type:application/json" \ + --fail \ + --silent \ + --show-error \ + --user "admin:${OPEN_SEARCH_ADMIN_PASSWORD}" \ --header "osd-xsrf:true" \ - --data '{ - "attributes": { - "title":"Logs", - "columns": ["pod", "container", "logger", "level", "message"], - "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references": [ - { - "id": "vector-*", - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern" - } - ] - }' \ - http://admin:${OPEN_SEARCH_ADMIN_PASSWORD}@opensearch-dashboards:5601/api/saved_objects/search/logs + --json '{"changes":{"defaultIndex":"logs-*","defaultRoute":"/app/discover#/view/logs"}}' \ + --output /dev/null \ + "http://opensearch-dashboards:5601/api/opensearch-dashboards/settings" restartPolicy: OnFailure + volumes: + - name: saved-objects + configMap: + name: setup-opensearch-dashboards-saved-objects backoffLimit: 50 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: setup-opensearch-dashboards-saved-objects +data: + # The index pattern `logs-*` and the saved search `Logs` were created in the user interface and + # exported with `POST /api/saved_objects/_export`. The generated IDs were replaced with stable + # ones. The cached field list of the index pattern must be kept because Discover resolves the + # time field `@timestamp` in it and fails with + # "Could not locate that index-pattern-field (id: @timestamp)" if it is missing. + saved-objects.ndjson: | + {"id":"logs-*","type":"index-pattern","migrationVersion":{"index-pattern":"7.6.0"},"attributes":{"title":"logs-*","timeFieldName":"@timestamp","fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"arch\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"arch.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"arch\"}}},{\"count\":3,\"name\":\"cluster\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"component_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"component_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"component_id\"}}},{\"count\":0,\"name\":\"component_kind\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"component_kind.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"component_kind\"}}},{\"count\":0,\"name\":\"component_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"component_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"component_type\"}}},{\"count\":1,\"name\":\"container\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"data_stream.dataset\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"data_stream.namespace\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"data_stream.type\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"debug\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"debug.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"debug\"}}},{\"count\":0,\"name\":\"error\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"error.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"error\"}}},{\"count\":0,\"name\":\"errors\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"exclude\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"exclude.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"exclude\"}}},{\"count\":0,\"name\":\"file\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"include\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"include.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"include\"}}},{\"count\":1,\"name\":\"level\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"logger\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":1,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"msg\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"msg.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"msg\"}}},{\"count\":1,\"name\":\"namespace\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"paths\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"paths.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"paths\"}}},{\"count\":1,\"name\":\"pod\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"processed_file\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"processed_file.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"processed_file\"}}},{\"count\":0,\"name\":\"revision\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"revision.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"revision\"}}},{\"count\":2,\"name\":\"role\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":2,\"name\":\"roleGroup\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"source_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"source_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"source_type\"}}},{\"count\":0,\"name\":\"thread\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"vector.component_id\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"vector.component_id.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"vector.component_id\"}}},{\"count\":0,\"name\":\"vector.component_kind\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"vector.component_kind.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"vector.component_kind\"}}},{\"count\":0,\"name\":\"vector.component_type\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"vector.component_type.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"vector.component_type\"}}},{\"count\":0,\"name\":\"version\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"version.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"version\"}}}]"},"references":[]} + {"id":"logs","type":"search","migrationVersion":{"search":"7.9.3"},"attributes":{"columns":["level","cluster","pod","container","logger","message"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"highlightAll\":true,\"version\":true,\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Logs","version":1},"references":[{"id":"logs-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]} diff --git a/stacks/logging/setup-opensearch.yaml b/stacks/logging/setup-opensearch.yaml new file mode 100644 index 00000000..c1aec792 --- /dev/null +++ b/stacks/logging/setup-opensearch.yaml @@ -0,0 +1,163 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: setup-opensearch +spec: + template: + spec: + # Use the ServiceAccount of OpenSearch because its permissions are already configured to work + # on OpenShift. + serviceAccountName: opensearch-serviceaccount + containers: + - name: setup-opensearch + image: oci.stackable.tech/sdp/testing-tools:0.3.0-stackable0.0.0-dev + env: + - name: OPENSEARCH_HOSTS + valueFrom: + configMapKeyRef: + name: opensearch + key: OPENSEARCH_HOSTS + - name: OPEN_SEARCH_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: opensearch-user + key: password + volumeMounts: + - name: tls + mountPath: /certs + command: + - bash + - -euo + - pipefail + - -c + - | + #!/bin/bash + opensearch_api() { + local method="$1" + local path="$2" + local data="$3" + local status + + # Truncate the file so that a failed request cannot show the response of the + # previous one, see + # https://stackoverflow.com/questions/13575477/unix-shell-script-to-truncate-a-large-file + : > /tmp/response.json + + status=$(curl \ + --silent \ + --show-error \ + --retry-connrefused \ + --retry 120 \ + --retry-delay 5 \ + --cacert /certs/ca.crt \ + --user "admin:${OPEN_SEARCH_ADMIN_PASSWORD}" \ + --request "$method" \ + --json "$data" \ + --output /tmp/response.json \ + --write-out '%{http_code}' \ + "${OPENSEARCH_HOSTS}${path}") || true + + echo "$method $path -> $status" + cat /tmp/response.json + echo + + # A conflict (409) means that the object already exists which is fine if this job + # is retried. + [[ "$status" =~ ^2 || "$status" == 409 ]] + } + + # The backing indices of a data stream are not rolled over automatically. Without + # this Index State Management policy, all log documents would end up in the first + # backing index. The policy is applied when a backing index is created, so it must + # exist before the first log data are written. + opensearch_api PUT /_plugins/_ism/policies/logs '{ + "policy": { + "description": "Roll the backing indices of the log data streams over daily and delete them after a week", + "default_state": "hot", + "ism_template": [ + { + "index_patterns": ["logs-*-*"], + "priority": 200 + } + ], + "states": [ + { + "name": "hot", + "actions": [ + { + "rollover": { + "min_index_age": "1d", + "min_primary_shard_size": "10gb" + } + } + ], + "transitions": [ + { + "state_name": "delete", + "conditions": { + "min_index_age": "7d" + } + } + ] + }, + { + "name": "delete", + "actions": [ + { + "delete": {} + } + ], + "transitions": [] + } + ] + } + }' + + # Vector creates the data streams on the first write, but only if a matching index + # template with a `data_stream` section exists. Otherwise OpenSearch creates plain + # indices instead, so this template must exist before the Vector aggregator is + # installed. + # The fields are the ones which the Vector agents of the Stackable operators send to + # the aggregator. Vector renames `timestamp` to `@timestamp` in data stream mode. + opensearch_api PUT /_index_template/logs '{ + "index_patterns": ["logs-*-*"], + "data_stream": {}, + "priority": 200, + "template": { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0 + }, + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "message": { "type": "text" }, + "level": { "type": "keyword" }, + "logger": { "type": "keyword" }, + "thread": { "type": "keyword" }, + "pod": { "type": "keyword" }, + "container": { "type": "keyword" }, + "namespace": { "type": "keyword" }, + "cluster": { "type": "keyword" }, + "role": { "type": "keyword" }, + "roleGroup": { "type": "keyword" }, + "file": { "type": "keyword" }, + "errors": { "type": "keyword" }, + "data_stream": { + "properties": { + "type": { "type": "keyword" }, + "dataset": { "type": "keyword" }, + "namespace": { "type": "keyword" } + } + } + } + } + } + }' + restartPolicy: OnFailure + volumes: + - name: tls + configMap: + name: truststore-pem + backoffLimit: 50 diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b70a34b3..ea97ce31 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -82,8 +82,10 @@ stacks: - opensearch-dashboards - vector manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/truststore.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/vector-aggregator-discovery.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/logging/opensearch.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/logging/setup-opensearch.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/opensearch-dashboards.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/logging/setup-opensearch-dashboards.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/vector-aggregator.yaml @@ -155,6 +157,7 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/garage.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-airflow.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/truststore.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/kafka.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/s3-connection.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/airflow/hive-metastores.yaml