Update to v9.0.0

List of changes impacting docker-elk:

- [kibana]: Several features now require an encryption key for saved
  objects. Such key must be set via the configuration option
  `xpack.encryptedSavedObjects.encryptionKey`.

  Ref: https://www.elastic.co/guide/en/kibana/8.18/xpack-security-secure-saved-objects.html

- [enterprise-search]: Discontinued in 9.x versions.

  Refs:
  - https://www.elastic.co/guide/en/enterprise-search/8.18/upgrading-to-9-x.html
  - https://www.elastic.co/blog/app-search-to-elasticsearch

.env

@@ -1,4 +1,4 @@
-ELASTIC_VERSION=8.17.4
+ELASTIC_VERSION=9.0.0
 
 ## Passwords for stack users
 #

.github/workflows/ci.yml

@@ -35,7 +35,6 @@ jobs:
           -f extensions/metricbeat/metricbeat-compose.yml
           -f extensions/filebeat/filebeat-compose.yml
           -f extensions/heartbeat/heartbeat-compose.yml
-          -f extensions/enterprise-search/enterprise-search-compose.yml
           build
 
       ########################################################
@@ -58,9 +57,7 @@ jobs:
           .env
 
       - name: Generate Kibana encryption keys
-        run: |
+        run: docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml
-          docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml
-          sed -i '/Kibana is currently running with legacy OpenSSL providers enabled/d' kibana/config/kibana.yml
 
       - name:  Set up users and roles
         run: docker compose up setup
@@ -123,33 +120,6 @@ jobs:
           docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat
           .github/workflows/scripts/run-tests-heartbeat.sh
 
-      #
-      # Enterprise Search
-      #
-
-      - name: Execute Enterprise Search test suite
-        run: |
-
-          # Set mandatory Elasticsearch settings
-
-          sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml
-
-          # Restart Elasticsearch for changes to take effect
-
-          docker compose restart elasticsearch
-
-          # Run Enterprise Search and execute tests
-
-          sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
-
-          docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search
-          .github/workflows/scripts/run-tests-enterprise-search.sh
-
-          # Revert changes to Elasticsearch configuration
-
-          sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
-          docker compose restart elasticsearch
-
       - name: Collect troubleshooting data
         id: debug-data
         if: failure()
@@ -163,7 +133,6 @@ jobs:
           -f extensions/metricbeat/metricbeat-compose.yml \
           -f extensions/filebeat/filebeat-compose.yml \
           -f extensions/heartbeat/heartbeat-compose.yml \
-          -f extensions/enterprise-search/enterprise-search-compose.yml \
           ps >"$debug_data_dir"/docker_ps.log
 
           docker compose \
@@ -173,7 +142,6 @@ jobs:
           -f extensions/metricbeat/metricbeat-compose.yml \
           -f extensions/filebeat/filebeat-compose.yml \
           -f extensions/heartbeat/heartbeat-compose.yml \
-          -f extensions/enterprise-search/enterprise-search-compose.yml \
           logs >"$debug_data_dir"/docker_logs.log
 
           echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT"
@@ -201,5 +169,4 @@ jobs:
           -f extensions/metricbeat/metricbeat-compose.yml
           -f extensions/filebeat/filebeat-compose.yml
           -f extensions/heartbeat/heartbeat-compose.yml
-          -f extensions/enterprise-search/enterprise-search-compose.yml
           down -v

.github/workflows/scripts/run-tests-enterprise-search.sh

@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-set -o pipefail
-
-
-source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
-
-
-cid_es="$(container_id elasticsearch)"
-cid_en="$(container_id enterprise-search)"
-
-ip_es="$(service_ip elasticsearch)"
-ip_en="$(service_ip enterprise-search)"
-
-grouplog 'Wait for readiness of Elasticsearch'
-poll_ready "$cid_es" 'http://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" -u 'elastic:testpasswd'
-endgroup
-
-grouplog 'Wait for readiness of Enterprise Search'
-poll_ready "$cid_en" 'http://enterprise-search:3002/api/ent/v1/internal/health' --resolve "enterprise-search:3002:${ip_en}" -u 'elastic:testpasswd'
-endgroup
-
-log 'Ensuring that App Search API keys were created in Elasticsearch'
-
-query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
-{
-  "query": {
-    "terms": {
-      "name": [ "search-key", "private-key" ]
-    }
-  }
-}
-EOD
-)
-
-declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
-	'http://elasticsearch:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?pretty'
-	'--resolve' "elasticsearch:9200:${ip_es}"
-	'-H' 'Content-Type: application/json'
-	'-d' "${query}"
-)
-
-echo "curl arguments: ${search_args[*]}"
-
-response="$(curl "${search_args[@]}")"
-echo "$response"
-declare -i count
-count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
-if (( count != 2)); then
-	echo "Expected search and private keys, got ${count} result(s)"
-	exit 1
-fi

.github/workflows/update-merge.yml

@@ -9,7 +9,7 @@ on:
       - update/main
       - update/tls
       - update/release-7.x
-      - update/release-9.x
+      - update/release-8.x
 
 jobs:
 

.github/workflows/update.yml

@@ -15,7 +15,7 @@ jobs:
           - main
           - tls
           - release-7.x
-          - release-9.x
+          - release-8.x
 
     steps:
       - uses: actions/checkout@v4

README.md

@@ -1,7 +1,7 @@
 # Elastic stack (ELK) on Docker
 
-[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-8.17.4-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases)
+[![Elastic Stack version](https://img.shields.io/badge/Elastic%20Stack-9.0.0-00bfb3?style=flat&logo=elastic-stack)](https://www.elastic.co/blog/category/releases)
-[![Build Status](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml?query=branch%3Amain)
+[![Build Status](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml/badge.svg?branch=release-9.x)](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml?query=branch%3Arelease-9.x)
 [![Join the chat](https://badges.gitter.im/Join%20Chat.svg)](https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im)
 
 Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose.
@@ -268,7 +268,7 @@ docker compose down -v
 ### Version selection
 
 This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major
-version (8.x).
+version (9.x).
 
 To use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env)
 file. If you are upgrading an existing stack, remember to rebuild all container images using the `docker compose build`
@@ -280,6 +280,7 @@ command.
 
 Older major versions are also supported on separate branches:
 
+* [`release-8.x`](https://github.com/deviantony/docker-elk/tree/release-8.x): 8.x series
 * [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series
 * [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life)
 * [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life)

elasticsearch/Dockerfile

@@ -1,7 +1,7 @@
 ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.0}
 
 # Add your elasticsearch plugins setup here
 # Example: RUN elasticsearch-plugin install analysis-icu

extensions/enterprise-search/.dockerignore

@@ -1,6 +0,0 @@
-# Ignore Docker build files
-Dockerfile
-.dockerignore
-
-# Ignore OS artifacts
-**/.DS_Store

extensions/enterprise-search/Dockerfile

@@ -1,4 +0,0 @@
-ARG ELASTIC_VERSION
-
-# https://www.docker.elastic.co/
-FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION:-8.17.4}

extensions/enterprise-search/README.md

@@ -1,144 +0,0 @@
-# Enterprise Search extension
-
-Elastic Enterprise Search is a suite of products for search applications backed by the Elastic Stack.
-
-## Requirements
-
-* 2 GB of free RAM, on top of the resources required by the other stack components and extensions.
-
-The Enterprise Search web application is served on the TCP port `3002`.
-
-## Usage
-
-### Generate an encryption key
-
-Enterprise Search requires one or more [encryption keys][enterprisesearch-encryption] to be configured before the
-initial startup. Failing to do so prevents the server from starting.
-
-Encryption keys can contain any series of characters. Elastic recommends using 256-bit keys for optimal security.
-
-Those encryption keys must be added manually to the [`config/enterprise-search.yml`][config-enterprisesearch] file. By
-default, the list of encryption keys is empty and must be populated using one of the following formats:
-
-```yaml
-secret_management.encryption_keys:
-  - my_first_encryption_key
-  - my_second_encryption_key
-  - ...
-```
-
-```yaml
-secret_management.encryption_keys: [my_first_encryption_key, my_second_encryption_key, ...]
-```
-
-> [!NOTE]
-> To generate a strong random encryption key, you can use the OpenSSL utility or any other online/offline tool of your
-> choice:
->
-> ```console
-> $ openssl rand -hex 32
-> 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546
-> ```
-
-### Enable Elasticsearch's API key service
-
-Enterprise Search requires Elasticsearch's built-in [API key service][es-security] to be enabled in order to start.
-Unless Elasticsearch is configured to enable TLS on the HTTP interface (disabled by default), this service is disabled
-by default.
-
-To enable it, modify the Elasticsearch configuration file in [`elasticsearch/config/elasticsearch.yml`][config-es] and
-add the following setting:
-
-```yaml
-xpack.security.authc.api_key.enabled: true
-```
-
-### Configure the Enterprise Search host in Kibana
-
-Kibana acts as the [management interface][enterprisesearch-kb] to Enterprise Search.
-
-To enable the management experience for Enterprise Search, modify the Kibana configuration file in
-[`kibana/config/kibana.yml`][config-kbn] and add the following setting:
-
-```yaml
-enterpriseSearch.host: http://enterprise-search:3002
-```
-
-### Start the server
-
-To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command
-line argument referencing the `enterprise-search-compose.yml` file:
-
-```console
-$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up
-```
-
-Allow a few minutes for the stack to start, then open your web browser at the address <http://localhost:3002> to see the
-Enterprise Search home page.
-
-Enterprise Search is configured on first boot with the following default credentials:
-
-* user: *enterprise_search*
-* password: *changeme*
-
-## Security
-
-The Enterprise Search password is defined inside the Compose file via the `ENT_SEARCH_DEFAULT_PASSWORD` environment
-variable. We highly recommend choosing a more secure password than the default one for security reasons.
-
-To do so, change the value `ENT_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first
-boot**:
-
-```yaml
-enterprise-search:
-
-  environment:
-    ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}}
-```
-
-> [!WARNING]
-> The default Enterprise Search password can only be set during the initial boot. Once the password is persisted in
-> Elasticsearch, it can only be changed via the Elasticsearch API.
-
-For more information, please refer to [User Management and Security][enterprisesearch-security].
-
-## Configuring Enterprise Search
-
-The Enterprise Search configuration is stored in [`config/enterprise-search.yml`][config-enterprisesearch]. You can
-modify this file using the [Default Enterprise Search configuration][enterprisesearch-config] as a reference.
-
-You can also specify the options you want to override by setting environment variables inside the Compose file:
-
-```yaml
-enterprise-search:
-
-  environment:
-    ent_search.auth.source: standard
-    worker.threads: '6'
-```
-
-Any change to the Enterprise Search configuration requires a restart of the Enterprise Search container:
-
-```console
-$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml restart enterprise-search
-```
-
-Please refer to the following documentation page for more details about how to configure Enterprise Search inside a
-Docker container: [Running Enterprise Search Using Docker][enterprisesearch-docker].
-
-## See also
-
-[Enterprise Search documentation][enterprisesearch-docs]
-
-[config-enterprisesearch]: ./config/enterprise-search.yml
-
-[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/current/encryption-keys.html
-[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/current/workplace-search-security.html
-[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
-[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html
-[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html
-[enterprisesearch-kb]: https://www.elastic.co/guide/en/kibana/current/enterprise-search-settings-kb.html
-
-[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings
-[config-es]: ../../elasticsearch/config/elasticsearch.yml
-[config-kbn]: ../../kibana/config/kibana.yml

extensions/enterprise-search/config/enterprise-search.yml

@@ -1,28 +0,0 @@
----
-## Enterprise Search core configuration
-## https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
-#
-
-## --------------------- REQUIRED ---------------------
-
-# Encryption keys to protect application secrets.
-secret_management.encryption_keys:
-  # example:
-  #- 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546
-
-## ----------------------------------------------------
-
-# IP address Enterprise Search listens on
-ent_search.listen_host: 0.0.0.0
-
-# URL at which users reach Enterprise Search / Kibana
-ent_search.external_url: http://localhost:3002
-kibana.host: http://localhost:5601
-
-# Elasticsearch URL and credentials
-elasticsearch.host: http://elasticsearch:9200
-elasticsearch.username: elastic
-elasticsearch.password: ${ELASTIC_PASSWORD}
-
-# Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes.
-allow_es_settings_modification: true

extensions/enterprise-search/enterprise-search-compose.yml

@@ -1,18 +0,0 @@
-services:
-  enterprise-search:
-    build:
-      context: extensions/enterprise-search/
-      args:
-        ELASTIC_VERSION: ${ELASTIC_VERSION}
-    volumes:
-      - ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z
-    environment:
-      JAVA_OPTS: -Xms2g -Xmx2g
-      ENT_SEARCH_DEFAULT_PASSWORD: 'changeme'
-      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
-    ports:
-      - 3002:3002
-    networks:
-      - elk
-    depends_on:
-      - elasticsearch

extensions/filebeat/Dockerfile

@@ -1,3 +1,3 @@
 ARG ELASTIC_VERSION
 
-FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-9.0.0}

extensions/fleet/Dockerfile

@@ -1,6 +1,6 @@
 ARG ELASTIC_VERSION
 
-FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-9.0.0}
 
 # Ensure the 'state' directory exists and is owned by the 'elastic-agent' user,
 # otherwise mounting a named volume in that location creates a directory owned

extensions/heartbeat/Dockerfile

@@ -1,3 +1,3 @@
 ARG ELASTIC_VERSION
 
-FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-9.0.0}

extensions/metricbeat/Dockerfile

@@ -1,3 +1,3 @@
 ARG ELASTIC_VERSION
 
-FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-9.0.0}

kibana/Dockerfile

@@ -1,7 +1,7 @@
 ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-9.0.0}
 
 # Add your kibana plugins setup here
 # Example: RUN kibana-plugin install <name|url>

kibana/config/kibana.yml

@@ -17,7 +17,7 @@ elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
 ## Encryption keys (optional but highly recommended)
 ##
 ## Generate with either
-##  $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate
+##  $ docker container run --rm docker.elastic.co/kibana/kibana:9.0.0 bin/kibana-encryption-keys generate
 ##  $ openssl rand -hex 32
 ##
 ## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html

logstash/Dockerfile

@@ -1,7 +1,7 @@
 ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-9.0.0}
 
 # Add your logstash plugins setup here
 # Example: RUN logstash-plugin install logstash-filter-json

logstash/config/logstash.yml

@@ -2,6 +2,6 @@
 ## Default Logstash configuration from Logstash base image.
 ## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml
 #
-http.host: 0.0.0.0
+api.http.host: 0.0.0.0
 
 node.name: logstash

setup/Dockerfile

@@ -1,6 +1,6 @@
 ARG ELASTIC_VERSION
 
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.17.4}
+FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.0}
 
 ENTRYPOINT ["/entrypoint.sh"]