mirror of
https://github.com/deviantony/docker-elk.git
synced 2025-12-11 17:30:11 +01:00
Update to v9.0.0
List of changes impacting docker-elk: - [kibana]: Several features now require an encryption key for saved objects. Such key must be set via the configuration option `xpack.encryptedSavedObjects.encryptionKey`. Ref: https://www.elastic.co/guide/en/kibana/8.18/xpack-security-secure-saved-objects.html - [enterprise-search]: Discontinued in 9.x versions. Refs: - https://www.elastic.co/guide/en/enterprise-search/8.18/upgrading-to-9-x.html - https://www.elastic.co/blog/app-search-to-elasticsearch
This commit is contained in:
Antoine Cotten
2
.env
2
.env
@@ -1,4 +1,4 @@
|
||||
ELASTIC_VERSION=8.17.4
|
||||
ELASTIC_VERSION=9.0.0
|
||||
|
||||
## Passwords for stack users
|
||||
#
|
||||
|
||||
35
.github/workflows/ci.yml
vendored
35
.github/workflows/ci.yml
vendored
@@ -35,7 +35,6 @@ jobs:
|
||||
-f extensions/metricbeat/metricbeat-compose.yml
|
||||
-f extensions/filebeat/filebeat-compose.yml
|
||||
-f extensions/heartbeat/heartbeat-compose.yml
|
||||
-f extensions/enterprise-search/enterprise-search-compose.yml
|
||||
build
|
||||
|
||||
########################################################
|
||||
@@ -58,9 +57,7 @@ jobs:
|
||||
.env
|
||||
|
||||
- name: Generate Kibana encryption keys
|
||||
run: |
|
||||
docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml
|
||||
sed -i '/Kibana is currently running with legacy OpenSSL providers enabled/d' kibana/config/kibana.yml
|
||||
run: docker container run --rm docker-elk-kibana bin/kibana-encryption-keys generate -q >>kibana/config/kibana.yml
|
||||
|
||||
- name: Set up users and roles
|
||||
run: docker compose up setup
|
||||
@@ -123,33 +120,6 @@ jobs:
|
||||
docker compose -f docker-compose.yml -f extensions/heartbeat/heartbeat-compose.yml up --remove-orphans -d heartbeat
|
||||
.github/workflows/scripts/run-tests-heartbeat.sh
|
||||
|
||||
#
|
||||
# Enterprise Search
|
||||
#
|
||||
|
||||
- name: Execute Enterprise Search test suite
|
||||
run: |
|
||||
|
||||
# Set mandatory Elasticsearch settings
|
||||
|
||||
sed -i '$ a xpack.security.authc.api_key.enabled: true' elasticsearch/config/elasticsearch.yml
|
||||
|
||||
# Restart Elasticsearch for changes to take effect
|
||||
|
||||
docker compose restart elasticsearch
|
||||
|
||||
# Run Enterprise Search and execute tests
|
||||
|
||||
sed -i 's/\(secret_management.encryption_keys:\)/\1 [test-encrypt]/g' extensions/enterprise-search/config/enterprise-search.yml
|
||||
|
||||
docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up --remove-orphans -d enterprise-search
|
||||
.github/workflows/scripts/run-tests-enterprise-search.sh
|
||||
|
||||
# Revert changes to Elasticsearch configuration
|
||||
|
||||
sed -i '/xpack.security.authc.api_key.enabled: true/d' elasticsearch/config/elasticsearch.yml
|
||||
docker compose restart elasticsearch
|
||||
|
||||
- name: Collect troubleshooting data
|
||||
id: debug-data
|
||||
if: failure()
|
||||
@@ -163,7 +133,6 @@ jobs:
|
||||
-f extensions/metricbeat/metricbeat-compose.yml \
|
||||
-f extensions/filebeat/filebeat-compose.yml \
|
||||
-f extensions/heartbeat/heartbeat-compose.yml \
|
||||
-f extensions/enterprise-search/enterprise-search-compose.yml \
|
||||
ps >"$debug_data_dir"/docker_ps.log
|
||||
|
||||
docker compose \
|
||||
@@ -173,7 +142,6 @@ jobs:
|
||||
-f extensions/metricbeat/metricbeat-compose.yml \
|
||||
-f extensions/filebeat/filebeat-compose.yml \
|
||||
-f extensions/heartbeat/heartbeat-compose.yml \
|
||||
-f extensions/enterprise-search/enterprise-search-compose.yml \
|
||||
logs >"$debug_data_dir"/docker_logs.log
|
||||
|
||||
echo "path=${debug_data_dir}" >>"$GITHUB_OUTPUT"
|
||||
@@ -201,5 +169,4 @@ jobs:
|
||||
-f extensions/metricbeat/metricbeat-compose.yml
|
||||
-f extensions/filebeat/filebeat-compose.yml
|
||||
-f extensions/heartbeat/heartbeat-compose.yml
|
||||
-f extensions/enterprise-search/enterprise-search-compose.yml
|
||||
down -v
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
|
||||
source "${BASH_SOURCE[0]%/*}"/lib/testing.sh
|
||||
|
||||
|
||||
cid_es="$(container_id elasticsearch)"
|
||||
cid_en="$(container_id enterprise-search)"
|
||||
|
||||
ip_es="$(service_ip elasticsearch)"
|
||||
ip_en="$(service_ip enterprise-search)"
|
||||
|
||||
grouplog 'Wait for readiness of Elasticsearch'
|
||||
poll_ready "$cid_es" 'http://elasticsearch:9200/' --resolve "elasticsearch:9200:${ip_es}" -u 'elastic:testpasswd'
|
||||
endgroup
|
||||
|
||||
grouplog 'Wait for readiness of Enterprise Search'
|
||||
poll_ready "$cid_en" 'http://enterprise-search:3002/api/ent/v1/internal/health' --resolve "enterprise-search:3002:${ip_en}" -u 'elastic:testpasswd'
|
||||
endgroup
|
||||
|
||||
log 'Ensuring that App Search API keys were created in Elasticsearch'
|
||||
|
||||
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
|
||||
{
|
||||
"query": {
|
||||
"terms": {
|
||||
"name": [ "search-key", "private-key" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
EOD
|
||||
)
|
||||
|
||||
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
|
||||
'http://elasticsearch:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?pretty'
|
||||
'--resolve' "elasticsearch:9200:${ip_es}"
|
||||
'-H' 'Content-Type: application/json'
|
||||
'-d' "${query}"
|
||||
)
|
||||
|
||||
echo "curl arguments: ${search_args[*]}"
|
||||
|
||||
response="$(curl "${search_args[@]}")"
|
||||
echo "$response"
|
||||
declare -i count
|
||||
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
|
||||
if (( count != 2)); then
|
||||
echo "Expected search and private keys, got ${count} result(s)"
|
||||
exit 1
|
||||
fi
|
||||
2
.github/workflows/update-merge.yml
vendored
2
.github/workflows/update-merge.yml
vendored
@@ -9,7 +9,7 @@ on:
|
||||
- update/main
|
||||
- update/tls
|
||||
- update/release-7.x
|
||||
- update/release-9.x
|
||||
- update/release-8.x
|
||||
|
||||
jobs:
|
||||
|
||||
|
||||
2
.github/workflows/update.yml
vendored
2
.github/workflows/update.yml
vendored
@@ -15,7 +15,7 @@ jobs:
|
||||
- main
|
||||
- tls
|
||||
- release-7.x
|
||||
- release-9.x
|
||||
- release-8.x
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Elastic stack (ELK) on Docker
|
||||
|
||||
[](https://www.elastic.co/blog/category/releases)
|
||||
[](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml?query=branch%3Amain)
|
||||
[](https://www.elastic.co/blog/category/releases)
|
||||
[](https://github.com/deviantony/docker-elk/actions/workflows/ci.yml?query=branch%3Arelease-9.x)
|
||||
[](https://app.gitter.im/#/room/#deviantony_docker-elk:gitter.im)
|
||||
|
||||
Run the latest version of the [Elastic stack][elk-stack] with Docker and Docker Compose.
|
||||
@@ -268,7 +268,7 @@ docker compose down -v
|
||||
### Version selection
|
||||
|
||||
This repository stays aligned with the latest version of the Elastic stack. The `main` branch tracks the current major
|
||||
version (8.x).
|
||||
version (9.x).
|
||||
|
||||
To use a different version of the core Elastic components, simply change the version number inside the [`.env`](.env)
|
||||
file. If you are upgrading an existing stack, remember to rebuild all container images using the `docker compose build`
|
||||
@@ -280,6 +280,7 @@ command.
|
||||
|
||||
Older major versions are also supported on separate branches:
|
||||
|
||||
* [`release-8.x`](https://github.com/deviantony/docker-elk/tree/release-8.x): 8.x series
|
||||
* [`release-7.x`](https://github.com/deviantony/docker-elk/tree/release-7.x): 7.x series
|
||||
* [`release-6.x`](https://github.com/deviantony/docker-elk/tree/release-6.x): 6.x series (End-of-life)
|
||||
* [`release-5.x`](https://github.com/deviantony/docker-elk/tree/release-5.x): 5.x series (End-of-life)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
# Add your elasticsearch plugins setup here
|
||||
# Example: RUN elasticsearch-plugin install analysis-icu
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
# Ignore Docker build files
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
|
||||
# Ignore OS artifacts
|
||||
**/.DS_Store
|
||||
@@ -1,4 +0,0 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/enterprise-search/enterprise-search:${ELASTIC_VERSION:-8.17.4}
|
||||
@@ -1,144 +0,0 @@
|
||||
# Enterprise Search extension
|
||||
|
||||
Elastic Enterprise Search is a suite of products for search applications backed by the Elastic Stack.
|
||||
|
||||
## Requirements
|
||||
|
||||
* 2 GB of free RAM, on top of the resources required by the other stack components and extensions.
|
||||
|
||||
The Enterprise Search web application is served on the TCP port `3002`.
|
||||
|
||||
## Usage
|
||||
|
||||
### Generate an encryption key
|
||||
|
||||
Enterprise Search requires one or more [encryption keys][enterprisesearch-encryption] to be configured before the
|
||||
initial startup. Failing to do so prevents the server from starting.
|
||||
|
||||
Encryption keys can contain any series of characters. Elastic recommends using 256-bit keys for optimal security.
|
||||
|
||||
Those encryption keys must be added manually to the [`config/enterprise-search.yml`][config-enterprisesearch] file. By
|
||||
default, the list of encryption keys is empty and must be populated using one of the following formats:
|
||||
|
||||
```yaml
|
||||
secret_management.encryption_keys:
|
||||
- my_first_encryption_key
|
||||
- my_second_encryption_key
|
||||
- ...
|
||||
```
|
||||
|
||||
```yaml
|
||||
secret_management.encryption_keys: [my_first_encryption_key, my_second_encryption_key, ...]
|
||||
```
|
||||
|
||||
> [!NOTE]
|
||||
> To generate a strong random encryption key, you can use the OpenSSL utility or any other online/offline tool of your
|
||||
> choice:
|
||||
>
|
||||
> ```console
|
||||
> $ openssl rand -hex 32
|
||||
> 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546
|
||||
> ```
|
||||
|
||||
### Enable Elasticsearch's API key service
|
||||
|
||||
Enterprise Search requires Elasticsearch's built-in [API key service][es-security] to be enabled in order to start.
|
||||
Unless Elasticsearch is configured to enable TLS on the HTTP interface (disabled by default), this service is disabled
|
||||
by default.
|
||||
|
||||
To enable it, modify the Elasticsearch configuration file in [`elasticsearch/config/elasticsearch.yml`][config-es] and
|
||||
add the following setting:
|
||||
|
||||
```yaml
|
||||
xpack.security.authc.api_key.enabled: true
|
||||
```
|
||||
|
||||
### Configure the Enterprise Search host in Kibana
|
||||
|
||||
Kibana acts as the [management interface][enterprisesearch-kb] to Enterprise Search.
|
||||
|
||||
To enable the management experience for Enterprise Search, modify the Kibana configuration file in
|
||||
[`kibana/config/kibana.yml`][config-kbn] and add the following setting:
|
||||
|
||||
```yaml
|
||||
enterpriseSearch.host: http://enterprise-search:3002
|
||||
```
|
||||
|
||||
### Start the server
|
||||
|
||||
To include Enterprise Search in the stack, run Docker Compose from the root of the repository with an additional command
|
||||
line argument referencing the `enterprise-search-compose.yml` file:
|
||||
|
||||
```console
|
||||
$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml up
|
||||
```
|
||||
|
||||
Allow a few minutes for the stack to start, then open your web browser at the address <http://localhost:3002> to see the
|
||||
Enterprise Search home page.
|
||||
|
||||
Enterprise Search is configured on first boot with the following default credentials:
|
||||
|
||||
* user: *enterprise_search*
|
||||
* password: *changeme*
|
||||
|
||||
## Security
|
||||
|
||||
The Enterprise Search password is defined inside the Compose file via the `ENT_SEARCH_DEFAULT_PASSWORD` environment
|
||||
variable. We highly recommend choosing a more secure password than the default one for security reasons.
|
||||
|
||||
To do so, change the value `ENT_SEARCH_DEFAULT_PASSWORD` environment variable inside the Compose file **before the first
|
||||
boot**:
|
||||
|
||||
```yaml
|
||||
enterprise-search:
|
||||
|
||||
environment:
|
||||
ENT_SEARCH_DEFAULT_PASSWORD: {{some strong password}}
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> The default Enterprise Search password can only be set during the initial boot. Once the password is persisted in
|
||||
> Elasticsearch, it can only be changed via the Elasticsearch API.
|
||||
|
||||
For more information, please refer to [User Management and Security][enterprisesearch-security].
|
||||
|
||||
## Configuring Enterprise Search
|
||||
|
||||
The Enterprise Search configuration is stored in [`config/enterprise-search.yml`][config-enterprisesearch]. You can
|
||||
modify this file using the [Default Enterprise Search configuration][enterprisesearch-config] as a reference.
|
||||
|
||||
You can also specify the options you want to override by setting environment variables inside the Compose file:
|
||||
|
||||
```yaml
|
||||
enterprise-search:
|
||||
|
||||
environment:
|
||||
ent_search.auth.source: standard
|
||||
worker.threads: '6'
|
||||
```
|
||||
|
||||
Any change to the Enterprise Search configuration requires a restart of the Enterprise Search container:
|
||||
|
||||
```console
|
||||
$ docker compose -f docker-compose.yml -f extensions/enterprise-search/enterprise-search-compose.yml restart enterprise-search
|
||||
```
|
||||
|
||||
Please refer to the following documentation page for more details about how to configure Enterprise Search inside a
|
||||
Docker container: [Running Enterprise Search Using Docker][enterprisesearch-docker].
|
||||
|
||||
## See also
|
||||
|
||||
[Enterprise Search documentation][enterprisesearch-docs]
|
||||
|
||||
[config-enterprisesearch]: ./config/enterprise-search.yml
|
||||
|
||||
[enterprisesearch-encryption]: https://www.elastic.co/guide/en/enterprise-search/current/encryption-keys.html
|
||||
[enterprisesearch-security]: https://www.elastic.co/guide/en/workplace-search/current/workplace-search-security.html
|
||||
[enterprisesearch-config]: https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
|
||||
[enterprisesearch-docker]: https://www.elastic.co/guide/en/enterprise-search/current/docker.html
|
||||
[enterprisesearch-docs]: https://www.elastic.co/guide/en/enterprise-search/current/index.html
|
||||
[enterprisesearch-kb]: https://www.elastic.co/guide/en/kibana/current/enterprise-search-settings-kb.html
|
||||
|
||||
[es-security]: https://www.elastic.co/guide/en/elasticsearch/reference/current/security-settings.html#api-key-service-settings
|
||||
[config-es]: ../../elasticsearch/config/elasticsearch.yml
|
||||
[config-kbn]: ../../kibana/config/kibana.yml
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
## Enterprise Search core configuration
|
||||
## https://www.elastic.co/guide/en/enterprise-search/current/configuration.html
|
||||
#
|
||||
|
||||
## --------------------- REQUIRED ---------------------
|
||||
|
||||
# Encryption keys to protect application secrets.
|
||||
secret_management.encryption_keys:
|
||||
# example:
|
||||
#- 680f94e568c90364bedf927b2f0f49609702d3eab9098688585a375b14274546
|
||||
|
||||
## ----------------------------------------------------
|
||||
|
||||
# IP address Enterprise Search listens on
|
||||
ent_search.listen_host: 0.0.0.0
|
||||
|
||||
# URL at which users reach Enterprise Search / Kibana
|
||||
ent_search.external_url: http://localhost:3002
|
||||
kibana.host: http://localhost:5601
|
||||
|
||||
# Elasticsearch URL and credentials
|
||||
elasticsearch.host: http://elasticsearch:9200
|
||||
elasticsearch.username: elastic
|
||||
elasticsearch.password: ${ELASTIC_PASSWORD}
|
||||
|
||||
# Allow Enterprise Search to modify Elasticsearch settings. Used to enable auto-creation of Elasticsearch indexes.
|
||||
allow_es_settings_modification: true
|
||||
@@ -1,18 +0,0 @@
|
||||
services:
|
||||
enterprise-search:
|
||||
build:
|
||||
context: extensions/enterprise-search/
|
||||
args:
|
||||
ELASTIC_VERSION: ${ELASTIC_VERSION}
|
||||
volumes:
|
||||
- ./extensions/enterprise-search/config/enterprise-search.yml:/usr/share/enterprise-search/config/enterprise-search.yml:ro,Z
|
||||
environment:
|
||||
JAVA_OPTS: -Xms2g -Xmx2g
|
||||
ENT_SEARCH_DEFAULT_PASSWORD: 'changeme'
|
||||
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
|
||||
ports:
|
||||
- 3002:3002
|
||||
networks:
|
||||
- elk
|
||||
depends_on:
|
||||
- elasticsearch
|
||||
@@ -1,3 +1,3 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/beats/filebeat:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/elastic-agent/elastic-agent:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
# Ensure the 'state' directory exists and is owned by the 'elastic-agent' user,
|
||||
# otherwise mounting a named volume in that location creates a directory owned
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/beats/heartbeat:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/beats/metricbeat:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/kibana/kibana:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
# Add your kibana plugins setup here
|
||||
# Example: RUN kibana-plugin install <name|url>
|
||||
|
||||
@@ -17,7 +17,7 @@ elasticsearch.password: ${KIBANA_SYSTEM_PASSWORD}
|
||||
## Encryption keys (optional but highly recommended)
|
||||
##
|
||||
## Generate with either
|
||||
## $ docker container run --rm docker.elastic.co/kibana/kibana:8.6.2 bin/kibana-encryption-keys generate
|
||||
## $ docker container run --rm docker.elastic.co/kibana/kibana:9.0.0 bin/kibana-encryption-keys generate
|
||||
## $ openssl rand -hex 32
|
||||
##
|
||||
## https://www.elastic.co/guide/en/kibana/current/using-kibana-with-security.html
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/logstash/logstash:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
# Add your logstash plugins setup here
|
||||
# Example: RUN logstash-plugin install logstash-filter-json
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
## Default Logstash configuration from Logstash base image.
|
||||
## https://github.com/elastic/logstash/blob/main/docker/data/logstash/config/logstash-full.yml
|
||||
#
|
||||
http.host: 0.0.0.0
|
||||
api.http.host: 0.0.0.0
|
||||
|
||||
node.name: logstash
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
ARG ELASTIC_VERSION
|
||||
|
||||
# https://www.docker.elastic.co/
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.17.4}
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-9.0.0}
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
Reference in New Issue
Block a user