Mirrors/docker-elk
1
0
Fork 0
mirror of https://github.com/deviantony/docker-elk.git synced 2026-01-06 04:53:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

ci: Switch from plain Lucene to Query DSL

Browse Source
This commit is contained in:
Antoine Cotten
2024-10-07 12:23:21 +02:00
parent 58e9bc012f
commit 39bf7d6ee6
6 changed files with 243 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/scripts/run-tests-core.sh vendored
View File

@@ -46,14 +46,17 @@ if ((was_retried)); then
echo >&2
fi
# It might take a few seconds before the indices and alias are created, so we
# need to be resilient here.
was_retried=0
declare -a refresh_args=( '-X' 'POST' '-s' '-w' '%{http_code}' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/logs-generic-default/_refresh'
'--resolve' "elasticsearch:9200:${ip_es}"
)
echo "curl arguments: ${refresh_args[*]}"
# It might take a few seconds before the indices and alias are created, so we
# need to be resilient here.
was_retried=0
# retry for max 10s (10*1s)
for _ in $(seq 1 10); do
output="$(curl "${refresh_args[@]}")"
@@ -72,16 +75,32 @@ fi
log 'Searching message in Elasticsearch'
# We don't know how much time it will take Logstash to create our document, so
# we need to be resilient here too.
was_retried=0
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"term": {
"message": "dockerelk"
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/logs-generic-default/_search?q=message:dockerelk&pretty'
'http://elasticsearch:9200/logs-generic-default/_search?pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
declare -i count
declare response
echo "curl arguments: ${search_args[*]}"
# We don't know how much time it will take Logstash to create our document, so
# we need to be resilient here too.
was_retried=0
# retry for max 10s (10*1s)
for _ in $(seq 1 10); do
response="$(curl "${search_args[@]}")"

23
.github/workflows/scripts/run-tests-enterprise-search.sh vendored
View File

@@ -22,7 +22,28 @@ poll_ready "$cid_en" 'http://enterprise-search:3002/api/ent/v1/internal/health'
endgroup
log 'Ensuring that App Search API keys were created in Elasticsearch'
response="$(curl 'http://elasticsearch:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?q=*:*&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"terms": {
"name": [ "search-key", "private-key" ]
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/.ent-search-actastic-app_search_api_tokens_v3/_search?pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
echo "curl arguments: ${search_args[*]}"
response="$(curl "${search_args[@]}")"
echo "$response"
declare -i count
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

44
.github/workflows/scripts/run-tests-filebeat.sh vendored
View File

@@ -21,23 +21,51 @@ grouplog 'Wait for readiness of Filebeat'
poll_ready "$cid_fb" 'http://filebeat:5066/?pretty' --resolve "filebeat:5066:${ip_fb}"
endgroup
# We expect to find log entries for the 'elasticsearch' Compose service using
# the following query:
#
# agent.type:"filebeat"
# AND input.type:"container"
# AND container.name:"docker-elk-elasticsearch-1"
#
log 'Searching documents generated by Filebeat'
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"bool": {
"must": [
{
"term": {
"agent.type": "filebeat"
}
},
{
"term": {
"input.type": "container"
}
},
{
"term": {
"container.name": "docker-elk-elasticsearch-1"
}
}
]
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/filebeat-*/_search?size=1&pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
declare response
declare -i count
echo "curl arguments: ${search_args[*]}"
declare -i was_retried=0
# retry for max 60s (30*2s)
for _ in $(seq 1 30); do
response="$(curl 'http://elasticsearch:9200/filebeat-*/_search?q=agent.type:%22filebeat%22%20AND%20input.type:%22container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
response="$(curl "${search_args[@]}")"
set +u # prevent "unbound variable" if assigned value is not an integer
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

99
.github/workflows/scripts/run-tests-fleet.sh vendored
View File

@@ -27,24 +27,61 @@ grouplog 'Wait for readiness of APM Server'
poll_ready "$cid_apm" 'http://apm-server:8200/' --resolve "apm-server:8200:${ip_apm}"
endgroup
# We expect to find metrics entries using the following query:
#
# agent.name:"fleet-server"
# AND agent.type:"metricbeat"
# AND event.module:"system"
# AND event.dataset:"system.cpu"
# AND metricset.name:"cpu"
#
log 'Searching a system document generated by Fleet Server'
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"bool": {
"must": [
{
"term": {
"agent.name": "fleet-server"
}
},
{
"term": {
"agent.type": "metricbeat"
}
},
{
"term": {
"event.module": "system"
}
},
{
"term": {
"event.dataset": "system.cpu"
}
},
{
"term": {
"metricset.name": "cpu"
}
}
]
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/metrics-system.cpu-default/_search?size=1&pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
declare response
declare -i count
echo "curl arguments: ${search_args[*]}"
declare -i was_retried=0
# retry for max 60s (30*2s)
for _ in $(seq 1 30); do
response="$(curl 'http://elasticsearch:9200/metrics-system.cpu-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22metricbeat%22%20AND%20event.module:%22system%22%20AND%20event.dataset:%22system.cpu%22%20AND%20metricset.name:%22cpu%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
response="$(curl "${search_args[@]}")"
set +u # prevent "unbound variable" if assigned value is not an integer
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
@@ -71,23 +108,51 @@ if (( count == 0 )); then
exit 1
fi
# We expect to find log entries for the 'elasticsearch' Compose service using
# the following query:
#
# agent.name:"fleet-server"
# AND agent.type:"filebeat"
# AND container.name:"docker-elk-elasticsearch-1"
#
log 'Searching a container document generated by Fleet Server'
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"bool": {
"must": [
{
"term": {
"agent.name": "fleet-server"
}
},
{
"term": {
"agent.type": "filebeat"
}
},
{
"term": {
"container.name": "docker-elk-elasticsearch-1"
}
}
]
}
}
}
EOD
)
search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/logs-docker.container_logs-default/_search?size=1&pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
response=
count=0
echo "curl arguments: ${search_args[*]}"
was_retried=0
# retry for max 60s (30*2s)
for _ in $(seq 1 30); do
response="$(curl 'http://elasticsearch:9200/logs-docker.container_logs-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22filebeat%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
response="$(curl "${search_args[@]}")"
set +u # prevent "unbound variable" if assigned value is not an integer
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

44
.github/workflows/scripts/run-tests-heartbeat.sh vendored
View File

@@ -21,23 +21,51 @@ grouplog 'Wait for readiness of Heartbeat'
poll_ready "$cid_hb" 'http://heartbeat:5066/?pretty' --resolve "heartbeat:5066:${ip_hb}"
endgroup
# We expect to find heartbeat entries for the 'elasticsearch' HTTP service
# using the following query:
#
# agent.type:"heartbeat"
# AND monitor.type:"http"
# AND url.domain:"elasticsearch"
#
log 'Searching a document generated by Heartbeat'
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"bool": {
"must": [
{
"term": {
"agent.type": "heartbeat"
}
},
{
"term": {
"monitor.type": "http"
}
},
{
"term": {
"url.domain": "elasticsearch"
}
}
]
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/heartbeat-*/_search?size=1&pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
declare response
declare -i count
echo "curl arguments: ${search_args[*]}"
declare -i was_retried=0
# retry for max 60s (30*2s)
for _ in $(seq 1 30); do
response="$(curl 'http://elasticsearch:9200/heartbeat-*/_search?q=agent.type:%22heartbeat%22%20AND%20monitor.type:%22http%22%20AND%20url.domain:%22elasticsearch%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
response="$(curl "${search_args[@]}")"
set +u # prevent "unbound variable" if assigned value is not an integer
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"

50
.github/workflows/scripts/run-tests-metricbeat.sh vendored
View File

@@ -21,24 +21,56 @@ grouplog 'Wait for readiness of Metricbeat'
poll_ready "$cid_mb" 'http://metricbeat:5066/?pretty' --resolve "metricbeat:5066:${ip_mb}"
endgroup
# We expect to find monitoring entries for the 'elasticsearch' Compose service
# using the following query:
#
# agent.type:"metricbeat"
# AND event.module:"docker"
# AND event.dataset:"docker.container"
# AND container.name:"docker-elk-elasticsearch-1"
#
log 'Searching a document generated by Metricbeat'
query=$( (IFS= read -r -d '' data || echo "$data" | jq -c) <<EOD
{
"query": {
"bool": {
"must": [
{
"term": {
"agent.type": "metricbeat"
}
},
{
"term": {
"event.module": "docker"
}
},
{
"term": {
"event.dataset": "docker.container"
}
},
{
"term": {
"container.name": "docker-elk-elasticsearch-1"
}
}
]
}
}
}
EOD
)
declare -a search_args=( '-s' '-u' 'elastic:testpasswd'
'http://elasticsearch:9200/metricbeat-*/_search?size=1&pretty'
'--resolve' "elasticsearch:9200:${ip_es}"
'-H' 'Content-Type: application/json'
'-d' "${query}"
)
declare response
declare -i count
echo "curl arguments: ${search_args[*]}"
declare -i was_retried=0
# retry for max 60s (30*2s)
for _ in $(seq 1 30); do
response="$(curl 'http://elasticsearch:9200/metricbeat-*/_search?q=agent.type:%22metricbeat%22%20AND%20event.module:%22docker%22%20AND%20event.dataset:%22docker.container%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&size=1&pretty' -s --resolve "elasticsearch:9200:${ip_es}" -u elastic:testpasswd)"
response="$(curl "${search_args[@]}")"
set +u # prevent "unbound variable" if assigned value is not an integer
count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"