From 5f3408c67cdee90bb3a12dd38fb2d7579d7cde9c Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 11:58:12 -0600 Subject: [PATCH 01/18] make fleet optional first try --- .../docker-compose.yml | 0 compose/no-fleet-docker-compose.yml | 146 ++++++++++++++++++ elastic-container.sh | 67 +++++--- 3 files changed, 189 insertions(+), 24 deletions(-) rename docker-compose.yml => compose/docker-compose.yml (100%) create mode 100644 compose/no-fleet-docker-compose.yml diff --git a/docker-compose.yml b/compose/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to compose/docker-compose.yml diff --git a/compose/no-fleet-docker-compose.yml b/compose/no-fleet-docker-compose.yml new file mode 100644 index 0000000..684bd90 --- /dev/null +++ b/compose/no-fleet-docker-compose.yml @@ -0,0 +1,146 @@ +services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: ecp-elasticsearch-security-setup + volumes: + - certs:/usr/share/elasticsearch/config/certs:z + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: elasticsearch\n"\ + " dns:\n"\ + " - ecp-elasticsearch\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - ecp-kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + cat config/certs/elasticsearch/elasticsearch.crt config/certs/ca/ca.crt > config/certs/elasticsearch/elasticsearch.chain.pem + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://ecp-elasticsearch:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://ecp-elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/elasticsearch/elasticsearch.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + + elasticsearch: + depends_on: + setup: + condition: service_healthy + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: ecp-elasticsearch + volumes: + - certs:/usr/share/elasticsearch/config/certs + - esdata01:/usr/share/elasticsearch/data + ports: + - ${ES_PORT}:9200 + restart: always + environment: + - node.name=ecp-elasticsearch + - cluster.name=${CLUSTER_NAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/elasticsearch/elasticsearch.key + - xpack.security.http.ssl.certificate=certs/elasticsearch/elasticsearch.chain.pem + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.http.ssl.verification_mode=certificate + - xpack.security.http.ssl.client_authentication=optional + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/elasticsearch/elasticsearch.key + - xpack.security.transport.ssl.certificate=certs/elasticsearch/elasticsearch.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.security.transport.ssl.client_authentication=optional + - xpack.license.self_generated.type=${LICENSE} + mem_limit: ${MEM_LIMIT} + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", + ] + interval: 10s + timeout: 10s + retries: 120 + + kibana: + depends_on: + elasticsearch: + condition: service_healthy + image: docker.elastic.co/kibana/kibana:${STACK_VERSION} + container_name: ecp-kibana + volumes: + - certs:/usr/share/kibana/config/certs:z + - kibanadata:/usr/share/kibana/data + - ./kibana.yml:/usr/share/kibana/config/kibana.yml:Z + ports: + - ${KIBANA_PORT}:5601 + restart: always + environment: + - SERVER_NAME=ecp-kibana + - ELASTICSEARCH_HOSTS=https://ecp-elasticsearch:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + - SERVER_SSL_ENABLED=true + - SERVER_SSL_CERTIFICATE=config/certs/kibana/kibana.crt + - SERVER_SSL_KEY=config/certs/kibana/kibana.key + - SERVER_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt + mem_limit: ${MEM_LIMIT} + healthcheck: + test: + [ + "CMD-SHELL", + "curl -I -s --cacert config/certs/ca/ca.crt https://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", + ] + interval: 10s + timeout: 10s + retries: 120 + +volumes: + certs: + driver: local + esdata01: + driver: local + kibanadata: + driver: local \ No newline at end of file diff --git a/elastic-container.sh b/elastic-container.sh index d0a5ec4..d2572a0 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -45,6 +45,7 @@ usage() { help print this message flags: -v enable verbose output + -n does not enable fleet EOF } @@ -139,27 +140,31 @@ get_host_ip() { } set_fleet_values() { - # Get the current Fleet settings - CURRENT_SETTINGS=$(curl -k -s -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X GET "${LOCAL_KBN_URL}/api/fleet/agents/setup" -H "Content-Type: application/json") + if fleet=1 + # Get the current Fleet settings + CURRENT_SETTINGS=$(curl -k -s -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X GET "${LOCAL_KBN_URL}/api/fleet/agents/setup" -H "Content-Type: application/json") + + # Check if Fleet is already set up + if echo "$CURRENT_SETTINGS" | grep -q '"isInitialized": true'; then + echo "Fleet settings are already configured." + return + fi - # Check if Fleet is already set up - if echo "$CURRENT_SETTINGS" | grep -q '"isInitialized": true'; then - echo "Fleet settings are already configured." - return + echo "Fleet is not initialized, setting up Fleet..." + + fingerprint=$(${COMPOSE} exec -w /usr/share/elasticsearch/config/certs/ca elasticsearch cat ca.crt | openssl x509 -noout -fingerprint -sha256 | cut -d "=" -f 2 | tr -d :) + printf '{"fleet_server_hosts": ["%s"]}' "https://${ipvar}:${FLEET_PORT}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @- | jq + printf '{"hosts": ["%s"]}' "https://${ipvar}:9200" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + printf '{"ca_trusted_fingerprint": "%s"}' "${fingerprint}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') + pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') + printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq + } + else + echo "Fleet is not being used..." fi - echo "Fleet is not initialized, setting up Fleet..." - - fingerprint=$(${COMPOSE} exec -w /usr/share/elasticsearch/config/certs/ca elasticsearch cat ca.crt | openssl x509 -noout -fingerprint -sha256 | cut -d "=" -f 2 | tr -d :) - printf '{"fleet_server_hosts": ["%s"]}' "https://${ipvar}:${FLEET_PORT}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @- | jq - printf '{"hosts": ["%s"]}' "https://${ipvar}:9200" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - printf '{"ca_trusted_fingerprint": "%s"}' "${fingerprint}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') - pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') - printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq -} - clear_documents() { if (($(curl -k --silent "${HEADERS[@]}" --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X DELETE "https://${ipvar}:9200/_data_stream/logs-*" | grep -c "true") > 0)); then printf "Successfully cleared logs data stream" @@ -179,16 +184,23 @@ clear_documents() { OPTIND=1 # Reset in case getopts has been used previously in the shell. verbose=0 +fleet=1 -while getopts "v" opt; do +while getopts ":vn" opt; do case "$opt" in v) verbose=1 + echo "Moar info!!!" + ;; + n) + fleet=0 + echo "What did fleet ever do to you?" ;; *) ;; esac done + shift $((OPTIND - 1)) [ "${1:-}" = "--" ] && shift @@ -210,6 +222,11 @@ else exit 2 fi +if fleet=0 + COMPOSE_STRING = "-f compose/np-fleet-docker-compose.yml" +else + COMPOSE_STRING = "-f compose/docker-compose.yml" +fi case "${ACTION}" in "stage") @@ -225,8 +242,8 @@ case "${ACTION}" in get_host_ip echo "Starting Elastic Stack network and containers." - - ${COMPOSE} up -d --no-deps + + ${COMPOSE} up -d ${COMPOSE_STRING} --no-deps configure_kbn 1>&2 2>&3 @@ -247,17 +264,19 @@ case "${ACTION}" in echo ;; + + "stop") echo "Stopping running containers." - ${COMPOSE} stop + ${COMPOSE} stop ${COMPOSE_STRING} ;; "destroy") echo "#####" echo "Stopping and removing the containers, network, and volumes created." echo "#####" - ${COMPOSE} down -v + ${COMPOSE} down -v ${COMPOSE_STRING} ;; "restart") @@ -268,7 +287,7 @@ case "${ACTION}" in ;; "status") - ${COMPOSE} ps | grep -v setup + ${COMPOSE} ps ${COMPOSE_STRING}| grep -v setup ;; "clear") From 2447dbdb2d0781321300532df976019adb0b1c62 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:01:37 -0600 Subject: [PATCH 02/18] fix curly --- elastic-container.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index d2572a0..392dc81 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -159,8 +159,7 @@ set_fleet_values() { printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') - printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq - } + printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq} else echo "Fleet is not being used..." fi From 58d7b7323f58bae73932f5662c9d81c50d0b9544 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:02:51 -0600 Subject: [PATCH 03/18] put it in the right place --- elastic-container.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastic-container.sh b/elastic-container.sh index 392dc81..7f18d67 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -159,10 +159,12 @@ set_fleet_values() { printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') - printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq} + printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq + else echo "Fleet is not being used..." fi + } clear_documents() { if (($(curl -k --silent "${HEADERS[@]}" --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X DELETE "https://${ipvar}:9200/_data_stream/logs-*" | grep -c "true") > 0)); then From f9a7712c25bb9a70fdbe388063c9a3cfbdc344e5 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:10:25 -0600 Subject: [PATCH 04/18] fix if statement --- elastic-container.sh | 61 ++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index 7f18d67..cbe2d25 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -140,31 +140,26 @@ get_host_ip() { } set_fleet_values() { - if fleet=1 - # Get the current Fleet settings - CURRENT_SETTINGS=$(curl -k -s -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X GET "${LOCAL_KBN_URL}/api/fleet/agents/setup" -H "Content-Type: application/json") - - # Check if Fleet is already set up - if echo "$CURRENT_SETTINGS" | grep -q '"isInitialized": true'; then - echo "Fleet settings are already configured." - return - fi + # Get the current Fleet settings + CURRENT_SETTINGS=$(curl -k -s -u "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X GET "${LOCAL_KBN_URL}/api/fleet/agents/setup" -H "Content-Type: application/json") - echo "Fleet is not initialized, setting up Fleet..." - - fingerprint=$(${COMPOSE} exec -w /usr/share/elasticsearch/config/certs/ca elasticsearch cat ca.crt | openssl x509 -noout -fingerprint -sha256 | cut -d "=" -f 2 | tr -d :) - printf '{"fleet_server_hosts": ["%s"]}' "https://${ipvar}:${FLEET_PORT}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @- | jq - printf '{"hosts": ["%s"]}' "https://${ipvar}:9200" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - printf '{"ca_trusted_fingerprint": "%s"}' "${fingerprint}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq - policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') - pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') - printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq - - else - echo "Fleet is not being used..." + # Check if Fleet is already set up + if echo "$CURRENT_SETTINGS" | grep -q '"isInitialized": true'; then + echo "Fleet settings are already configured." + return fi - } + + echo "Fleet is not initialized, setting up Fleet..." + + fingerprint=$(${COMPOSE} exec -w /usr/share/elasticsearch/config/certs/ca elasticsearch cat ca.crt | openssl x509 -noout -fingerprint -sha256 | cut -d "=" -f 2 | tr -d :) + printf '{"fleet_server_hosts": ["%s"]}' "https://${ipvar}:${FLEET_PORT}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/settings" -d @- | jq + printf '{"hosts": ["%s"]}' "https://${ipvar}:9200" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + printf '{"ca_trusted_fingerprint": "%s"}' "${fingerprint}" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + printf '{"config_yaml": "%s"}' "ssl.verification_mode: certificate" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPUT "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/outputs/fleet-default-output" -d @- | jq + policy_id=$(printf '{"name": "%s", "description": "%s", "namespace": "%s", "monitoring_enabled": ["logs","metrics"], "inactivity_timeout": 1209600}' "Endpoint Policy" "" "default" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/agent_policies?sys_monitoring=true" -d @- | jq -r '.item.id') + pkg_version=$(curl -k --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XGET "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/epm/packages/endpoint" -d : | jq -r '.item.version') + printf "{\"name\": \"%s\", \"description\": \"%s\", \"namespace\": \"%s\", \"policy_id\": \"%s\", \"enabled\": %s, \"inputs\": [{\"enabled\": true, \"streams\": [], \"type\": \"ENDPOINT_INTEGRATION_CONFIG\", \"config\": {\"_config\": {\"value\": {\"type\": \"endpoint\", \"endpointConfig\": {\"preset\": \"EDRComplete\"}}}}}], \"package\": {\"name\": \"endpoint\", \"title\": \"Elastic Defend\", \"version\": \"${pkg_version}\"}}" "Elastic Defend" "" "default" "${policy_id}" "true" | curl -k --silent --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -XPOST "${HEADERS[@]}" "${LOCAL_KBN_URL}/api/fleet/package_policies" -d @- | jq +} clear_documents() { if (($(curl -k --silent "${HEADERS[@]}" --user "${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" -X DELETE "https://${ipvar}:9200/_data_stream/logs-*" | grep -c "true") > 0)); then @@ -223,7 +218,7 @@ else exit 2 fi -if fleet=0 +if fleet=0; then COMPOSE_STRING = "-f compose/np-fleet-docker-compose.yml" else COMPOSE_STRING = "-f compose/docker-compose.yml" @@ -248,14 +243,20 @@ case "${ACTION}" in configure_kbn 1>&2 2>&3 - echo "Waiting 40 seconds for Fleet Server setup." - echo + if fleet=0; then + + echo "Not waiting 40 seconds for Fleet Server setup." + echo - sleep 40 + else + echo "Waiting 40 seconds for Fleet Server setup." + echo - echo "Populating Fleet Settings." - set_fleet_values > /dev/null 2>&1 - echo + sleep 40 + + echo "Populating Fleet Settings." + set_fleet_values > /dev/null 2>&1 + echo echo "READY SET GO!" echo From 5cf88bb8b94200338cbd8c8f98656b99520f4e57 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:11:50 -0600 Subject: [PATCH 05/18] more fixes --- elastic-container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index cbe2d25..186bad2 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -219,9 +219,9 @@ else fi if fleet=0; then - COMPOSE_STRING = "-f compose/np-fleet-docker-compose.yml" + COMPOSE_STRING="-f compose/np-fleet-docker-compose.yml" else - COMPOSE_STRING = "-f compose/docker-compose.yml" + COMPOSE_STRING="-f compose/docker-compose.yml" fi case "${ACTION}" in From 0c06c2fb59f5230f1c9f0d3186f02a371e1d2710 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:12:56 -0600 Subject: [PATCH 06/18] close if statement --- elastic-container.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/elastic-container.sh b/elastic-container.sh index 186bad2..e91c2a5 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -257,6 +257,7 @@ case "${ACTION}" in echo "Populating Fleet Settings." set_fleet_values > /dev/null 2>&1 echo + fi echo "READY SET GO!" echo From 74b3fc4e6c0002b51d5e995c1eebbfbad44d95a8 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:16:57 -0600 Subject: [PATCH 07/18] add file path --- elastic-container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index e91c2a5..e1aec76 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -219,9 +219,9 @@ else fi if fleet=0; then - COMPOSE_STRING="-f compose/np-fleet-docker-compose.yml" + COMPOSE_STRING=" --file compose/no-fleet-docker-compose.yml" else - COMPOSE_STRING="-f compose/docker-compose.yml" + COMPOSE_STRING=" --file compose/docker-compose.yml" fi case "${ACTION}" in From e54c3832d0c14aec8e92fd24a395f88fb69335e8 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:20:35 -0600 Subject: [PATCH 08/18] where to put compose files? --- elastic-container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index e1aec76..1e545f5 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -239,7 +239,7 @@ case "${ACTION}" in echo "Starting Elastic Stack network and containers." - ${COMPOSE} up -d ${COMPOSE_STRING} --no-deps + ${COMPOSE} up ${COMPOSE_STRING} -d --no-deps configure_kbn 1>&2 2>&3 @@ -279,7 +279,7 @@ case "${ACTION}" in echo "#####" echo "Stopping and removing the containers, network, and volumes created." echo "#####" - ${COMPOSE} down -v ${COMPOSE_STRING} + ${COMPOSE} down ${COMPOSE_STRING} -v ;; "restart") From a3070fca28dbc3fe2866c478727952d6fbde1c13 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 12:23:56 -0600 Subject: [PATCH 09/18] still trying to figure out where it goes --- elastic-container.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index 1e545f5..8b7d6e3 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -239,7 +239,7 @@ case "${ACTION}" in echo "Starting Elastic Stack network and containers." - ${COMPOSE} up ${COMPOSE_STRING} -d --no-deps + ${COMPOSE} ${COMPOSE_STRING} up -d :--no-deps configure_kbn 1>&2 2>&3 @@ -272,14 +272,14 @@ case "${ACTION}" in "stop") echo "Stopping running containers." - ${COMPOSE} stop ${COMPOSE_STRING} + ${COMPOSE} ${COMPOSE_STRING} stop ;; "destroy") echo "#####" echo "Stopping and removing the containers, network, and volumes created." echo "#####" - ${COMPOSE} down ${COMPOSE_STRING} -v + ${COMPOSE} ${COMPOSE_STRING} down -v ;; "restart") @@ -290,7 +290,7 @@ case "${ACTION}" in ;; "status") - ${COMPOSE} ps ${COMPOSE_STRING}| grep -v setup + ${COMPOSE} ${COMPOSE_STRING} ps | grep -v setup ;; "clear") From 955138dd6a76fb2ca1b02b4586f6d0c65a11f59a Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 16:48:39 -0600 Subject: [PATCH 10/18] move some stuff --- README.md | 2 +- .env => compose/.env | 0 elastic-container.sh | 8 ++++---- 3 files changed, 5 insertions(+), 5 deletions(-) rename .env => compose/.env (100%) diff --git a/README.md b/README.md index a9559e1..bff88f8 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you're interested in more details regarding this project and what to do once 1. `Git clone` this repo 2. Install prerequisites (see below) 3. Change into the `elastic-container/` folder -4. Change the default password of `changeme` in the `.env` file (don't change the `elastic` username, it's a [required built-in user](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html)) +4. Change the default password of `changeme` in the `.env` file located in the `compose` directory (don't change the `elastic` username, it's a [required built-in user](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html)) 5. Bulk enable pre-built detection rules by OS in the `.env` file (not required, see usage below) 6. Make the `elastic-container.sh` shell script executable by running `chmod +x elastic-container.sh` 7. Execute the `elastic-container.sh` shell script with the start argument `./elastic-container.sh start` diff --git a/.env b/compose/.env similarity index 100% rename from .env rename to compose/.env diff --git a/elastic-container.sh b/elastic-container.sh index 8b7d6e3..d3bdd5a 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -21,7 +21,7 @@ HEADERS=( ) passphrase_reset() { - if grep -Fq "changeme" .env; then + if grep -Fq "changeme" compose/.env; then echo "Sorry, looks like you haven't updated the passphrase from the default" echo "Please update the changeme passphrases in the .env file." exit 1 @@ -33,7 +33,7 @@ passphrase_reset() { # Create the script usage menu usage() { cat <&2 2>&3 From 77d952e1d8f5cbec58585f06bd46f137aa6311b2 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 17:55:06 -0600 Subject: [PATCH 11/18] move out of directory --- compose/docker-compose.yml => docker-compose.yml | 0 .../no-fleet-docker-compose.yml => no-fleet-docker-compose.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename compose/docker-compose.yml => docker-compose.yml (100%) rename compose/no-fleet-docker-compose.yml => no-fleet-docker-compose.yml (100%) diff --git a/compose/docker-compose.yml b/docker-compose.yml similarity index 100% rename from compose/docker-compose.yml rename to docker-compose.yml diff --git a/compose/no-fleet-docker-compose.yml b/no-fleet-docker-compose.yml similarity index 100% rename from compose/no-fleet-docker-compose.yml rename to no-fleet-docker-compose.yml From 820b17e298d338f7c1a05c3a6b49a57302ef813b Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 17:55:22 -0600 Subject: [PATCH 12/18] rm dir --- elastic-container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index d3bdd5a..cbcb315 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -219,9 +219,9 @@ else fi if fleet=0; then - COMPOSE_STRING=" --file compose/no-fleet-docker-compose.yml" + COMPOSE_STRING=" --file no-fleet-docker-compose.yml" else - COMPOSE_STRING=" --file compose/docker-compose.yml" + COMPOSE_STRING=" --file docker-compose.yml" fi case "${ACTION}" in From 5031b3196b2c3624d04d1d34552d397174c2c0f5 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 17:57:11 -0600 Subject: [PATCH 13/18] more cleanup --- compose/.env => .env | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename compose/.env => .env (100%) diff --git a/compose/.env b/.env similarity index 100% rename from compose/.env rename to .env From f99a12ec22cb5b43deb20a9e1d83467d5dad901c Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 18:01:18 -0600 Subject: [PATCH 14/18] fix if --- elastic-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic-container.sh b/elastic-container.sh index cbcb315..8c538b2 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -218,7 +218,7 @@ else exit 2 fi -if fleet=0; then +if [ $fleet -eq 1 ]; then COMPOSE_STRING=" --file no-fleet-docker-compose.yml" else COMPOSE_STRING=" --file docker-compose.yml" From 51c0f471b1cc88152c09c1f63992934db2d607b8 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 18:09:21 -0600 Subject: [PATCH 15/18] fix if statement --- elastic-container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index 8c538b2..609faf4 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -218,7 +218,7 @@ else exit 2 fi -if [ $fleet -eq 1 ]; then +if [ $fleet -eq 0 ]; then COMPOSE_STRING=" --file no-fleet-docker-compose.yml" else COMPOSE_STRING=" --file docker-compose.yml" @@ -243,7 +243,7 @@ case "${ACTION}" in configure_kbn 1>&2 2>&3 - if fleet=0; then + if [ $fleet -eq 0 ]; then echo "Not waiting 40 seconds for Fleet Server setup." echo From 6a60a12039e7dbcb9589a2d80ed46289401fbb9c Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 18:16:24 -0600 Subject: [PATCH 16/18] more cleanup --- elastic-container.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/elastic-container.sh b/elastic-container.sh index 609faf4..4c831b6 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -186,11 +186,9 @@ while getopts ":vn" opt; do case "$opt" in v) verbose=1 - echo "Moar info!!!" ;; n) fleet=0 - echo "What did fleet ever do to you?" ;; *) ;; esac @@ -245,7 +243,7 @@ case "${ACTION}" in if [ $fleet -eq 0 ]; then - echo "Not waiting 40 seconds for Fleet Server setup." + echo "Not waiting 40 seconds for Fleet Server setup. It is not used." echo else From 7e62f1fadb7d7ca6b5ba63d898245294c564f992 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 20:18:26 -0600 Subject: [PATCH 17/18] Update elastic-container.sh --- elastic-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic-container.sh b/elastic-container.sh index 4c831b6..237dfc4 100755 --- a/elastic-container.sh +++ b/elastic-container.sh @@ -21,7 +21,7 @@ HEADERS=( ) passphrase_reset() { - if grep -Fq "changeme" compose/.env; then + if grep -Fq "changeme" .env; then echo "Sorry, looks like you haven't updated the passphrase from the default" echo "Please update the changeme passphrases in the .env file." exit 1 From 62805a0dec00b81c7ee60cf9a6815a4d2e4075d3 Mon Sep 17 00:00:00 2001 From: Kirk K Date: Thu, 26 Dec 2024 20:20:36 -0600 Subject: [PATCH 18/18] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bff88f8..a9559e1 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ If you're interested in more details regarding this project and what to do once 1. `Git clone` this repo 2. Install prerequisites (see below) 3. Change into the `elastic-container/` folder -4. Change the default password of `changeme` in the `.env` file located in the `compose` directory (don't change the `elastic` username, it's a [required built-in user](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html)) +4. Change the default password of `changeme` in the `.env` file (don't change the `elastic` username, it's a [required built-in user](https://www.elastic.co/guide/en/elasticsearch/reference/current/built-in-users.html)) 5. Bulk enable pre-built detection rules by OS in the `.env` file (not required, see usage below) 6. Make the `elastic-container.sh` shell script executable by running `chmod +x elastic-container.sh` 7. Execute the `elastic-container.sh` shell script with the start argument `./elastic-container.sh start`