diff --git a/docker/release/README.md b/docker/release/README.md index 4c79ddc8b7..b398a5ddfb 100644 --- a/docker/release/README.md +++ b/docker/release/README.md @@ -91,7 +91,7 @@ Here are three example scenarios of using above variables: ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:1.1.0 ``` - Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration. + **Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration.** ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e OPENSEARCH_INITIAL_ADMIN_PASSWORD= -e "discovery.type=single-node" opensearchproject/opensearch:2.12.0 ``` @@ -133,7 +133,7 @@ Here are some example scenarios of using above variables: ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch: ``` - Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration. + **Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration.** ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e OPENSEARCH_INITIAL_ADMIN_PASSWORD= -e "discovery.type=single-node" opensearchproject/opensearch: ``` @@ -143,7 +143,32 @@ Here are some example scenarios of using above variables: ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true" opensearchproject/opensearch: ``` - Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration. + **Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration.** ``` $ docker run -it -p 9200:9200 -p 9600:9600 -e OPENSEARCH_INITIAL_ADMIN_PASSWORD= -e "discovery.type=single-node" -e "DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true" opensearchproject/opensearch: ``` + +### Enable SIMD in k-NN Plugin Faiss Lib +(This change is added since OpenSearch 2.12.0) + + * 1 for OpenSearch: + * __KNN_SIMD_ENABLED__: Default to `null`, set to `true` enables [SIMD in k-NN Faiss Lib](https://github.com/opensearch-project/k-NN/issues/1138). + +Here are some example scenarios of using above variables: + + +#### Scenario 1: Original behavior, disable SIMD + * OpenSearch: + + **Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration.** + ``` + $ docker run -it -p 9200:9200 -p 9600:9600 -e OPENSEARCH_INITIAL_ADMIN_PASSWORD= -e "discovery.type=single-node" opensearchproject/opensearch: + ``` + +#### Scenario 2: Enable SIMD + * OpenSearch: + + **Note: For OpenSearch 2.12 and later, a custom password for the admin user is required to be passed to set-up and utilize demo configuration.** + ``` + $ docker run -it -p 9200:9200 -p 9600:9600 -e OPENSEARCH_INITIAL_ADMIN_PASSWORD= -e "discovery.type=single-node" -e "KNN_SIMD_ENABLED=true" opensearchproject/opensearch: + ``` diff --git a/docker/release/config/opensearch/opensearch-docker-entrypoint.sh b/docker/release/config/opensearch/opensearch-docker-entrypoint-1.x.sh similarity index 100% rename from docker/release/config/opensearch/opensearch-docker-entrypoint.sh rename to docker/release/config/opensearch/opensearch-docker-entrypoint-1.x.sh diff --git a/docker/release/config/opensearch/opensearch-docker-entrypoint-2.x.sh b/docker/release/config/opensearch/opensearch-docker-entrypoint-2.x.sh new file mode 100755 index 0000000000..7a196e0f79 --- /dev/null +++ b/docker/release/config/opensearch/opensearch-docker-entrypoint-2.x.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +# This script specify the entrypoint startup actions for opensearch +# It will start both opensearch and performance analyzer plugin cli +# If either process failed, the entire docker container will be removed +# in favor of a newly started container + +# Export OpenSearch Home +export OPENSEARCH_HOME=/usr/share/opensearch +export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config + +# The virtual file /proc/self/cgroup should list the current cgroup +# membership. For each hierarchy, you can follow the cgroup path from +# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and +# introspect the statistics for the cgroup for the given +# hierarchy. Alas, Docker breaks this by mounting the container +# statistics at the root while leaving the cgroup paths as the actual +# paths. Therefore, OpenSearch provides a mechanism to override +# reading the cgroup path from /proc/self/cgroup and instead uses the +# cgroup path defined the JVM system property +# opensearch.cgroups.hierarchy.override. Therefore, we set this value here so +# that cgroup statistics are available for the container this process +# will run in. +export OPENSEARCH_JAVA_OPTS="-Dopensearch.cgroups.hierarchy.override=/ $OPENSEARCH_JAVA_OPTS" + +# Security Plugin +function setupSecurityPlugin { + SECURITY_PLUGIN="opensearch-security" + + if [ -d "$OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN" ]; then + if [ "$DISABLE_INSTALL_DEMO_CONFIG" = "true" ]; then + echo "Disabling execution of install_demo_configuration.sh for OpenSearch Security Plugin" + else + echo -e "Enabling execution of install_demo_configuration.sh for OpenSearch Security Plugin \nOpenSearch 2.12.0 onwards, the OpenSearch Security Plugin a change that requires an initial password for 'admin' user. \nPlease define an environment variable 'OPENSEARCH_INITIAL_ADMIN_PASSWORD' with a strong password string. \nIf a password is not provided, the setup will quit. \n For more details, please visit: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/" + bash $OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN/tools/install_demo_configuration.sh -y -i -s || exit 1 + fi + + if [ "$DISABLE_SECURITY_PLUGIN" = "true" ]; then + echo "Disabling OpenSearch Security Plugin" + opensearch_opt="-Eplugins.security.disabled=true" + opensearch_opts+=("${opensearch_opt}") + else + echo "Enabling OpenSearch Security Plugin" + fi + else + echo "OpenSearch Security Plugin does not exist, disable by default" + fi +} + +# Performance Analyzer Plugin +function setupPerformanceAnalyzerPlugin { + PERFORMANCE_ANALYZER_PLUGIN="opensearch-performance-analyzer" + if [ -d "$OPENSEARCH_HOME/plugins/$PERFORMANCE_ANALYZER_PLUGIN" ]; then + if [ "$DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI" = "true" ]; then + echo "Disabling execution of $OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin" + else + echo "Enabling execution of OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin" + $OPENSEARCH_HOME/bin/opensearch-performance-analyzer/performance-analyzer-agent-cli > $OPENSEARCH_HOME/logs/PerformanceAnalyzer.log 2>&1 & disown + fi + else + echo "OpenSearch Performance Analyzer Plugin does not exist, disable by default" + fi +} + +# Setup k-NN Plugin Lib Loading Path +# This is required for OpenSearch 2.12 and above +# As SIMD is added to the k-NN Plugin Faiss Lib: https://github.com/opensearch-project/k-NN/issues/1138 +function setupKNNPlugin { + if [ "$KNN_SIMD_ENABLED" = "true" ]; + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib_simd + echo "Enable SIMD Feature for k-NN Plugin" + else + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib + echo "Disable SIMD Feature for k-NN Plugin" + fi + + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$KNN_LIB_DIR" +} + +# Start up the opensearch and performance analyzer agent processes. +# When either of them halts, this script exits, or we receive a SIGTERM or SIGINT signal then we want to kill both these processes. +function runOpensearch { + # Files created by OpenSearch should always be group writable too + umask 0002 + + if [[ "$(id -u)" == "0" ]]; then + echo "OpenSearch cannot run as root. Please start your container as another user." + exit 1 + fi + + # Parse Docker env vars to customize OpenSearch + # + # e.g. Setting the env var cluster.name=testcluster + # will cause OpenSearch to be invoked with -Ecluster.name=testcluster + opensearch_opts=() + while IFS='=' read -r envvar_key envvar_value + do + # OpenSearch settings need to have at least two dot separated lowercase + # words, e.g. `cluster.name`, except for `processors` which we handle + # specially + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then + if [[ ! -z $envvar_value ]]; then + opensearch_opt="-E${envvar_key}=${envvar_value}" + opensearch_opts+=("${opensearch_opt}") + fi + fi + done < <(env) + + setupSecurityPlugin + setupPerformanceAnalyzerPlugin + setupKNNPlugin + + # Start opensearch + "$@" "${opensearch_opts[@]}" + +} + +# Prepend "opensearch" command if no argument was provided or if the first +# argument looks like a flag (i.e. starts with a dash). +if [ $# -eq 0 ] || [ "${1:0:1}" = '-' ]; then + set -- opensearch "$@" +fi + +if [ "$1" = "opensearch" ]; then + # If the first argument is opensearch, then run the setup script. + runOpensearch "$@" +else + # Otherwise, just exec the command. + exec "$@" +fi diff --git a/docker/release/config/opensearch/opensearch-docker-entrypoint-default.x.sh b/docker/release/config/opensearch/opensearch-docker-entrypoint-default.x.sh new file mode 100755 index 0000000000..7a196e0f79 --- /dev/null +++ b/docker/release/config/opensearch/opensearch-docker-entrypoint-default.x.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 + +# This script specify the entrypoint startup actions for opensearch +# It will start both opensearch and performance analyzer plugin cli +# If either process failed, the entire docker container will be removed +# in favor of a newly started container + +# Export OpenSearch Home +export OPENSEARCH_HOME=/usr/share/opensearch +export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config + +# The virtual file /proc/self/cgroup should list the current cgroup +# membership. For each hierarchy, you can follow the cgroup path from +# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and +# introspect the statistics for the cgroup for the given +# hierarchy. Alas, Docker breaks this by mounting the container +# statistics at the root while leaving the cgroup paths as the actual +# paths. Therefore, OpenSearch provides a mechanism to override +# reading the cgroup path from /proc/self/cgroup and instead uses the +# cgroup path defined the JVM system property +# opensearch.cgroups.hierarchy.override. Therefore, we set this value here so +# that cgroup statistics are available for the container this process +# will run in. +export OPENSEARCH_JAVA_OPTS="-Dopensearch.cgroups.hierarchy.override=/ $OPENSEARCH_JAVA_OPTS" + +# Security Plugin +function setupSecurityPlugin { + SECURITY_PLUGIN="opensearch-security" + + if [ -d "$OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN" ]; then + if [ "$DISABLE_INSTALL_DEMO_CONFIG" = "true" ]; then + echo "Disabling execution of install_demo_configuration.sh for OpenSearch Security Plugin" + else + echo -e "Enabling execution of install_demo_configuration.sh for OpenSearch Security Plugin \nOpenSearch 2.12.0 onwards, the OpenSearch Security Plugin a change that requires an initial password for 'admin' user. \nPlease define an environment variable 'OPENSEARCH_INITIAL_ADMIN_PASSWORD' with a strong password string. \nIf a password is not provided, the setup will quit. \n For more details, please visit: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/docker/" + bash $OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN/tools/install_demo_configuration.sh -y -i -s || exit 1 + fi + + if [ "$DISABLE_SECURITY_PLUGIN" = "true" ]; then + echo "Disabling OpenSearch Security Plugin" + opensearch_opt="-Eplugins.security.disabled=true" + opensearch_opts+=("${opensearch_opt}") + else + echo "Enabling OpenSearch Security Plugin" + fi + else + echo "OpenSearch Security Plugin does not exist, disable by default" + fi +} + +# Performance Analyzer Plugin +function setupPerformanceAnalyzerPlugin { + PERFORMANCE_ANALYZER_PLUGIN="opensearch-performance-analyzer" + if [ -d "$OPENSEARCH_HOME/plugins/$PERFORMANCE_ANALYZER_PLUGIN" ]; then + if [ "$DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI" = "true" ]; then + echo "Disabling execution of $OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin" + else + echo "Enabling execution of OPENSEARCH_HOME/bin/$PERFORMANCE_ANALYZER_PLUGIN/performance-analyzer-agent-cli for OpenSearch Performance Analyzer Plugin" + $OPENSEARCH_HOME/bin/opensearch-performance-analyzer/performance-analyzer-agent-cli > $OPENSEARCH_HOME/logs/PerformanceAnalyzer.log 2>&1 & disown + fi + else + echo "OpenSearch Performance Analyzer Plugin does not exist, disable by default" + fi +} + +# Setup k-NN Plugin Lib Loading Path +# This is required for OpenSearch 2.12 and above +# As SIMD is added to the k-NN Plugin Faiss Lib: https://github.com/opensearch-project/k-NN/issues/1138 +function setupKNNPlugin { + if [ "$KNN_SIMD_ENABLED" = "true" ]; + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib_simd + echo "Enable SIMD Feature for k-NN Plugin" + else + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib + echo "Disable SIMD Feature for k-NN Plugin" + fi + + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$KNN_LIB_DIR" +} + +# Start up the opensearch and performance analyzer agent processes. +# When either of them halts, this script exits, or we receive a SIGTERM or SIGINT signal then we want to kill both these processes. +function runOpensearch { + # Files created by OpenSearch should always be group writable too + umask 0002 + + if [[ "$(id -u)" == "0" ]]; then + echo "OpenSearch cannot run as root. Please start your container as another user." + exit 1 + fi + + # Parse Docker env vars to customize OpenSearch + # + # e.g. Setting the env var cluster.name=testcluster + # will cause OpenSearch to be invoked with -Ecluster.name=testcluster + opensearch_opts=() + while IFS='=' read -r envvar_key envvar_value + do + # OpenSearch settings need to have at least two dot separated lowercase + # words, e.g. `cluster.name`, except for `processors` which we handle + # specially + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then + if [[ ! -z $envvar_value ]]; then + opensearch_opt="-E${envvar_key}=${envvar_value}" + opensearch_opts+=("${opensearch_opt}") + fi + fi + done < <(env) + + setupSecurityPlugin + setupPerformanceAnalyzerPlugin + setupKNNPlugin + + # Start opensearch + "$@" "${opensearch_opts[@]}" + +} + +# Prepend "opensearch" command if no argument was provided or if the first +# argument looks like a flag (i.e. starts with a dash). +if [ $# -eq 0 ] || [ "${1:0:1}" = '-' ]; then + set -- opensearch "$@" +fi + +if [ "$1" = "opensearch" ]; then + # If the first argument is opensearch, then run the setup script. + runOpensearch "$@" +else + # Otherwise, just exec the command. + exec "$@" +fi diff --git a/docker/release/dockercomposefiles/docker-compose-2.x.yml b/docker/release/dockercomposefiles/docker-compose-2.x.yml index da8b7e9575..38459206fd 100644 --- a/docker/release/dockercomposefiles/docker-compose-2.x.yml +++ b/docker/release/dockercomposefiles/docker-compose-2.x.yml @@ -12,6 +12,7 @@ services: - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} # Sets the demo admin user password when using demo configuration, required for OpenSearch 2.12 and higher + # - KNN_SIMD_ENABLED=true # Sets true to enable SIMD in k-NN Plugin Faiss Lib ulimits: memlock: soft: -1 @@ -37,6 +38,7 @@ services: - bootstrap.memory_lock=true - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + # - KNN_SIMD_ENABLED=true ulimits: memlock: soft: -1 diff --git a/docker/release/dockerfiles/opensearch.al2023.dockerfile b/docker/release/dockerfiles/opensearch.al2023.dockerfile index c6f853fb83..666196cc24 100644 --- a/docker/release/dockerfiles/opensearch.al2023.dockerfile +++ b/docker/release/dockerfiles/opensearch.al2023.dockerfile @@ -39,10 +39,14 @@ RUN groupadd -g $GID opensearch && \ COPY * $TEMP_DIR/ RUN ls -l $TEMP_DIR && \ tar -xzpf /tmp/opensearch/opensearch-`uname -p`.tgz -C $OPENSEARCH_HOME --strip-components=1 && \ + MAJOR_VERSION_ENTRYPOINT=`echo $VERSION | cut -d. -f1` && \ + echo $MAJOR_VERSION_ENTRYPOINT && \ + if ! (ls $TEMP_DIR | grep -E "opensearch-docker-entrypoint-.*.x.sh" | grep $MAJOR_VERSION_ENTRYPOINT); then MAJOR_VERSION_ENTRYPOINT="default"; fi && \ mkdir -p $OPENSEARCH_HOME/data && chown -Rv $UID:$GID $OPENSEARCH_HOME/data && \ if [[ -d $SECURITY_PLUGIN_DIR ]] ; then chmod -v 750 $SECURITY_PLUGIN_DIR/tools/* ; fi && \ if [[ -d $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR ]] ; then cp -v $TEMP_DIR/performance-analyzer.properties $PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR; fi && \ - cp -v $TEMP_DIR/opensearch-docker-entrypoint.sh $TEMP_DIR/opensearch-onetime-setup.sh $OPENSEARCH_HOME/ && \ + cp -v $TEMP_DIR/opensearch-docker-entrypoint-$MAJOR_VERSION_ENTRYPOINT.x.sh $OPENSEARCH_HOME/opensearch-docker-entrypoint.sh && \ + cp -v $TEMP_DIR/opensearch-onetime-setup.sh $OPENSEARCH_HOME/ && \ cp -v $TEMP_DIR/log4j2.properties $TEMP_DIR/opensearch.yml $OPENSEARCH_PATH_CONF/ && \ ls -l $OPENSEARCH_HOME && \ rm -rf $TEMP_DIR @@ -77,9 +81,6 @@ RUN echo "export JAVA_HOME=$OPENSEARCH_HOME/jdk" >> /etc/profile.d/java_home.sh ENV JAVA_HOME=$OPENSEARCH_HOME/jdk ENV PATH=$PATH:$JAVA_HOME/bin:$OPENSEARCH_HOME/bin -# Add k-NN lib directory to library loading path variable -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$OPENSEARCH_HOME/plugins/opensearch-knn/lib" - # Change user USER $UID diff --git a/scripts/startup/tar/linux/opensearch-tar-install.sh b/scripts/startup/tar/linux/opensearch-tar-install.sh index 47d820606e..8f4a55fa73 100755 --- a/scripts/startup/tar/linux/opensearch-tar-install.sh +++ b/scripts/startup/tar/linux/opensearch-tar-install.sh @@ -7,7 +7,7 @@ export OPENSEARCH_HOME=`dirname $(realpath $0)` export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config cd $OPENSEARCH_HOME -KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib + ##Security Plugin if [ -d "$OPENSEARCH_HOME/plugins/opensearch-security" ]; then echo -e "OpenSearch 2.12.0 onwards, the OpenSearch Security Plugin introduces a change that requires an initial password for 'admin' user. \nPlease define an environment variable 'OPENSEARCH_INITIAL_ADMIN_PASSWORD' with a strong password string. \nIf a password is not provided, the setup will quit. \nFor more details, please visit: https://opensearch.org/docs/latest/install-and-configure/install-opensearch/tar/" @@ -34,6 +34,17 @@ if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_PATH_CONF/jvm.opti fi echo "done plugins" +##Setup k-NN Plugin Lib Loading Path +##This is required for OpenSearch 2.12 and above +##As SIMD is added to the k-NN Plugin Faiss Lib: https://github.com/opensearch-project/k-NN/issues/1138 +if [ "$KNN_SIMD_ENABLED" = "true" ]; + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib_simd + echo "Enable SIMD Feature for k-NN Plugin" +else + KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/lib + echo "Disable SIMD Feature for k-NN Plugin" +fi + ##Set KNN Dylib Path for macOS and *nix systems if echo "$OSTYPE" | grep -qi "darwin"; then if echo "$JAVA_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then