Performance benchmark workflow

.github/workflows/benchmark.yml

@@ -0,0 +1,69 @@
+name: Benchmark
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      - 1.*
+      - 2.*
+  pull_request:
+    branches:
+      - main
+      - 1.*
+      - 2.*
+env:
+  JAVA_VERSION: 21
+  OPENSEARCH_VERSION: 3.0.0
+  OPENSEARCH_ADMIN_PASSWORD: Hello_World123
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    name: Benchmark
+    steps:
+      - name: Set up JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: ${{ env.JAVA_VERSION }}
+      - name: Checkout Branch
+        uses: actions/checkout@v4
+      - name: Set up gradle
+        uses: gradle/actions/setup-gradle@v3
+        with:
+          cache-disabled: false
+      - name: Assemble target plugin
+        run: ./gradlew assemble
+      - name: Prepare Security Plugin
+        run: mv ./build/distributions/opensearch-security-*.zip ./benchmark/docker/
+      - name: Build Docker Image
+        run: docker build --platform linux/amd64 --build-arg VERSION=${{ env.OPENSEARCH_VERSION }} -f benchmark/docker/benchmark.dockerfile -t opensearchproject/security-benchmark:latest benchmark/docker/
+      - name: Run Benchmark Cluster
+        uses: isbang/[email protected]
+        with:
+          compose-file: "./benchmark/docker-compose.yml"
+        env:
+          OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${{ env.OPENSEARCH_ADMIN_PASSWORD }}
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Benchmark
+        run: pip install opensearch-benchmark
+      - name: Execute Benchmarks
+        run: opensearch-benchmark execute-test --pipeline=benchmark-only --results-format=csv --results-file=./results.csv --on-error=abort --workload=percolator --target-host=https://localhost:9200 --client-options=basic_auth_user:admin,basic_auth_password:${{ env.OPENSEARCH_ADMIN_PASSWORD }},verify_certs:false,timeout:60
+      - name: Prepare Benchmark Results
+        run: python benchmark/result_rewriter.py ./results.csv ./results.json
+      - name: Download previous benchmark data
+        uses: actions/cache@v4
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      - name: Store Benchmark Results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: customSmallerIsBetter
+          output-file-path: ./results.json
+          external-data-json-path: ./cache/benchmark-data.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          fail-on-alert: true
+          alert-threshold: '150%'

benchmark/docker-compose.yml

@@ -0,0 +1,95 @@
+version: '3'
+services:
+  opensearch-node1:
+    image: opensearchproject/security-benchmark:latest
+    platform: linux/amd64
+    container_name: opensearch-node1
+    environment:
+      - bootstrap.system_call_filter=false
+      - network.host=0.0.0.0
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch-node1
+      - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
+      - plugins.security.restapi.password_min_length=1
+      - plugins.security.restapi.password_score_based_validation_strength=fair
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data1:/usr/share/opensearch/data
+    ports:
+      - "9200:9200"
+    expose:
+      - "9200"
+    networks:
+      - opensearch-net
+  opensearch-node2:
+    image: opensearchproject/security-benchmark:latest
+    platform: linux/amd64
+    container_name: opensearch-node2
+    environment:
+      - bootstrap.system_call_filter=false
+      - network.host=0.0.0.0
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch-node2
+      - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
+      - plugins.security.restapi.password_min_length=1
+      - plugins.security.restapi.password_score_based_validation_strength=fair
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data2:/usr/share/opensearch/data
+    networks:
+      - opensearch-net
+  opensearch-node3:
+    image: opensearchproject/security-benchmark:latest
+    platform: linux/amd64
+    container_name: opensearch-node3
+    environment:
+      - bootstrap.system_call_filter=false
+      - network.host=0.0.0.0
+      - cluster.name=opensearch-cluster
+      - node.name=opensearch-node3
+      - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3
+      - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3
+      - bootstrap.memory_lock=true
+      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
+      - plugins.security.restapi.password_min_length=1
+      - plugins.security.restapi.password_score_based_validation_strength=fair
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    volumes:
+      - opensearch-data3:/usr/share/opensearch/data
+    networks:
+      - opensearch-net
+
+volumes:
+  opensearch-data1:
+  opensearch-data2:
+  opensearch-data3:
+
+networks:
+  opensearch-net:

benchmark/docker/benchmark.dockerfile

@@ -0,0 +1,59 @@
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS linux_stage_0
+
+ARG UID=1000
+ARG GID=1000
+ARG VERSION
+ARG TEMP_DIR=/tmp/opensearch
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+ARG OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config
+ARG SECURITY_PLUGIN_DIR=$OPENSEARCH_HOME/plugins/opensearch-security
+ARG PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer
+
+RUN dnf update --releasever=latest -y && dnf install -y tar gzip shadow-utils which wget && dnf clean all
+
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \
+    mkdir $TEMP_DIR
+
+COPY entrypoint.sh $OPENSEARCH_HOME/entrypoint.sh
+
+RUN ls -l $TEMP_DIR && \
+    wget https://artifacts.opensearch.org/snapshots/core/opensearch/${VERSION}-SNAPSHOT/opensearch-min-${VERSION}-SNAPSHOT-linux-x64-latest.tar.gz && \
+    tar -xzpf opensearch-*.tar.gz -C $OPENSEARCH_HOME --strip-components=1 && \
+    mkdir -p $OPENSEARCH_HOME/data && chown -Rv $UID:$GID $OPENSEARCH_HOME/data && \
+    chmod +x $OPENSEARCH_HOME/entrypoint.sh && \
+    ls -l $OPENSEARCH_HOME
+
+
+
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+
+RUN dnf update --releasever=latest -y && dnf install -y tar gzip shadow-utils which && dnf clean all
+
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch
+
+COPY --from=linux_stage_0 --chown=$UID:$GID $OPENSEARCH_HOME $OPENSEARCH_HOME
+WORKDIR $OPENSEARCH_HOME
+
+RUN echo "export JAVA_HOME=$OPENSEARCH_HOME/jdk" >> /etc/profile.d/java_home.sh && \
+    echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> /etc/profile.d/java_home.sh && \
+    ls -l $OPENSEARCH_HOME
+
+ENV JAVA_HOME=$OPENSEARCH_HOME/jdk
+ENV PATH=$PATH:$JAVA_HOME/bin:$OPENSEARCH_HOME/bin
+
+COPY opensearch-security-*.zip $TEMP_DIR/opensearch-security.zip
+
+USER $UID
+
+RUN /bin/bash -c "yes | ./bin/opensearch-plugin install file:$TEMP_DIR/opensearch-security.zip"
+
+EXPOSE 9200 9300
+
+ENTRYPOINT ["./entrypoint.sh"]
+CMD ["opensearch"]

benchmark/docker/entrypoint.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+export OPENSEARCH_HOME=/usr/share/opensearch
+
+chmod +x  "$OPENSEARCH_HOME"/plugins/opensearch-security/tools/install_demo_configuration.sh
+bash "$OPENSEARCH_HOME"/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s || exit 1
+
+function runOpensearch {
+    umask 0002
+
+    if [[ "$(id -u)" == "0" ]]; then
+        echo "OpenSearch cannot run as root. Please start your container as another user."
+        exit 1
+    fi
+
+    opensearch_opts=()
+    while IFS='=' read -r envvar_key envvar_value
+    do
+        if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then
+            if [[ ! -z $envvar_value ]]; then
+            opensearch_opt="-E${envvar_key}=${envvar_value}"
+            opensearch_opts+=("${opensearch_opt}")
+            fi
+        fi
+    done < <(env)
+
+    "$@" "${opensearch_opts[@]}"
+}
+
+if [ $# -eq 0 ] || [ "${1:0:1}" = '-' ]; then
+    set -- opensearch "$@"
+fi
+
+if [ "$1" = "opensearch" ]; then
+    runOpensearch "$@"
+else
+    exec "$@"
+fi

benchmark/result_rewriter.py

@@ -0,0 +1,53 @@
+import json
+import csv
+import sys
+
+
+def create_entry(name, value, unit):
+    entry = {
+        "name": name,
+        "value": value,
+        "unit": unit
+    }
+    return entry
+
+
+def rewrite_csv(src_file):
+    try:
+        result = []
+        with open(src_file, 'r', newline='') as file:
+            reader = csv.reader(file)
+            next(reader)
+            for row in reader:
+                if row[1] == "" or row[1] is None:
+                    continue
+                else:
+                    name = f"{row[0]} {row[1]}"
+                if name.startswith("Min") or name.startswith("Max") or name.startswith("Median") or not name.startswith("50th"):
+                    continue
+                unit = row[3]
+                value = float(row[2])
+                if unit == "ops/s":
+                    value = 1 / value
+                    unit = "s/ops"
+                elif unit == "docs/s":
+                    value = 1 / value
+                    unit = "s/docs"
+                entry = create_entry(name, value, unit)
+                result.append(entry)
+            return result
+    except Exception as e:
+        print(f"Failed to rewrite benchmark results: {e} {row}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Wrong number of arguments")
+        sys.exit(1)
+    else:
+        src = sys.argv[1]
+        dest = sys.argv[2]
+        json_res = rewrite_csv(src)
+        with open(dest, 'w') as f:
+            json.dump(json_res, f, indent=4)