From 06015f08256d0b324ed2d056b47d6336bbcc1d12 Mon Sep 17 00:00:00 2001 From: Aki Abramowski Date: Tue, 23 Apr 2024 23:04:52 +0200 Subject: [PATCH] add security performance benchmark workflow Signed-off-by: Aki Abramowski --- .github/workflows/benchmark.yml | 69 +++++++++++++++++++ benchmark/docker-compose.yml | 95 +++++++++++++++++++++++++++ benchmark/docker/benchmark.dockerfile | 59 +++++++++++++++++ benchmark/docker/entrypoint.sh | 38 +++++++++++ benchmark/result_rewriter.py | 53 +++++++++++++++ 5 files changed, 314 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/docker-compose.yml create mode 100644 benchmark/docker/benchmark.dockerfile create mode 100644 benchmark/docker/entrypoint.sh create mode 100644 benchmark/result_rewriter.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..4116855577 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,69 @@ +name: Benchmark +on: + workflow_dispatch: + push: + branches: + - main + - 1.* + - 2.* + pull_request: + branches: + - main + - 1.* + - 2.* +env: + JAVA_VERSION: 21 + OPENSEARCH_VERSION: 3.0.0 + OPENSEARCH_ADMIN_PASSWORD: Hello_World123 +jobs: + benchmark: + runs-on: ubuntu-latest + name: Benchmark + steps: + - name: Set up JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ env.JAVA_VERSION }} + - name: Checkout Branch + uses: actions/checkout@v4 + - name: Set up gradle + uses: gradle/actions/setup-gradle@v3 + with: + cache-disabled: false + - name: Assemble target plugin + run: ./gradlew assemble + - name: Prepare Security Plugin + run: mv ./build/distributions/opensearch-security-*.zip ./benchmark/docker/ + - name: Build Docker Image + run: docker build --platform linux/amd64 --build-arg VERSION=${{ env.OPENSEARCH_VERSION }} -f benchmark/docker/benchmark.dockerfile -t opensearchproject/security-benchmark:latest benchmark/docker/ + - name: Run Benchmark Cluster + uses: isbang/compose-action@v1.5.1 + with: + compose-file: "./benchmark/docker-compose.yml" + env: + OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${{ env.OPENSEARCH_ADMIN_PASSWORD }} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Benchmark + run: pip install opensearch-benchmark + - name: Execute Benchmarks + run: opensearch-benchmark execute-test --pipeline=benchmark-only --results-format=csv --results-file=./results.csv --on-error=abort --workload=percolator --target-host=https://localhost:9200 --client-options=basic_auth_user:admin,basic_auth_password:${{ env.OPENSEARCH_ADMIN_PASSWORD }},verify_certs:false,timeout:60 + - name: Prepare Benchmark Results + run: python benchmark/result_rewriter.py ./results.csv ./results.json + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-benchmark + - name: Store Benchmark Results + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: customSmallerIsBetter + output-file-path: ./results.json + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + alert-threshold: '150%' diff --git a/benchmark/docker-compose.yml b/benchmark/docker-compose.yml new file mode 100644 index 0000000000..1aeff0ab03 --- /dev/null +++ b/benchmark/docker-compose.yml @@ -0,0 +1,95 @@ +version: '3' +services: + opensearch-node1: + image: opensearchproject/security-benchmark:latest + platform: linux/amd64 + container_name: opensearch-node1 + environment: + - bootstrap.system_call_filter=false + - network.host=0.0.0.0 + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + - plugins.security.restapi.password_min_length=1 + - plugins.security.restapi.password_score_based_validation_strength=fair + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data1:/usr/share/opensearch/data + ports: + - "9200:9200" + expose: + - "9200" + networks: + - opensearch-net + opensearch-node2: + image: opensearchproject/security-benchmark:latest + platform: linux/amd64 + container_name: opensearch-node2 + environment: + - bootstrap.system_call_filter=false + - network.host=0.0.0.0 + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + - plugins.security.restapi.password_min_length=1 + - plugins.security.restapi.password_score_based_validation_strength=fair + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data2:/usr/share/opensearch/data + networks: + - opensearch-net + opensearch-node3: + image: opensearchproject/security-benchmark:latest + platform: linux/amd64 + container_name: opensearch-node3 + environment: + - bootstrap.system_call_filter=false + - network.host=0.0.0.0 + - cluster.name=opensearch-cluster + - node.name=opensearch-node3 + - discovery.seed_hosts=opensearch-node1,opensearch-node2,opensearch-node3 + - cluster.initial_cluster_manager_nodes=opensearch-node1,opensearch-node2,opensearch-node3 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_INITIAL_ADMIN_PASSWORD} + - plugins.security.restapi.password_min_length=1 + - plugins.security.restapi.password_score_based_validation_strength=fair + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + volumes: + - opensearch-data3:/usr/share/opensearch/data + networks: + - opensearch-net + +volumes: + opensearch-data1: + opensearch-data2: + opensearch-data3: + +networks: + opensearch-net: diff --git a/benchmark/docker/benchmark.dockerfile b/benchmark/docker/benchmark.dockerfile new file mode 100644 index 0000000000..70bae04db0 --- /dev/null +++ b/benchmark/docker/benchmark.dockerfile @@ -0,0 +1,59 @@ +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS linux_stage_0 + +ARG UID=1000 +ARG GID=1000 +ARG VERSION +ARG TEMP_DIR=/tmp/opensearch +ARG OPENSEARCH_HOME=/usr/share/opensearch +ARG OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config +ARG SECURITY_PLUGIN_DIR=$OPENSEARCH_HOME/plugins/opensearch-security +ARG PERFORMANCE_ANALYZER_PLUGIN_CONFIG_DIR=$OPENSEARCH_PATH_CONF/opensearch-performance-analyzer + +RUN dnf update --releasever=latest -y && dnf install -y tar gzip shadow-utils which wget && dnf clean all + +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \ + mkdir $TEMP_DIR + +COPY entrypoint.sh $OPENSEARCH_HOME/entrypoint.sh + +RUN ls -l $TEMP_DIR && \ + wget https://artifacts.opensearch.org/snapshots/core/opensearch/${VERSION}-SNAPSHOT/opensearch-min-${VERSION}-SNAPSHOT-linux-x64-latest.tar.gz && \ + tar -xzpf opensearch-*.tar.gz -C $OPENSEARCH_HOME --strip-components=1 && \ + mkdir -p $OPENSEARCH_HOME/data && chown -Rv $UID:$GID $OPENSEARCH_HOME/data && \ + chmod +x $OPENSEARCH_HOME/entrypoint.sh && \ + ls -l $OPENSEARCH_HOME + + + +FROM public.ecr.aws/amazonlinux/amazonlinux:2023 + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_HOME=/usr/share/opensearch + +RUN dnf update --releasever=latest -y && dnf install -y tar gzip shadow-utils which && dnf clean all + +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch + +COPY --from=linux_stage_0 --chown=$UID:$GID $OPENSEARCH_HOME $OPENSEARCH_HOME +WORKDIR $OPENSEARCH_HOME + +RUN echo "export JAVA_HOME=$OPENSEARCH_HOME/jdk" >> /etc/profile.d/java_home.sh && \ + echo "export PATH=\$PATH:\$JAVA_HOME/bin" >> /etc/profile.d/java_home.sh && \ + ls -l $OPENSEARCH_HOME + +ENV JAVA_HOME=$OPENSEARCH_HOME/jdk +ENV PATH=$PATH:$JAVA_HOME/bin:$OPENSEARCH_HOME/bin + +COPY opensearch-security-*.zip $TEMP_DIR/opensearch-security.zip + +USER $UID + +RUN /bin/bash -c "yes | ./bin/opensearch-plugin install file:$TEMP_DIR/opensearch-security.zip" + +EXPOSE 9200 9300 + +ENTRYPOINT ["./entrypoint.sh"] +CMD ["opensearch"] diff --git a/benchmark/docker/entrypoint.sh b/benchmark/docker/entrypoint.sh new file mode 100644 index 0000000000..b88558474f --- /dev/null +++ b/benchmark/docker/entrypoint.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +export OPENSEARCH_HOME=/usr/share/opensearch + +chmod +x "$OPENSEARCH_HOME"/plugins/opensearch-security/tools/install_demo_configuration.sh +bash "$OPENSEARCH_HOME"/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s || exit 1 + +function runOpensearch { + umask 0002 + + if [[ "$(id -u)" == "0" ]]; then + echo "OpenSearch cannot run as root. Please start your container as another user." + exit 1 + fi + + opensearch_opts=() + while IFS='=' read -r envvar_key envvar_value + do + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then + if [[ ! -z $envvar_value ]]; then + opensearch_opt="-E${envvar_key}=${envvar_value}" + opensearch_opts+=("${opensearch_opt}") + fi + fi + done < <(env) + + "$@" "${opensearch_opts[@]}" +} + +if [ $# -eq 0 ] || [ "${1:0:1}" = '-' ]; then + set -- opensearch "$@" +fi + +if [ "$1" = "opensearch" ]; then + runOpensearch "$@" +else + exec "$@" +fi \ No newline at end of file diff --git a/benchmark/result_rewriter.py b/benchmark/result_rewriter.py new file mode 100644 index 0000000000..f2c5d6742f --- /dev/null +++ b/benchmark/result_rewriter.py @@ -0,0 +1,53 @@ +import json +import csv +import sys + + +def create_entry(name, value, unit): + entry = { + "name": name, + "value": value, + "unit": unit + } + return entry + + +def rewrite_csv(src_file): + try: + result = [] + with open(src_file, 'r', newline='') as file: + reader = csv.reader(file) + next(reader) + for row in reader: + if row[1] == "" or row[1] is None: + continue + else: + name = f"{row[0]} {row[1]}" + if name.startswith("Min") or name.startswith("Max") or name.startswith("Median") or not name.startswith("50th"): + continue + unit = row[3] + value = float(row[2]) + if unit == "ops/s": + value = 1 / value + unit = "s/ops" + elif unit == "docs/s": + value = 1 / value + unit = "s/docs" + entry = create_entry(name, value, unit) + result.append(entry) + return result + except Exception as e: + print(f"Failed to rewrite benchmark results: {e} {row}") + sys.exit(1) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Wrong number of arguments") + sys.exit(1) + else: + src = sys.argv[1] + dest = sys.argv[2] + json_res = rewrite_csv(src) + with open(dest, 'w') as f: + json.dump(json_res, f, indent=4)