Add OpenSearch implementation

.ci/opensearch/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker:stable
+
+RUN apk add --update bash
+
+COPY run-opensearch.sh /run-opensearch.sh
+
+ENTRYPOINT ["/run-opensearch.sh"]

.ci/opensearch/action.yml

@@ -0,0 +1,33 @@
+name: 'Run OpenSearch'
+description: 'This action spins up an Opensearch instance that can be accessed and used in your subsequent steps.'
+
+inputs:
+  opensearch-version:
+    description: 'The version of the OpenSearch you want to run'
+    required: true
+  security-enabled:
+    description: 'Enable or disable HTTPS, enabled by default'
+    default: 'false'
+    required: false
+  nodes:
+    description: 'Number of nodes in the cluster'
+    required: false
+    default: 1
+  port:
+    description: 'Port where you want to run OpenSearch'
+    required: false
+    default: 9200
+  opensearch-initial-admin-password:
+    description: 'The password for the user admin in your cluster'
+    required: false
+    default: 'myStrongPassword123!'
+
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  env:
+    OPENSEARCH_VERSION: ${{ inputs.opensearch-version }}
+    NODES: ${{ inputs.nodes }}
+    PORT: ${{ inputs.port }}
+    SECURITY_ENABLED: ${{ inputs.security-enabled }}
+    OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${{ inputs.opensearch-initial-admin-password }}

.ci/opensearch/functions/imports.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+# From https://github.com/opensearch-project/opensearch-py/blob/main/.ci/functions/imports.sh
+
+source ./.ci/opensearch/functions/wait-for-container.sh

.ci/opensearch/functions/wait-for-container.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+# From https://github.com/opensearch-project/opensearch-py/blob/main/.ci/functions/wait-for-container.sh
+
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 4;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1}\033[0m"
+    return 0
+  fi
+}

.ci/opensearch/run-opensearch.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+source ./.ci/opensearch/functions/imports.sh
+set -euxo pipefail
+
+if [[ -z $OPENSEARCH_VERSION ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [OPENSEARCH_VERSION] not set\033[0m"
+  exit 1
+fi
+
+OPENSEARCH_REQUIRED_VERSION="latest"
+# Starting in 2.12.0, security demo configuration script requires an initial admin password
+if [ "$OPENSEARCH_VERSION" != "$OPENSEARCH_REQUIRED_VERSION" ]; then
+  OPENSEARCH_INITIAL_ADMIN_PASSWORD="admin"
+fi
+
+for (( node=1; node<=${NODES-1}; node++ ))
+do
+  port=$((PORT + $node - 1))
+
+  if [[ "$SECURITY_ENABLED" == "true" ]]; then
+    healthcmd="curl -vvv -s --insecure -u admin:$OPENSEARCH_INITIAL_ADMIN_PASSWORD --fail https://localhost:$port/_cluster/health || exit 1"
+    security=($(cat <<-END
+
+END
+  ))
+  elif [[ "$SECURITY_ENABLED" == "false" ]]; then
+    healthcmd="curl -vvv -s --fail http://localhost:$port/_cluster/health || exit 1"
+    security=($(cat <<-END
+      --env plugins.security.disabled=true
+END
+  ))
+  fi
+
+  docker run \
+    --rm \
+    --detach \
+    --name="os${node}" \
+    --env "cluster.name=docker-opensearch" \
+    --env "http.port=${port}" \
+    --env discovery.type=single-node \
+    --env bootstrap.memory_lock=true \
+    --env "OPENSEARCH_JAVA_OPTS=-Xms4g -Xmx4g" \
+    --env OPENSEARCH_INITIAL_ADMIN_PASSWORD=$OPENSEARCH_INITIAL_ADMIN_PASSWORD \
+    "${security[@]}" \
+    --publish "${port}:${port}" \
+    --ulimit nofile=65536:65536 \
+    --ulimit memlock=-1:-1 \
+    --health-cmd="$(echo $healthcmd)" \
+    --health-interval=2s \
+    --health-retries=20 \
+    --health-timeout=2s \
+    opensearchproject/opensearch:${OPENSEARCH_VERSION}
+
+  if wait_for_container "os$node"; then
+    echo -e "\033[32;1mSUCCESS:\033[0m OpenSearch up and running\033[0m"
+  fi
+done

.ci/opensearch/test.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/functions/imports.sh
+set -euxo pipefail
+
+echo $script_path/functions/imports.sh

.flake8

@@ -0,0 +1,16 @@
+[flake8]
+select = B,BLK,C,E,F,I,S,W
+max-complexity = 30
+max-line-length = 88
+ignore = E203,W503 # ignore conflicts with black
+application-import-names =  abcd,tests
+import-order-style = google
+exclude = 
+    abcd/backends/atoms_pymongo.py,
+    abcd/frontends,
+    abcd/model.py,
+    abcd/parsers/queries_new.py,
+    abcd/parsers/queries.py,
+    abcd/parsers/extras.py,
+    abcd/server,
+    tests/__init__.py

.github/workflows/ci.yml

@@ -7,12 +7,29 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+        opensearch: ['1.0.1', '2.0.1', 'latest']
+        security-enabled: ["true", "false"]
 
     steps:
       - uses: actions/checkout@v4
 
+      - name: Configure sysctl limits
+        run: |
+          sudo swapoff -a
+          sudo sysctl -w vm.swappiness=1
+          sudo sysctl -w fs.file-max=262144
+          sudo sysctl -w vm.max_map_count=262144
+
+      - name: Start OpenSearch
+        uses: ./.ci/opensearch
+        with:
+          port: 9250
+          opensearch-version: ${{ matrix.opensearch }}
+          security-enabled: ${{ matrix.security-enabled }}
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
@@ -29,6 +46,10 @@ jobs:
       - name: Run unit tests
         run: |
           poetry run pytest --cov=abcd --cov-report xml --cov-report term:skip-covered
+        env:
+          port: 9250
+          security_enabled: ${{ matrix.security-enabled }}
+          opensearch-version: ${{ matrix.opensearch }}
 
       - name: Upload coverage reports to Codecov
         uses: codecov/[email protected]