[Backport 2.x] Add support for API versioning and fail open in weighted shard routing

.ci/bwcVersions

@@ -38,4 +38,24 @@ BWC_VERSION:
   - "1.2.5"
   - "1.3.0"
   - "1.3.1"
-  - "1.4.0"
+  - "1.3.2"
+  - "1.3.3"
+  - "1.3.4"
+  - "1.3.5"
+  - "1.3.6"
+  - "1.3.7"
+  - "1.3.8"
+  - "2.0.0"
+  - "2.0.1"
+  - "2.0.2"
+  - "2.1.0"
+  - "2.1.1"
+  - "2.2.0"
+  - "2.2.1"
+  - "2.2.2"
+  - "2.3.0"
+  - "2.3.1"
+  - "2.4.0"
+  - "2.4.1"
+  - "2.4.2"
+  - "2.5.0"

.ci/dockerOnLinuxExclusions

@@ -12,4 +12,4 @@ ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
-sles-15.2
+sles-15.2

.ci/documentation/issue.md

@@ -0,0 +1,11 @@
+**Is your feature request related to a problem?**
+A new feature has been added.
+
+**What solution would you like?**
+Document the usage of the new feature.
+
+**What alternatives have you considered?**
+N/A
+
+**Do you have any additional context?**
+See please 

.github/CODEOWNERS

@@ -1,3 +1 @@
-# This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/opensearch-core
-
+*   @reta @anasalkouz @andrross @reta @Bukhtawar @CEHENKLE @dblock @gbbafna @setiah @kartg @kotwanikunal @mch2 @nknize @owaiskazi19 @adnapibar @Rishikesh1159 @ryanbogan @saratvemulapalli @shwetathareja @dreamer-89 @tlfeng @VachaShah @xuezhou25

.github/pull_request_template.md

@@ -1,15 +1,16 @@
 ### Description
 [Describe what this change achieves]
- 
+
 ### Issues Resolved
 [List any issues this PR will resolve]
- 
+
 ### Check List
 - [ ] New functionality includes testing.
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
-- [ ] Commits are signed per the DCO using --signoff 
+- [ ] Commits are signed per the DCO using --signoff
+- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/auto-release.yml

@@ -0,0 +1,29 @@
+name: Releases
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: actions/checkout@v2
+      - uses: ncipollo/release-action@v1
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          bodyFile: release-notes/opensearch.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/backport.yml

@@ -22,7 +22,7 @@ jobs:
           installation_id: 22958780
 
       - name: Backport
-        uses: VachaShah/backport@v1.1.4
+        uses: VachaShah/backport@v2.1.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>

.github/workflows/changelog_verifier.yml

@@ -0,0 +1,18 @@
+name: "Changelog Verifier"
+on:
+  pull_request:
+    types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request
+  verify-changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: dangoslen/changelog-enforcer@v3
+        with:
+          skipLabels: "autocut, skip-changelog"

.github/workflows/code-hygiene.yml

@@ -0,0 +1,14 @@
+name: Code Hygiene
+
+on: [pull_request]
+
+jobs:
+  linelint:
+    runs-on: ubuntu-latest
+    name: Check if all files end in newline
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Linelint
+        uses: fernandrone/[email protected]

.github/workflows/create-documentation-issue.yml

@@ -0,0 +1,41 @@
+name: Create Documentation Issue
+on:
+  pull_request:
+    types:
+      - labeled
+env:
+  PR_NUMBER: ${{ github.event.number }}
+
+jobs:
+  create-issue:
+    if: ${{ github.event.label.name == 'needs-documentation' }}
+    runs-on: ubuntu-latest
+    name: Create Documentation Issue
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Edit the issue template
+        run: |
+          echo "https://github.com/opensearch-project/OpenSearch/pull/${{ env.PR_NUMBER }}." >> ./ci/documentation/issue.md
+
+      - name: Create Issue From File
+        id: create-issue
+        uses: peter-evans/create-issue-from-file@v4
+        with:
+          title: Add documentation related to new feature
+          content-filepath: ./ci/documentation/issue.md
+          labels: documentation
+          repository: opensearch-project/documentation-website
+          token: ${{ steps.github_app_token.outputs.token }}
+
+      - name: Print Issue
+        run: echo Created related documentation issue ${{ steps.create-issue.outputs.issue-number }}

.github/workflows/delete_backport_branch.yml

@@ -12,4 +12,4 @@ jobs:
       - name: Delete merged branch
         uses: SvanBoxel/delete-merged-branch@main
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/dependabot_pr.yml

@@ -47,3 +47,17 @@ jobs:
           commit_user_name: dependabot[bot]
           commit_user_email: [email protected]
           commit_options: '--signoff'
+
+      - name: Update the changelog
+        uses: dangoslen/dependabot-changelog-helper@v1
+        with:
+          version: 'Unreleased'
+
+      - name: Commit the changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update changelog"
+          branch: ${{ github.head_ref }}
+          commit_user_name: dependabot[bot]
+          commit_user_email: [email protected]
+          commit_options: '--signoff'

.github/workflows/gradle-check.yml

@@ -0,0 +1,118 @@
+name: Gradle Check (Jenkins)
+on:
+  push:
+    branches-ignore:
+      - 'backport/**'
+      - 'create-pull-request/**'
+      - 'dependabot/**'
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  gradle-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 130
+    steps:
+      - name: Checkout OpenSearch repo
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Setup environment variables (PR)
+        if: github.event_name == 'pull_request_target'
+        run: |
+            echo "pr_from_sha=$(jq --raw-output .pull_request.head.sha $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_from_clone_url=$(jq --raw-output .pull_request.head.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_to_clone_url=$(jq --raw-output .pull_request.base.repo.clone_url $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_title=$(jq --raw-output .pull_request.title $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+            echo "pr_number=$(jq --raw-output .pull_request.number $GITHUB_EVENT_PATH)" >> $GITHUB_ENV
+
+      - name: Setup environment variables (Push)
+        if: github.event_name == 'push'
+        run: |
+            repo_url="https://github.com/opensearch-project/OpenSearch"
+            ref_id=$(git rev-parse HEAD)
+            branch_name=$(git rev-parse --abbrev-ref HEAD)
+            echo "pr_from_sha=$ref_id" >> $GITHUB_ENV
+            echo "pr_from_clone_url=$repo_url" >> $GITHUB_ENV
+            echo "pr_to_clone_url=$repo_url" >> $GITHUB_ENV
+            echo "pr_title=Push trigger $branch_name $ref_id $repo_url" >> $GITHUB_ENV
+            echo "pr_number=Null" >> $GITHUB_ENV
+
+      - name: Checkout opensearch-build repo
+        uses: actions/checkout@v2
+        with:
+          repository: opensearch-project/opensearch-build
+          ref: main
+          path: opensearch-build
+
+      - name: Trigger jenkins workflow to run gradle check
+        run: |
+            set -e
+            set -o pipefail
+            bash opensearch-build/scripts/gradle/gradle-check.sh ${{ secrets.JENKINS_GRADLE_CHECK_GENERIC_WEBHOOK_TOKEN }} | tee -a gradle-check.log
+
+      - name: Setup Result Status
+        if: always()
+        run: |
+            WORKFLOW_URL=`cat gradle-check.log | grep 'WORKFLOW_URL' | awk '{print $2}'`
+            RESULT=`cat gradle-check.log | grep 'Result:' | awk '{print $2}'`
+            echo "workflow_url=$WORKFLOW_URL" >> $GITHUB_ENV
+            echo "result=$RESULT" >> $GITHUB_ENV
+
+      - name: Upload Coverage Report
+        if: success()
+        uses: codecov/codecov-action@v2
+        with:
+          files: ./codeCoverage.xml
+
+      - name: Create Comment Success
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+              ### Gradle Check (Jenkins) Run Completed with:
+              * **RESULT:** ${{ env.result }} :white_check_mark:
+              * **URL:** ${{ env.workflow_url }}
+              * **CommitID:** ${{ env.pr_from_sha }}
+
+      - name: Extract Test Failure
+        if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
+        run: |
+            TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
+            if [[ "$TEST_FAILURES" != "" ]]
+            then
+              echo "test_failures<<EOF" >> $GITHUB_ENV
+              echo "" >> $GITHUB_ENV
+              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "$TEST_FAILURES" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "EOF" >> $GITHUB_ENV
+            fi
+
+      - name: Create Comment Flaky
+        if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+            ### Gradle Check (Jenkins) Run Completed with:
+            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
+            * **URL:** ${{ env.workflow_url }}
+            * **CommitID:** ${{ env.pr_from_sha }}
+            Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
+
+      - name: Create Comment Failure
+        if: ${{ github.event_name == 'pull_request_target' && failure() }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          issue-number: ${{ env.pr_number }}
+          body: |
+              ### Gradle Check (Jenkins) Run Completed with:
+              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
+              * **URL:** ${{ env.workflow_url }}
+              * **CommitID:** ${{ env.pr_from_sha }}
+              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
+              Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?

.github/workflows/links.yml

@@ -16,4 +16,4 @@ jobs:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/lucene-snapshots.yml

@@ -0,0 +1,59 @@
+# This workflow will check out, build, and publish snapshots of lucene.
+
+name: OpenSearch Lucene snapshots
+
+on:
+  workflow_dispatch:
+    # Inputs the workflow accepts.
+    inputs:
+      ref:
+        description:
+        required: false
+        default: 'main'
+
+jobs:
+  publish-snapshots:
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'adopt'
+
+      - name: Checkout Lucene
+        uses: actions/checkout@v2
+        with:
+          repository: 'apache/lucene'
+          path: lucene
+          ref: ${{ github.event.inputs.ref }}
+
+      - name: Set hash
+        working-directory: ./lucene
+        run: |
+          echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
+        id: version
+
+      - name: Initialize gradle settings
+        working-directory: ./lucene
+        run: ./gradlew localSettings
+
+      - name: Publish Lucene to local maven repo.
+        working-directory: ./lucene
+        run: ./gradlew publishJarsPublicationToMavenLocal -Pversion.suffix=snapshot-${{ steps.version.outputs.REVISION }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.LUCENE_SNAPSHOTS_ROLE }}
+          aws-region: us-west-2
+
+      - name: Copy files to S3 with the aws CLI.
+        run: |
+          aws s3 cp ~/.m2/repository/org/apache/lucene/ s3://${{ secrets.LUCENE_SNAPSHOTS_BUCKET }}/snapshots/lucene/org/apache/lucene/  --recursive --no-progress