Skip to content

Commit

Permalink
Merge branch 'opensearch-project:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
LDrago27 authored Mar 12, 2024
2 parents 4e9743b + 4b89ad0 commit 03483e4
Show file tree
Hide file tree
Showing 195 changed files with 12,801 additions and 631 deletions.
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @ananzh @kavilla @AMoo-Miki @ashwin-pc @joshuarrrr @abbyhu2000 @zengyan-amazon @kristenTian @zhongnansu @manasvinibs @ZilongX @Flyingliuhub @BSFishy @curq @bandinib-amzn @SuZhou-Joe
* @ananzh @kavilla @AMoo-Miki @ashwin-pc @joshuarrrr @abbyhu2000 @zengyan-amazon @kristenTian @zhongnansu @manasvinibs @ZilongX @Flyingliuhub @BSFishy @curq @bandinib-amzn @SuZhou-Joe @ruanyl @BionIT
26 changes: 19 additions & 7 deletions .github/workflows/build_and_test_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ name: Build and test
# trigger on every commit push and PR for all branches except pushes for backport branches
on:
push:
branches: ['**', '!backport/**']
branches: ['main', '[0-9]+\.x', '[0-9]+\.[0-9]+'] # Run the functional test on push for only release branches
paths-ignore:
- '**/*.md'
- 'docs/**'
Expand All @@ -25,7 +25,7 @@ env:
TEST_OPENSEARCH_TRANSPORT_PORT: 9403
TEST_OPENSEARCH_PORT: 9400
OSD_SNAPSHOT_SKIP_VERIFY_CHECKSUM: true
NODE_OPTIONS: "--max-old-space-size=6144 --dns-result-order=ipv4first"
NODE_OPTIONS: '--max-old-space-size=6144 --dns-result-order=ipv4first'

jobs:
build-lint-test:
Expand Down Expand Up @@ -53,7 +53,7 @@ jobs:
with:
minimum-size: 16GB
maximum-size: 64GB
disk-root: "C:"
disk-root: 'C:'

- name: Checkout code
uses: actions/checkout@v3
Expand Down Expand Up @@ -160,7 +160,7 @@ jobs:
with:
minimum-size: 16GB
maximum-size: 64GB
disk-root: "C:"
disk-root: 'C:'

- name: Checkout code
uses: actions/checkout@v3
Expand Down Expand Up @@ -254,7 +254,7 @@ jobs:
with:
minimum-size: 16GB
maximum-size: 64GB
disk-root: "C:"
disk-root: 'C:'

- name: Checkout code
uses: actions/checkout@v3
Expand Down Expand Up @@ -364,7 +364,7 @@ jobs:
with:
minimum-size: 16GB
maximum-size: 64GB
disk-root: "C:"
disk-root: 'C:'

- name: Checkout code
uses: actions/checkout@v3
Expand Down Expand Up @@ -441,7 +441,19 @@ jobs:
working-directory: ./artifacts
strategy:
matrix:
version: [osd-2.0.0, osd-2.1.0, osd-2.2.0, osd-2.3.0, osd-2.4.0, osd-2.5.0, osd-2.6.0, osd-2.7.0, osd-2.8.0, osd-2.9.0]
version:
[
osd-2.0.0,
osd-2.1.0,
osd-2.2.0,
osd-2.3.0,
osd-2.4.0,
osd-2.5.0,
osd-2.6.0,
osd-2.7.0,
osd-2.8.0,
osd-2.9.0,
]
steps:
- name: Checkout code
uses: actions/checkout@v3
Expand Down
83 changes: 71 additions & 12 deletions .github/workflows/cypress_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:
strategy:
fail-fast: false
matrix:
group: [1, 2, 3, 4, 5, 6, 7, 8, 9]
group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
container:
image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
options: --user 1001
Expand Down Expand Up @@ -101,7 +101,7 @@ jobs:
- name: Build plugins
run: node scripts/build_opensearch_dashboards_platform_plugins --no-examples --workers 12

- name: Checkout
- name: Checkout FT repo
uses: actions/checkout@v2
with:
path: ${{ env.FTR_PATH }}
Expand All @@ -113,8 +113,9 @@ jobs:
run: |
echo "SPEC=${{ inputs.specs }}" >> $GITHUB_ENV
# Setup spec files for existing Functional Test repo cypress tests
- name: Setup spec files
if: ${{ inputs.specs == '' }}
if: ${{ inputs.specs == '' && matrix.group < 10 }}
working-directory: ${{ env.FTR_PATH }}
shell: bash
run: |
Expand All @@ -126,11 +127,13 @@ jobs:
echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
- name: Get Cypress version
if: ${{ matrix.group < 10 }}
id: cypress_version
run: |
echo "name=cypress_version::$(cat ./${{ env.FTR_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
- name: Cache Cypress
if: ${{ matrix.group < 10 }}
id: cache-cypress
uses: actions/cache@v1
with:
Expand All @@ -141,36 +144,92 @@ jobs:
- run: npx cypress cache list
- run: npx cypress cache path

- name: Run tests
# Setup spec files for Dashboards in-house cypress tests
- name: Setup spec files for Dashboards tests
if: ${{ inputs.specs == '' && matrix.group > 9 }}
shell: bash
run: |
IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.group }})
DASHBOARDS_SPEC=''
for i in "${SPEC_ARRAY[@]}"; do
DASHBOARDS_SPEC+="cypress/integration/core_opensearch_dashboards/${i},"
done
echo "DASHBOARDS_SPEC=${DASHBOARDS_SPEC}" >> $GITHUB_ENV
# Run FT repo tests
- name: Run FT repo tests
if: ${{ matrix.group < 10 }}
uses: cypress-io/github-action@v2
with:
working-directory: ${{ env.FTR_PATH }}
start: ${{ env.OPENSEARCH_SNAPSHOT_CMD }}, ${{ env.START_CMD }}
wait-on: 'http://localhost:9200, http://localhost:5601'
command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.SPEC }}

# Clear Cypress Cache before running Dashboards tests
- name: Clear Cypress Cache
if: ${{ matrix.group > 9 }}
run: npx cypress cache clear

# Run Dashboards Cypress tests within the source repo
- name: Run Dashboards Cypress tests
if: ${{ matrix.group > 9 }}
uses: cypress-io/github-action@v6
with:
install-command: npx cypress install --force
start: node scripts/opensearch snapshot -E cluster.routing.allocation.disk.threshold_enabled=false, node scripts/opensearch_dashboards --dev --no-base-path --no-watch --savedObjects.maxImportPayloadBytes=10485760 --server.maxPayloadBytes=1759977 --logging.json=false --data.search.aggs.shardDelay.enabled=true
wait-on: 'http://localhost:9200, http://localhost:5601'
command: yarn cypress:run-without-security --browser ${{ env.CYPRESS_BROWSER }} --spec ${{ env.DASHBOARDS_SPEC }}

# Screenshots are only captured on failure, will change this once we do visual regression tests
- uses: actions/upload-artifact@v3
if: failure()
- name: Upload FT repo screenshots
uses: actions/upload-artifact@v3
if: failure() && (matrix.group < 10)
with:
name: ftr-cypress-screenshots
path: ${{ env.FTR_PATH }}/cypress/screenshots
retention-days: 1

- uses: actions/upload-artifact@v3
if: always()

- name: Upload FT repo videos
uses: actions/upload-artifact@v3
if: always() && (matrix.group < 10)
with:
name: ftr-cypress-videos
path: ${{ env.FTR_PATH }}/cypress/videos
retention-days: 1

- uses: actions/upload-artifact@v3
if: always()
- name: Upload FT repo results
uses: actions/upload-artifact@v3
if: always() && (matrix.group < 10)
with:
name: ftr-cypress-results
path: ${{ env.FTR_PATH }}/cypress/results
retention-days: 1

- name: Upload Dashboards screenshots
if: failure() && (matrix.group > 9)
uses: actions/upload-artifact@v3
with:
name: dashboards-cypress-screenshots
path: cypress/screenshots
retention-days: 1

- name: Upload Dashboards repo videos
uses: actions/upload-artifact@v3
if: always() && (matrix.group > 9)
with:
name: dashboards-cypress-videos
path: cypress/videos
retention-days: 1

- name: Upload Dashboards repo results
uses: actions/upload-artifact@v3
if: always() && (matrix.group > 9)
with:
name: dashboards-cypress-results
path: cypress/results
retention-days: 1

add-comment:
needs: [cypress-tests]
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.pr_number != '' }}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: Orchestrator cypress workflow
run-name: dashboards_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator.
run-name: release_cypress_workflow ${{ inputs.UNIQUE_ID != '' && inputs.UNIQUE_ID || '' }} # Unique id number appended to the workflow run-name to reference the run within the orchestrator.
# Trigger on dispatch event sent from FT repo orchestrator
on:
workflow_dispatch:
Expand Down Expand Up @@ -41,27 +41,38 @@ env:
TEST_REPO: ${{ inputs.test_repo != '' && inputs.test_repo || github.repository }}
TEST_BRANCH: "${{ inputs.test_branch != '' && inputs.test_branch || github.base_ref }}"
OSD_PATH: 'osd'
CYPRESS_BROWSER: 'chromium'
CYPRESS_BROWSER: 'electron'
JOB_ID: ${{ inputs.UNIQUE_ID}}
OPENSEARCH: ${{ inputs.OS_URL != '' && inputs.OS_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/linux/x64/tar/dist/opensearch/opensearch-$VERSION-linux-x64.tar.gz' }}
DASHBOARDS: ${{ inputs.OSD_URL != '' && inputs.OSD_URL || 'https://ci.opensearch.org/ci/dbc/distribution-build-opensearch-dashboards/$VERSION/latest/linux/x64/tar/dist/opensearch-dashboards/opensearch-dashboards-$VERSION-linux-x64.tar.gz' }}
OPENSEARCH_DIR: 'cypress/opensearch'
DASHBOARDS_DIR: 'cypress/opensearch-dashboards'
SECURITY_ENABLED: ${{ inputs.SECURITY_ENABLED != '' && inputs.SECURITY_ENABLED || 'false' }}
SPEC: 'cypress/integration/core_opensearch_dashboards/*'

jobs:
Get-CI-Image-Tag:
uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
with:
product: opensearch-dashboards

cypress-tests:
needs: Get-CI-Image-Tag
runs-on: ubuntu-latest
strategy:
matrix:
spec_group: [10, 11] # Add more spec files as needed
container:
image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
options: --user 1001
# using the same image which is used by opensearch-build team to build the OpenSearch Distribution
# this image tag is subject to change as more dependencies and updates will arrive over time
image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
# need to switch to root so that github actions can install runner binary on container without permission issues.
options: --user root
env:
# prevents extra Cypress installation progress messages
CI: 1
# avoid warnings like "tput: No value for $TERM and no -T specified"
TERM: xterm
name: Run cypress tests ${{ inputs.UNIQUE_ID}}
name: Run cypress tests (osd:ciGroup${{ matrix.spec_group }}) ${{ inputs.UNIQUE_ID}}
steps:
- name: Checkout code
uses: actions/checkout@v2
Expand All @@ -82,21 +93,24 @@ jobs:
npm i -g [email protected]
yarn config set network-timeout 1000000 -g
- name: Setup spec files
if: ${{ inputs.specs == '' }}
shell: bash
run: |
cd ${{ env.OSD_PATH }}
IFS="," read -a SPEC_ARRAY <<< $(yarn --silent osd:ciGroup${{ matrix.spec_group }})
FORMATTED_SPEC=''
for i in "${SPEC_ARRAY[@]}"; do
FORMATTED_SPEC+="cypress/integration/core_opensearch_dashboards/${i},"
done
SPEC=${FORMATTED_SPEC} # Set SPEC to FORMATTED_SPEC
echo "SPEC=${FORMATTED_SPEC}" >> $GITHUB_ENV
echo "SPEC found: $SPEC"
- name: Get Cypress version
id: cypress_version
run: |
echo "name=cypress_version::$(cat ./${{ env.OSD_PATH }}/package.json | jq '.devDependencies.cypress' | tr -d '"')" >> $GITHUB_OUTPUT
- name: Cache Cypress
id: cache-cypress
uses: actions/cache@v1
with:
path: ~/.cache/Cypress
key: cypress-cache-v2-${{ runner.os }}-${{ hashFiles('**/package.json') }}
env:
CYPRESS_INSTALL_BINARY: ${{ steps.cypress_version.outputs.cypress_version }}
- run: npx cypress cache list
- run: npx cypress cache path
- name: Get package version (Linux)
run: |
Expand All @@ -106,16 +120,18 @@ jobs:
- name: Run bootstrap
run: |
cd ${{ env.OSD_PATH }}
yarn osd bootstrap
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "source $NVM_DIR/nvm.sh && nvm use && node -v && yarn -v &&
yarn osd bootstrap"
- name: Download and extract Opensearch artifacts
- name: Download and extract OpenSearch artifacts
run: |
CWD=$(pwd)
mkdir -p $CWD/${{ env.OPENSEARCH_DIR }}
source ${{ env.OSD_PATH }}/scripts/common/utils.sh
open_artifact $CWD/${{ env.OPENSEARCH_DIR }} ${{ env.OPENSEARCH }}
- name: Download and extract Opensearch Dashboards artifacts
- name: Download and extract OpenSearch Dashboards artifacts
run: |
CWD=$(pwd)
mkdir -p $CWD/${{ env.DASHBOARDS_DIR }}
Expand All @@ -124,20 +140,20 @@ jobs:
- name: Run Cypress tests
run: |
source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh
run_dashboards_cypress_tests
chown -R 1000:1000 `pwd`
su `id -un 1000` -c "source ${{ env.OSD_PATH }}/scripts/cypress_tests.sh && run_dashboards_cypress_tests"
# Screenshots are only captured on failures
- uses: actions/upload-artifact@v3
if: failure()
with:
name: osd-cypress-screenshots
name: release-osd-cypress-screenshots
path: ${{ env.OSD_PATH }}/cypress/screenshots
retention-days: 1

- uses: actions/upload-artifact@v3
if: always()
with:
name: osd-cypress-videos
name: release-osd-cypress-videos
path: ${{ env.OSD_PATH }}/cypress/videos
retention-days: 1
Empty file added .nojekyll
Empty file.
Loading

0 comments on commit 03483e4

Please sign in to comment.