feat: update data types to int64_t for improved precision and add ToR… #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Cluster | |
| on: | |
| push: | |
| branches: [ "master" ] | |
| pull_request: | |
| branches: [ "master" ] | |
| workflow_dispatch: | |
| jobs: | |
| cluster-build-and-test: | |
| strategy: | |
| matrix: | |
| include: | |
| - platform: amd64 | |
| runs-on: ubuntu-latest | |
| docker-arch: "" | |
| - platform: arm64 | |
| runs-on: ubuntu-latest | |
| docker-arch: "--platform linux/arm64" | |
| runs-on: ${{ matrix.runs-on }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set cluster env | |
| uses: ./.github/actions/set_cluster_env | |
| with: | |
| build-type: github | |
| - name: Create Minio bucket | |
| run: | | |
| mkdir test/oss_data | |
| docker run -d --name minio -p 10000:9000 --network vearch_network_cluster minio/minio server test/oss_data | |
| wget -q https://dl.min.io/client/mc/release/linux-amd64/mc | |
| chmod +x mc | |
| ./mc alias set myminio http://127.0.0.1:10000 minioadmin minioadmin | |
| ./mc mb myminio/test | |
| - name: Run Python tests | |
| run: | | |
| cd test | |
| pytest test_vearch.py -x --log-cli-level=INFO | |
| pytest test_document_* -k "not test_vearch_document_upsert_benchmark" -x --log-cli-level=INFO | |
| pytest test_module_* -x --log-cli-level=INFO | |
| - name: Test Go SDK | |
| run: | | |
| cd sdk/go/test | |
| go test -v | |
| - name: Build python sdk | |
| run: | | |
| cd sdk/python | |
| python setup.py bdist_wheel | |
| pip install dist/pyvearch* | |
| - name: Test Python SDK | |
| run: | | |
| cd sdk/python/test | |
| pytest -x --log-cli-level=INFO | |
| - name: Install Docker Compose | |
| run: | | |
| sudo curl -L "https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose | |
| sudo chmod +x /usr/local/bin/docker-compose | |
| - name: Run tests for master down and restart | |
| run: | | |
| cd cloud | |
| docker-compose stop master1 | |
| cd ../test | |
| pytest test_vearch.py -x -k "test_vearch_basic_usage" --log-cli-level=INFO | |
| pytest test_cluster_master.py -x -k "TestClusterMasterPrepare" --log-cli-level=INFO | |
| pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO | |
| # prepare for 2 masters down | |
| pytest test_cluster_master.py -x -k "TestClusterMasterPrepare" --log-cli-level=INFO | |
| cd ../cloud && docker-compose stop master2 | |
| cd ../test | |
| pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO | |
| # all down | |
| cd ../cloud && docker-compose stop master3 | |
| sleep 120 | |
| cd ../test | |
| pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose start master1 && sleep 30 && docker ps | |
| docker-compose start master2 && sleep 30 && docker ps | |
| docker-compose start master3 && sleep 90 && docker ps | |
| cd ../test | |
| pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO | |
| pytest test_vearch.py -x -k "test_vearch_basic_usage" --log-cli-level=INFO | |
| - name: Run tests for restart ps | |
| run: | | |
| cd test | |
| pytest test_cluster_ps.py -k "TestClusterPartitionServerAdd" -x --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose stop ps1 | |
| sleep 30 | |
| docker-compose start ps1 | |
| cd ../test | |
| pytest test_cluster_ps.py -x -k "TestClusterPartitionServerRecover" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -x -k "TestClusterPartitionServerCheckSpace" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -k "TestClusterPartitionChange" --log-cli-level=INFO | |
| - name: Run tests for faulty ps | |
| run: | | |
| cd test | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerCreateSpace" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose stop ps1 | |
| cd ../test | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerPrepareData" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO | |
| cd ../cloud && docker-compose stop ps2 | |
| cd ../test | |
| # TODO remove sleep | |
| sleep 60 | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerSearch" --log-cli-level=INFO | |
| pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose start ps1 | |
| docker-compose start ps2 | |
| sleep 60 | |
| cd ../test | |
| pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO | |
| - name: Run tests for incomplete shared | |
| run: | | |
| cd test | |
| pytest test_cluster_ps.py -x -k "TestIncompleteShardPrepare" --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose stop ps1 | |
| docker-compose stop ps2 | |
| cd ../test | |
| # TODO remove sleep | |
| sleep 60 | |
| pytest test_cluster_ps.py -x -k "TestIncompleteShardSearch" --log-cli-level=INFO | |
| cd ../cloud | |
| docker-compose stop ps3 | |
| docker-compose start ps1 | |
| docker-compose start ps2 | |
| docker-compose start ps3 | |
| sleep 60 | |
| cd ../test | |
| pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO | |
| - name: Run tests for fail server | |
| run: | | |
| cd test | |
| pytest test_cluster_ps.py -x -k "TestFailServerPrepare" --log-cli-level=INFO | |
| sed -i 's/name = "vearch"/name = "test"/' ../cloud/config_cluster.toml | |
| cd .. && docker-compose -f cloud/docker-compose.yml restart ps1 | |
| docker-compose -f cloud/docker-compose.yml restart router1 | |
| sed -i 's/name = "test"/name = "vearch"/' cloud/config_cluster.toml | |
| sleep 60 | |
| status=$(curl -u root:secret http://127.0.0.1:8817/cluster/health | jq -r '.data[0].spaces[0].status') | |
| if [ "$status" != "yellow" ]; then | |
| echo "Error: Status is not yellow. Status is $status." | |
| exit 1 | |
| else | |
| echo "Status is yellow." | |
| fi | |
| router_status=$(curl -u root:secret http://172.16.238.14:9001 || true) | |
| if [ $? != 7 ]; then | |
| echo "Router status is not OK. Status is $router_status." | |
| else | |
| echo "Router is OK. Status is $router_status." | |
| exit 1 | |
| fi | |
| docker-compose -f cloud/docker-compose.yml up ps4 -d | |
| docker-compose -f cloud/docker-compose.yml restart router1 | |
| sleep 60 | |
| status=$(curl -u root:secret http://127.0.0.1:8817/cluster/health | jq -r '.data[0].spaces[0].status') | |
| if [ "$status" != "green" ]; then | |
| echo "Error: Status is not green. Status is $status." | |
| exit 1 | |
| else | |
| echo "Status is green." | |
| fi | |
| cd test && pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO | |
| cd .. && docker-compose -f cloud/docker-compose.yml down ps4 | |
| sleep 30 | |
| docker-compose -f cloud/docker-compose.yml restart ps1 | |
| sleep 90 && docker ps | |
| - name: Run tests for master replace member | |
| run: | | |
| cd test | |
| pytest test_cluster_master.py -x -k "TestClusterChangeMasterMemberPrePare" --log-cli-level=INFO | |
| docker-compose -f ../cloud/docker-compose.yml stop master1 | |
| pytest test_cluster_master.py -x -k "TestClusterMemberInfomation" --log-cli-level=INFO | |
| sed -i 's/m1/m4/' ../cloud/config_cluster.toml | |
| sed -i 's/vearch-master1/vearch-master4/' ../cloud/config_cluster.toml | |
| sed -i 's/cluster_state = "new"/cluster_state = "existing"/g' ../cloud/config_cluster.toml | |
| pytest test_cluster_master.py -x -k "TestClusterRemoveMasterMember" --log-cli-level=INFO | |
| pytest test_cluster_master.py -x -k "TestClusterMemberInfomation" --log-cli-level=INFO | |
| pytest test_cluster_master.py -x -k "TestClusterAddMasterMember" --log-cli-level=INFO | |
| docker-compose -f ../cloud/docker-compose.yml up master4 -d | |
| sleep 10 | |
| pytest test_cluster_master.py -x -k "TestClusterMemberInfomation" --log-cli-level=INFO | |
| pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO | |
| pytest test_vearch.py -x -k "test_vearch_basic_usage" --log-cli-level=INFO | |
| - name: Clean cluster | |
| run: | | |
| docker-compose -f cloud/docker-compose.yml --profile cluster down |