diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 74cf149f..00000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: CI - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - workflow_dispatch: - -jobs: - build-and-test: - strategy: - matrix: - include: - - platform: amd64 - runs-on: ubuntu-latest - docker-arch: "" - - platform: arm64 - runs-on: ubuntu-latest - docker-arch: "--platform linux/arm64" - runs-on: ${{ matrix.runs-on }} - - steps: - - uses: actions/checkout@v4 - - name: Set standalone env - uses: ./.github/actions/set_standalone_env - with: - build-type: github - - - name: Run Python tests - run: | - cd test - pytest test_vearch.py -x --log-cli-level=INFO - pytest test_document_* -k "not test_vearch_document_upsert_benchmark" -x --log-cli-level=INFO - pytest test_module_* -x --log-cli-level=INFO - - - name: Test Go SDK - run: | - cd sdk/go/test - go test -v - - - name: Build python sdk - run: | - cd sdk/python - python setup.py bdist_wheel - pip install dist/pyvearch* - - - name: Test Python SDK - run: | - cd sdk/python/test - pytest -x --log-cli-level=INFO - - - name: Test Benchmark Scripts - run: | - cd scripts/benchmarks - python restful.py - python pysdk.py \ No newline at end of file diff --git a/.github/workflows/CI_cluster.yml b/.github/workflows/CI_cluster.yml deleted file mode 100644 index 979d628d..00000000 --- a/.github/workflows/CI_cluster.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: CI Cluster - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - workflow_dispatch: - -jobs: - cluster-build-and-test: - strategy: - matrix: - include: - - platform: amd64 - runs-on: ubuntu-latest - docker-arch: "" - - platform: arm64 - runs-on: ubuntu-latest - docker-arch: "--platform linux/arm64" - runs-on: ${{ matrix.runs-on }} - - steps: - - uses: actions/checkout@v4 - - name: Set cluster env - uses: ./.github/actions/set_cluster_env - with: - build-type: github - - - name: Create Minio bucket - run: | - mkdir test/oss_data - docker run -d --name minio -p 10000:9000 --network vearch_network_cluster minio/minio server test/oss_data - wget -q https://dl.min.io/client/mc/release/linux-amd64/mc - chmod +x mc - ./mc alias set myminio http://127.0.0.1:10000 minioadmin minioadmin - ./mc mb myminio/test - - - name: Run Python tests - run: | - cd test - pytest test_vearch.py -x --log-cli-level=INFO - pytest test_document_* -k "not test_vearch_document_upsert_benchmark" -x --log-cli-level=INFO - pytest test_module_* -x --log-cli-level=INFO - - - name: Test Go SDK - run: | - cd sdk/go/test - go test -v - - - name: Build python sdk - run: | - cd sdk/python - python setup.py bdist_wheel - pip install dist/pyvearch* - - - name: Test Python SDK - run: | - cd sdk/python/test - pytest -x --log-cli-level=INFO - - - name: Install Docker Compose - run: | - sudo curl -L "https://github.com/docker/compose/releases/download/$(curl -s https://api.github.com/repos/docker/compose/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")')/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - - - name: Run tests for master down and restart - run: | - cd cloud - docker-compose stop master1 - cd ../test - pytest test_vearch.py -x -k "test_vearch_basic_usage" --log-cli-level=INFO - pytest test_cluster_master.py -x -k "TestClusterMasterPrepare" --log-cli-level=INFO - pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO - # prepare for tow master down - pytest test_cluster_master.py -x -k "TestClusterMasterPrepare" --log-cli-level=INFO - cd ../cloud && docker-compose stop master2 - cd ../test - pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO - # all down - cd ../cloud && docker-compose stop master3 - sleep 120 - cd ../test - pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO - cd ../cloud - docker-compose start master1 && sleep 30 && docker ps - docker-compose start master2 && sleep 30 && docker ps - docker-compose start master3 && sleep 90 && docker ps - cd ../test - pytest test_cluster_master.py -x -k "TestClusterMasterOperate" --log-cli-level=INFO - pytest test_vearch.py -x -k "test_vearch_basic_usage" --log-cli-level=INFO - - - name: Run tests for restart ps - run: | - cd test - pytest test_cluster_ps.py -k "TestClusterPartitionServerAdd" -x --log-cli-level=INFO - cd ../cloud - docker-compose stop ps1 - sleep 30 - docker-compose start ps1 - cd ../test - pytest test_cluster_ps.py -x -k "TestClusterPartitionServerRecover" --log-cli-level=INFO - pytest test_cluster_ps.py -x -k "TestClusterPartitionServerCheckSpace" --log-cli-level=INFO - pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO - pytest test_cluster_ps.py -k "TestClusterPartitionChange" --log-cli-level=INFO - - - name: Run tests for faulty ps - run: | - cd test - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerCreateSpace" --log-cli-level=INFO - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO - cd ../cloud - docker-compose stop ps1 - cd ../test - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerPrepareData" --log-cli-level=INFO - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO - cd ../cloud && docker-compose stop ps2 - cd ../test - # TODO remove sleep - sleep 60 - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerSearch" --log-cli-level=INFO - pytest test_cluster_ps.py -x -k "TestClusterFaultyPartitionServerGetMetaData" --log-cli-level=INFO - cd ../cloud - docker-compose start ps1 - docker-compose start ps2 - sleep 60 - cd ../test - pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO - - - name: Run tests for incomplete shared - run: | - cd test - pytest test_cluster_ps.py -x -k "TestIncompleteShardPrepare" --log-cli-level=INFO - cd ../cloud - docker-compose stop ps1 - docker-compose stop ps2 - cd ../test - # TODO remove sleep - sleep 60 - pytest test_cluster_ps.py -x -k "TestIncompleteShardSearch" --log-cli-level=INFO - cd ../cloud - docker-compose stop ps3 - docker-compose start ps1 - docker-compose start ps2 - docker-compose start ps3 - sleep 60 - cd ../test - pytest test_cluster_ps.py -x -k "TestClusterPartitionServerDestroy" --log-cli-level=INFO - \ No newline at end of file diff --git a/.github/workflows/CI_document.yml b/.github/workflows/CI_document.yml deleted file mode 100644 index 2b0001b1..00000000 --- a/.github/workflows/CI_document.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: CI Document Tasks - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - workflow_dispatch: - -jobs: - document-tasks: - strategy: - matrix: - # avoid pytest -k mix with IVFFLAT, so flat as BFFLAT - index-type: ["BFFLAT", "IVFFLAT", "IVFPQ", "HNSW"] - platform: [amd64, arm64] - include: - - platform: amd64 - runs-on: ubuntu-latest - docker-arch: "" - - platform: arm64 - runs-on: ubuntu-latest - docker-arch: "--platform linux/arm64" - - runs-on: ${{ matrix.runs-on }} - - steps: - - uses: actions/checkout@v4 - - name: Set cluster env - uses: ./.github/actions/set_cluster_env - with: - build-type: github - - - name: Run test task for ${{ matrix.index-type }} with ${{ matrix.platform }} - run: | - cd test - pytest test_cluster_document.py -k "${{ matrix.index-type }} or test_vearch_index" -x --log-cli-level=INFO \ No newline at end of file diff --git a/.github/workflows/CI_index.yml b/.github/workflows/CI_index.yml deleted file mode 100644 index d418feeb..00000000 --- a/.github/workflows/CI_index.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: CI Index Tasks - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - workflow_dispatch: - -jobs: - index-tasks: - strategy: - matrix: - index-type: ["flat", "ivfflat", "ivfpq", "hnsw"] - platform: [amd64, arm64] - include: - - platform: amd64 - runs-on: ubuntu-latest - docker-arch: "" - - platform: arm64 - runs-on: ubuntu-latest - docker-arch: "--platform linux/arm64" - - runs-on: ${{ matrix.runs-on }} - - steps: - - uses: actions/checkout@v4 - - name: Set cluster env - uses: ./.github/actions/set_cluster_env - with: - build-type: github - - - name: Run test task for ${{ matrix.index-type }} with ${{ matrix.platform }} - run: | - cd test - if [[ "${{ matrix.index-type }}" == "ivfpq" && "${{ matrix.platform }}" == "arm64" ]]; then - pytest test_vector_index_${{ matrix.index-type }}.py -k "test_vearch_index_ivfpq_without_nsubvector or test_vearch_index_ivfpq_ip" -x --log-cli-level=INFO - else - pytest test_vector_index_${{ matrix.index-type }}.py -x --log-cli-level=INFO - fi \ No newline at end of file diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml deleted file mode 100644 index 91a3e4ea..00000000 --- a/.github/workflows/docker-image.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Docker Image - Release - -on: [workflow_dispatch] - -env: - REGISTRY_IMAGE: vearch/vearch - -jobs: - build: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY_IMAGE }} - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: compile - run: | - cd cloud - docker run --platform ${{ matrix.platform }} --privileged -i -v $(dirname "$PWD"):/vearch vearch/vearch-dev-env:latest /vearch/cloud/compile/compile.sh - cp -r ../build/bin compile/; cp -r ../build/lib compile/ - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v5 - with: - context: cloud - file: cloud/Dockerfile - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge: - runs-on: ubuntu-latest - needs: - - build - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digests-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Get the current tag - run: | - BUILD_VERSION=$(awk '/VEARCH_VERSION_MAJOR/ {major=$2} /VEARCH_VERSION_MINOR/ {minor=$2} /VEARCH_VERSION_PATCH/ {patch=$2} END {print major"."minor"."patch}' VERSION) - echo "BUILD_VERSION=$BUILD_VERSION" - echo "TAG=$BUILD_VERSION" >> $GITHUB_ENV - echo "version=$BUILD_VERSION" >> $GITHUB_ENV - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY_IMAGE }} - tags: | - type=raw,value=${{ env.TAG }} - type=raw,value=latest - - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) - - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/scripts/benchmarks/pysdk.py b/scripts/benchmarks/pysdk.py index 150e73c9..2562caf6 100644 --- a/scripts/benchmarks/pysdk.py +++ b/scripts/benchmarks/pysdk.py @@ -199,10 +199,11 @@ def process_upsert_data(items: tuple): param_dict["field_string"] = str(param_dict["field_int"]) data.append(param_dict) rs = vc.upsert(args.db, args.space, data) + logger.info(f"code:{rs.code}, total:{rs.total}") if rs.code != 0: - logger.error(rs.msg) + logger.info(rs.msg) if len(rs.get_document_ids()) != size: - logger.debug(rs.get_document_ids()) + logger.info(rs.get_document_ids()) assert len(rs.get_document_ids()) == size def upsert(args: argparse.Namespace, xb: np.ndarray = None):