From 5d2cb57f35fbcfebc1d799d6467737908fb2fb1f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 17 Oct 2023 22:02:06 +0800 Subject: [PATCH] refactor(tests): Apply OPENDAL_TEST for behavior test (#3322) --- .env.example | 38 ----- .github/actions/test-core/action.yaml | 4 +- .github/services/fs/local-fs/action.yml | 1 - .github/services/s3/aws-s3/action.yml | 1 - .../workflows/service_test_atomicserver.yml | 6 +- .github/workflows/service_test_azblob.yml | 9 +- .github/workflows/service_test_azdls.yml | 22 ++- .github/workflows/service_test_cacache.yml | 4 +- .github/workflows/service_test_cos.yml | 5 +- .github/workflows/service_test_dashmap.yml | 4 +- .github/workflows/service_test_dropbox.yml | 5 +- .github/workflows/service_test_etcd.yml | 18 +-- .../workflows/service_test_foundationdb.yml | 7 +- .github/workflows/service_test_ftp.yml | 4 +- .github/workflows/service_test_gcs.yml | 10 +- .github/workflows/service_test_gdrive.yml | 5 +- .github/workflows/service_test_ghac.yml | 4 +- .github/workflows/service_test_hdfs.yml | 8 +- .github/workflows/service_test_http.yml | 8 +- .github/workflows/service_test_ipfs.yml | 8 +- .github/workflows/service_test_ipmfs.yml | 9 +- .github/workflows/service_test_libsql.yml | 8 +- .github/workflows/service_test_memcached.yml | 4 +- .github/workflows/service_test_memory.yml | 4 +- .github/workflows/service_test_mini_moka.yml | 4 +- .github/workflows/service_test_moka.yml | 4 +- .github/workflows/service_test_mysql.yml | 4 +- .github/workflows/service_test_obs.yml | 5 +- .github/workflows/service_test_oss.yml | 4 +- .github/workflows/service_test_persy.yml | 4 +- .github/workflows/service_test_postgresql.yml | 4 +- .github/workflows/service_test_redb.yml | 6 +- .github/workflows/service_test_redis.yml | 24 +-- .github/workflows/service_test_rocksdb.yml | 4 +- .github/workflows/service_test_s3.yml | 24 +-- .github/workflows/service_test_sftp.yml | 10 +- .github/workflows/service_test_sled.yml | 8 +- .github/workflows/service_test_sqlite.yml | 4 +- .github/workflows/service_test_supabase.yml | 5 +- .github/workflows/service_test_tikv.yml | 14 +- .github/workflows/service_test_wasabi.yml | 5 +- .github/workflows/service_test_webdav.yml | 24 +-- .github/workflows/service_test_webhdfs.yml | 8 +- core/src/services/azdls/writer.rs | 22 +++ core/src/types/operator/builder.rs | 14 +- core/src/types/scheme.rs | 2 +- core/tests/behavior/README.md | 22 +-- core/tests/behavior/main.rs | 140 +++--------------- core/tests/behavior/utils.rs | 35 ++--- 49 files changed, 251 insertions(+), 345 deletions(-) diff --git a/.env.example b/.env.example index d50da55072dc..f84d0bec6287 100644 --- a/.env.example +++ b/.env.example @@ -1,155 +1,122 @@ # memory -OPENDAL_MEMORY_TEST=on # fs -OPENDAL_FS_TEST=false OPENDAL_FS_ROOT=/path/to/dir OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir # cos -OPENDAL_COS_TEST=false OPENDAL_COS_BUCKET=opendal-testing-1318209832 OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com OPENDAL_COS_SECRET_ID= OPENDAL_COS_SECRET_KEY= # s3 -OPENDAL_S3_TEST=false OPENDAL_S3_BUCKET= OPENDAL_S3_ENDPOINT= OPENDAL_S3_REGION= OPENDAL_S3_ACCESS_KEY_ID= OPENDAL_S3_SECRET_ACCESS_KEY= # azblob -OPENDAL_AZBLOB_TEST=false OPENDAL_AZBLOB_ROOT=/path/to/dir OPENDAL_AZBLOB_CONTAINER= OPENDAL_AZBLOB_ENDPOINT= OPENDAL_AZBLOB_ACCOUNT_NAME= OPENDAL_AZBLOB_ACCOUNT_KEY= # hdfs -OPENDAL_HDFS_TEST=false OPENDAL_HDFS_ROOT=/path/to/dir OPENDAL_HDFS_NAME_NODE= # gcs -OPENDAL_GCS_TEST=false OPENDAL_GCS_ROOT=/path/to/dir OPENDAL_GCS_BUCKET= OPENDAL_GCS_CREDENTIAL= # obs -OPENDAL_OBS_TEST=false OPENDAL_OBS_BUCKET= OPENDAL_OBS_ENDPOINT= OPENDAL_OBS_ACCESS_KEY_ID= OPENDAL_OBS_SECRET_ACCESS_KEY= # oss -OPENDAL_OSS_TEST=false OPENDAL_OSS_BUCKET= OPENDAL_OSS_ENDPOINT= OPENDAL_OSS_ACCESS_KEY_ID= OPENDAL_OSS_ACCESS_KEY_SECRET= # ipmfs -OPENDAL_IPMFS_TEST=false OPENDAL_IPMFS_ROOT=/path/to/dir OPENDAL_IPMFS_ENDPOINT=http://localhost:5001 # ftp -OPENDAL_FTP_TEST=false OPENDAL_FTP_ENDPOINT=ftp:// OPENDAL_FTP_ROOT=/path/to/dir OPENDAL_FTP_USER= OPENDAL_FTP_PASSWORD= # ipfs -OPENDAL_IPFS_TEST=false OPENDAL_IPFS_ROOT=/ipfs/Qmxxxxxxxx OPENDAL_IPFS_ENDPOINT=http://localhost:8080 # redis -OPENDAL_REDIS_TEST=false OPENDAL_REDIS_ENDPOINT=tcp://127.0.0.1:6379 # OPENDAL_REDIS_CLUSTER_ENDPOINTS=rediss://127.0.0.1:6380,rediss://127.0.0.1:6381,rediss://127.0.0.1:6382,rediss://127.0.0.1:6383,rediss://127.0.0.1:6384,rediss://127.0.0.1:6385 OPENDAL_REDIS_ROOT=/ OPENDAL_REDIS_DB=0 # rocksdb -OPENDAL_ROCKSDB_TEST=false OPENDAL_ROCKSDB_DATADIR=/path/to/database OPENDAL_ROCKSDB_ROOT=/path/to/root # sftp -OPENDAL_SFTP_TEST=false OPENDAL_SFTP_ENDPOINT=ssh:// OPENDAL_SFTP_ROOT=/path/to/dir OPENDAL_SFTP_USER= OPENDAL_SFTP_KEY= OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY= # sled -OPENDAL_SLED_TEST=false OPENDAL_SLED_DATADIR=/path/to/database OPENDAL_SLED_TREE=sled-tree # mini-moka -OPENDAL_MINI_MOKA_TEST=false # moka -OPENDAL_MOKA_TEST=false # ghac -OPENDAL_GHAC_TEST=false # memcached -OPENDAL_MEMCACHED_TEST=false OPENDAL_MEMCACHED_ENDPOINT=tcp://127.0.0.1:11211 OPENDAL_MEMCACHED_ROOT=/ # webdav -OPENDAL_WEBDAV_TEST=false OPENDAL_WEBDAV_ROOT=/tmp/opendal/ OPENDAL_WEBDAV_ENDPOINT=http://127.0.0.1:8080 # webhfds -OPENDAL_WEBHDFS_TEST=false OPENDAL_WEBHDFS_ROOT=/tmp/opendal/ OPENDAL_WEBHDFS_ENDPOINT=http://127.0.0.1:9870 OPENDAL_WEBHDFS_DELEGATION= OPENDAL_WEBHDFS_DISABLE_LIST_BATCH=false # supbase -OPENDAL_SUPABASE_TEST=false OPENDAL_SUPABASE_BUCKET= OPENDAL_SUPABASE_ENDPOINT= OPENDAL_SUPABASE_KEY= # vercel artifacts -OPENDAL_VERCEL_ARTIFACTS_TEST=false OPENDAL_VERCEL_ARTIFACTS_ACCESS_TOKEN= # onedrive -OPENDAL_ONEDRIVE_TEST=false OPENDAL_ONEDRIVE_ACCESS_TOKEN= # wasabi -OPENDAL_WASABI_TEST=false OPENDAL_WASABI_REGION= OPENDAL_WASABI_BUCKET= OPENDAL_WASABI_ENDPOINT= OPENDAL_WASABI_ACCESS_KEY_ID= OPENDAL_WASABI_SECRET_ACCESS_KEY= # atomicserver -OPENDAL_ATOMICSERVER_TEST=false OPENDAL_ATOMICSERVER_ROOT=/path/to/dir OPENDAL_ATOMICSERVER_ENDPOINT=http://localhost:9883 OPENDAL_ATOMICSERVER_PRIVATE_KEY= OPENDAL_ATOMICSERVER_PUBLIC_KEY= OPENDAL_ATOMICSERVER_PARENT_RESOURCE_ID=http://localhost:9883 # foundationdb -OPENDAL_FOUNDATIONDB_TEST=false OPENDAL_FOUNDATIONDB_ROOT=/path/to/dir OPENDAL_FOUNDATIONDB_CONFIG_PATH=/tmp/opendal/foundationdb.conf # redb -OPENDAL_REDB_TEST=false OPENDAL_REDB_DATADIR=/tmp/redb OPENDAL_REDB_TABLE=redb-table # cacache -OPENDAL_CACACHE_TEST=false OPENDAL_CACACHE_DATADIR=/tmp/opendal/cacache/ # persy -OPENDAL_PERSY_TEST=false OPENDAL_PERSY_DATAFILE=/tmp/opendal/test.persy OPENDAL_PERSY_SEGMENT=data OPENDAL_PERSY_INDEX=index #dropbox -OPENDAL_DROPBOX_TEST=false OPENDAL_DROPBOX_ROOT=/tmp/opendal/ OPENDAL_DROPBOX_ACCESS_TOKEN= OPENDAL_DROPBOX_REFRESH_TOKEN= OPENDAL_DROPBOX_CLIENT_ID= OPENDAL_DROPBOX_CLIENT_SECRET= # etcd -OPENDAL_ETCD_TEST=false OPENDAL_ETCD_ENDPOINTS=127.0.0.1:2379 OPENDAL_ETCD_ROOT=/tmp/opendal/ OPENDAL_ETCD_USERNAME= @@ -158,14 +125,12 @@ OPENDAL_ETCD_CA_PATH= OPENDAL_ETCD_CERT_PATH= OPENDAL_ETCD_KEY_PATH= # google drive -OPENDAL_GDRIVE_TEST=false OPENDAL_GDRIVE_ROOT=/tmp/opendal/ OPENDAL_GDRIVE_ACCESS_TOKEN= OPENDAL_GDRIVE_REFRESH_TOKEN= OPENDAL_GDRIVE_CLIENT_ID= OPENDAL_GDRIVE_CLIENT_SECRET= # libsql -OPENDAL_LIBSQL_TEST=false OPENDAL_LIBSQL_ROOT=/tmp/opendal/ OPENDAL_LIBSQL_CONNECTION_STRING=https://example.com/db OPENDAL_LIBSQL_AUTH_TOKEN= @@ -173,13 +138,11 @@ OPENDAL_LIBSQL_TABLE=t_opendal OPENDAL_LIBSQL_KEY_FIELD=key OPENDAL_LIBSQL_VALUE_FIELD=val # sqlite -OPENDAL_SQLITE_TEST=on OPENDAL_SQLITE_CONNECTION_STRING=file:///tmp/opendal/test.db OPENDAL_SQLITE_TABLE=data OPENDAL_SQLITE_KEY_FIELD=key OPENDAL_SQLITE_VALUE_FIELD=data # d1 -OPENDAL_D1_TEST=false OPENDAL_D1_TOKEN= OPENDAL_D1_ACCOUNT_ID= OPENDAL_D1_DATABASE_ID= @@ -187,7 +150,6 @@ OPENDAL_D1_TABLE= OPENDAL_D1_KEY_FIELD= OPENDAL_D1_VALUE_FIELD= # azfile -OPENDAL_AZFILE_TEST=false OPENDAL_AZFILE_ENDPOINT= OPENDAL_AZFILE_ROOT=/tmp/opendal/ OPENDAL_AZFILE_ACCOUNT_NAME= diff --git a/.github/actions/test-core/action.yaml b/.github/actions/test-core/action.yaml index da5759fe14ed..7e29b6b411f0 100644 --- a/.github/actions/test-core/action.yaml +++ b/.github/actions/test-core/action.yaml @@ -44,7 +44,9 @@ runs: - name: Run Test Core shell: bash working-directory: core - run: cargo nextest run services_${{ inputs.service }} --archive-file=./core-test.tar.zst + run: cargo nextest run behavior --archive-file=./core-test.tar.zst + env: + OPENDAL_TEST: ${{ inputs.service }} EOF - name: Run id: run diff --git a/.github/services/fs/local-fs/action.yml b/.github/services/fs/local-fs/action.yml index 4175de1eb822..72d54608c061 100644 --- a/.github/services/fs/local-fs/action.yml +++ b/.github/services/fs/local-fs/action.yml @@ -24,5 +24,4 @@ runs: - name: Setup shell: bash run: | - echo "OPENDAL_FS_TEST=on" >> $GITHUB_ENV echo "OPENDAL_FS_ROOT=${{ runner.temp }}/" >> $GITHUB_ENV diff --git a/.github/services/s3/aws-s3/action.yml b/.github/services/s3/aws-s3/action.yml index 7591937d5a81..b560b443416a 100644 --- a/.github/services/s3/aws-s3/action.yml +++ b/.github/services/s3/aws-s3/action.yml @@ -30,7 +30,6 @@ runs: - name: Setup shell: bash run: | - echo "OPENDAL_S3_TEST=${{ fromJson(inputs.secrets).OPENDAL_S3_TEST }}" >> $GITHUB_ENV echo "OPENDAL_S3_ROOT=${{ fromJson(inputs.secrets).OPENDAL_S3_ROOT }}" >> $GITHUB_ENV echo "OPENDAL_S3_BUCKET=${{ fromJson(inputs.secrets).OPENDAL_S3_BUCKET }}" >> $GITHUB_ENV echo "OPENDAL_S3_ENDPOINT=${{ fromJson(inputs.secrets).OPENDAL_S3_ENDPOINT }}" >> $GITHUB_ENV diff --git a/.github/workflows/service_test_atomicserver.yml b/.github/workflows/service_test_atomicserver.yml index 87c738a4e0bf..fb2e78fce28a 100644 --- a/.github/workflows/service_test_atomicserver.yml +++ b/.github/workflows/service_test_atomicserver.yml @@ -46,10 +46,10 @@ jobs: shell: bash working-directory: core run: | - cargo test atomicserver --features services-atomicserver -j=1 + cargo test behavior --features services-atomicserver -j=1 env: - RUST_BACKTRACE: full - OPENDAL_ATOMICSERVER_TEST: off + # TODO: enable test when we are ready. + # OPENDAL_TEST: atomicserver OPENDAL_ATOMICSERVER_ROOT: TODO - ${{ secrets.OPENDAL_ATOMICSERVER_ROOT }} OPENDAL_ATOMICSERVER_ENDPOINT: TODO - ${{ secrets.OPENDAL_ATOMICSERVER_ENDPOINT }} OPENDAL_ATOMICSERVER_PRIVATE_KEY: TODO - ${{ secrets.OPENDAL_ATOMICSERVER_PRIVATE_KEY }} diff --git a/.github/workflows/service_test_azblob.yml b/.github/workflows/service_test_azblob.yml index f66108fbe196..102d36eb649b 100644 --- a/.github/workflows/service_test_azblob.yml +++ b/.github/workflows/service_test_azblob.yml @@ -63,11 +63,11 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run azblob + run: cargo nextest run behavior env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_AZBLOB_TEST: on + OPENDAL_TEST: azblob OPENDAL_AZBLOB_CONTAINER: test OPENDAL_AZBLOB_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1" OPENDAL_AZBLOB_ACCOUNT_NAME: devstoreaccount1 @@ -90,7 +90,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_AZBLOB_TEST: op://services/azblob/test OPENDAL_AZBLOB_CONTAINER: op://services/azblob/container OPENDAL_AZBLOB_ENDPOINT: op://services/azblob/endpoint OPENDAL_AZBLOB_ACCOUNT_NAME: op://services/azblob/account_name @@ -99,4 +98,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run azblob + run: cargo nextest run behavior + env: + OPENDAL_TEST: azblob diff --git a/.github/workflows/service_test_azdls.yml b/.github/workflows/service_test_azdls.yml index cb90c1ffac94..b16b5c734a16 100644 --- a/.github/workflows/service_test_azdls.yml +++ b/.github/workflows/service_test_azdls.yml @@ -46,13 +46,23 @@ jobs: uses: ./.github/actions/setup with: need-nextest: true + + - name: Load secret + id: op-load-secret + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + OPENDAL_AZDLS_FILESYSTEM: op://services/azdls/filesystem + OPENDAL_AZDLS_ENDPOINT: op://services/azdls/endpoint + OPENDAL_AZDLS_ACCOUNT_NAME: op://services/azdls/account_name + OPENDAL_AZDLS_ACCOUNT_KEY: op://services/azdls/account_key + - name: Test shell: bash working-directory: core - run: cargo nextest run azdls + run: cargo nextest run behavior env: - OPENDAL_AZDLS_TEST: ${{ secrets.OPENDAL_AZDLS_TEST }} - OPENDAL_AZDLS_FILESYSTEM: ${{ secrets.OPENDAL_AZDLS_FILESYSTEM }} - OPENDAL_AZDLS_ENDPOINT: ${{ secrets.OPENDAL_AZDLS_ENDPOINT }} - OPENDAL_AZDLS_ACCOUNT_NAME: ${{ secrets.OPENDAL_AZDLS_ACCOUNT_NAME }} - OPENDAL_AZDLS_ACCOUNT_KEY: ${{ secrets.OPENDAL_AZDLS_ACCOUNT_KEY }} + OPENDAL_TEST: azdls + OPENDAL_AZDLS_ROOT: test/ diff --git a/.github/workflows/service_test_cacache.yml b/.github/workflows/service_test_cacache.yml index 6de7046f5cc7..961756a45b7a 100644 --- a/.github/workflows/service_test_cacache.yml +++ b/.github/workflows/service_test_cacache.yml @@ -46,9 +46,9 @@ jobs: - name: Test cacache shell: bash working-directory: core - run: cargo test cacache --features services-cacache -j=1 + run: cargo test behavior --features services-cacache -j=1 env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_CACACHE_TEST: on + OPENDAL_TEST: cacache OPENDAL_CACACHE_DATADIR: /tmp/opendal/cacache/ diff --git a/.github/workflows/service_test_cos.yml b/.github/workflows/service_test_cos.yml index 794e4a0c7b38..11f0f85a4ba6 100644 --- a/.github/workflows/service_test_cos.yml +++ b/.github/workflows/service_test_cos.yml @@ -54,7 +54,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_COS_TEST: op://services/cos/test OPENDAL_COS_BUCKET: op://services/cos/bucket OPENDAL_COS_ENDPOINT: op://services/cos/endpoint OPENDAL_COS_SECRET_ID: op://services/cos/secret_id @@ -63,4 +62,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run cos + run: cargo nextest run behavior + env: + OPENDAL_TEST: cos diff --git a/.github/workflows/service_test_dashmap.yml b/.github/workflows/service_test_dashmap.yml index fdab6a4df4a9..4c4afca982fa 100644 --- a/.github/workflows/service_test_dashmap.yml +++ b/.github/workflows/service_test_dashmap.yml @@ -52,6 +52,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run dashmap --features services-dashmap + run: cargo nextest run behavior --features services-dashmap env: - OPENDAL_DASHMAP_TEST: on + OPENDAL_TEST: dashmap diff --git a/.github/workflows/service_test_dropbox.yml b/.github/workflows/service_test_dropbox.yml index 5a459568e3d8..3b81d33f50a4 100644 --- a/.github/workflows/service_test_dropbox.yml +++ b/.github/workflows/service_test_dropbox.yml @@ -52,7 +52,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_DROPBOX_TEST: op://services/dropbox/test OPENDAL_DROPBOX_ROOT: op://services/dropbox/root OPENDAL_DROPBOX_REFRESH_TOKEN: op://services/dropbox/refresh_token OPENDAL_DROPBOX_CLIENT_ID: op://services/dropbox/client_id @@ -63,4 +62,6 @@ jobs: working-directory: core # It's easily for dropbox to trigger too_many_write_operations error. # So we run tests one by one. - run: cargo test dropbox --features=services-dropbox -- --test-threads=1 + run: cargo test behavior --features=services-dropbox -- --test-threads=1 + env: + OPENDAL_TEST: dropbox diff --git a/.github/workflows/service_test_etcd.yml b/.github/workflows/service_test_etcd.yml index 2575e9a128f2..6380df620b5a 100644 --- a/.github/workflows/service_test_etcd.yml +++ b/.github/workflows/service_test_etcd.yml @@ -40,7 +40,7 @@ concurrency: jobs: etcd: runs-on: ubuntu-latest - + services: etcd: image: bitnami/etcd:latest @@ -64,15 +64,15 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run etcd --features services-etcd + run: cargo nextest run behavior --features services-etcd env: - OPENDAL_ETCD_TEST: on + OPENDAL_TEST: etcd OPENDAL_ETCD_ENDPOINTS: http://127.0.0.1:2379 OPENDAL_ETCD_ROOT: /tmp/opendal etcd-cluster: runs-on: ubuntu-latest - + services: etcd1: image: bitnami/etcd:latest @@ -133,9 +133,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run etcd --features services-etcd + run: cargo nextest run behavior --features services-etcd env: - OPENDAL_ETCD_TEST: on + OPENDAL_TEST: etcd OPENDAL_ETCD_ENDPOINTS: http://127.0.0.1:23790,http://127.0.0.1:23791,http://127.0.0.1:23792 OPENDAL_ETCD_ROOT: /tmp/opendal @@ -161,7 +161,7 @@ jobs: for no in `seq 0 2`; do \ num=$((no+1)) ip=$((no+2)) - + docker run -d \ -p 2379${no}:2379 \ -p 2380${no}:2380 \ @@ -204,9 +204,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run etcd --features services-etcd + run: cargo nextest run behavior --features services-etcd env: - OPENDAL_ETCD_TEST: on + OPENDAL_TEST: etcd OPENDAL_ETCD_ENDPOINTS: https://127.0.0.1:23790,https://127.0.0.1:23791,https://127.0.0.1:23792 OPENDAL_ETCD_ROOT: /tmp/opendal OPENDAL_ETCD_USERNAME: root diff --git a/.github/workflows/service_test_foundationdb.yml b/.github/workflows/service_test_foundationdb.yml index 2fa403473a5e..029e87edf9cb 100644 --- a/.github/workflows/service_test_foundationdb.yml +++ b/.github/workflows/service_test_foundationdb.yml @@ -51,10 +51,9 @@ jobs: working-directory: core run: | sudo service foundationdb start - cargo test foundationdb --features services-foundationdb -j=1 + cargo test behavior --features services-foundationdb -j=1 env: - RUST_BACKTRACE: full - RUST_LOG: debug - OPENDAL_FOUNDATIONDB_TEST: off + # TODO: enable test when we are ready. + # OPENDAL_TEST: foundationdb OPENDAL_FOUNDATIONDB_ROOT: /tmp/opendal OPENDAL_FOUNDATIONDB_CONFIG_PATH: /etc/foundationdb/foundationdb.conf diff --git a/.github/workflows/service_test_ftp.yml b/.github/workflows/service_test_ftp.yml index ef002ea9f2ef..6a9671ab91d2 100644 --- a/.github/workflows/service_test_ftp.yml +++ b/.github/workflows/service_test_ftp.yml @@ -82,9 +82,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run ftp --features services-ftp + run: cargo nextest run behavior --features services-ftp env: - OPENDAL_FTP_TEST: on + OPENDAL_TEST: ftp OPENDAL_FTP_ENDPOINT: ftp://127.0.0.1:2121 OPENDAL_FTP_ROOT: / OPENDAL_FTP_USER: admin diff --git a/.github/workflows/service_test_gcs.yml b/.github/workflows/service_test_gcs.yml index 5ad5504ab44d..38bbedf9684e 100644 --- a/.github/workflows/service_test_gcs.yml +++ b/.github/workflows/service_test_gcs.yml @@ -54,7 +54,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_GCS_TEST: op://services/gcs/test OPENDAL_GCS_ROOT: op://services/gcs/root OPENDAL_GCS_BUCKET: op://services/gcs/bucket OPENDAL_GCS_CREDENTIAL: op://services/gcs/credential @@ -62,7 +61,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run gcs + run: cargo nextest run behavior + env: + OPENDAL_TEST: gcs gcs-with-default-storage-class: runs-on: ubuntu-latest @@ -81,7 +82,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_GCS_TEST: op://services/gcs/test OPENDAL_GCS_ROOT: op://services/gcs/root OPENDAL_GCS_BUCKET: op://services/gcs/bucket OPENDAL_GCS_CREDENTIAL: op://services/gcs/credential @@ -90,4 +90,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run gcs + run: cargo nextest run behavior + env: + OPENDAL_TEST: gcs diff --git a/.github/workflows/service_test_gdrive.yml b/.github/workflows/service_test_gdrive.yml index 8f3044720bfe..a6c405563d73 100644 --- a/.github/workflows/service_test_gdrive.yml +++ b/.github/workflows/service_test_gdrive.yml @@ -52,7 +52,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_GDRIVE_TEST: op://services/gdrive/test OPENDAL_GDRIVE_ROOT: op://services/gdrive/root OPENDAL_GDRIVE_REFRESH_TOKEN: op://services/gdrive/refresh_token OPENDAL_GDRIVE_CLIENT_ID: op://services/gdrive/client_id @@ -61,4 +60,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo test gdrive --features=services-gdrive + run: cargo test behavior --features=services-gdrive + env: + OPENDAL_TEST: gdrive diff --git a/.github/workflows/service_test_ghac.yml b/.github/workflows/service_test_ghac.yml index b96c6fae4f00..788bc5d3ec78 100644 --- a/.github/workflows/service_test_ghac.yml +++ b/.github/workflows/service_test_ghac.yml @@ -57,8 +57,8 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run ghac + run: cargo nextest run behavior env: - OPENDAL_GHAC_TEST: ${{ secrets.OPENDAL_GHAC_TEST }} + OPENDAL_TEST: ghac OPENDAL_GHAC_ENABLE_CREATE_SIMULATION: true GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml index 112f00fa9257..d3284fbb54ed 100644 --- a/.github/workflows/service_test_hdfs.yml +++ b/.github/workflows/service_test_hdfs.yml @@ -64,10 +64,10 @@ jobs: export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native - cargo test services_hdfs --features services-hdfs + cargo test behavior --features services-hdfs env: HADOOP_HOME: "/home/runner/hadoop-3.3.5" - OPENDAL_HDFS_TEST: on + OPENDAL_TEST: hdfs OPENDAL_HDFS_ROOT: /tmp/opendal/ OPENDAL_HDFS_NAME_NODE: default @@ -122,9 +122,9 @@ jobs: export CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath --glob) export LD_LIBRARY_PATH=${{ env.JAVA_HOME }}/lib/server:${{ env.HADOOP_HOME }}/lib/native - cargo test services_hdfs --features services-hdfs + cargo test behavior --features services-hdfs env: HADOOP_HOME: "/home/runner/hadoop-3.2.4" - OPENDAL_HDFS_TEST: on + OPENDAL_TEST: hdfs OPENDAL_HDFS_ROOT: /tmp/opendal/ OPENDAL_HDFS_NAME_NODE: hdfs://localhost:8020 diff --git a/.github/workflows/service_test_http.yml b/.github/workflows/service_test_http.yml index ffec934d8893..e8e4607c78a8 100644 --- a/.github/workflows/service_test_http.yml +++ b/.github/workflows/service_test_http.yml @@ -60,9 +60,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run http + run: cargo nextest run behavior env: - OPENDAL_HTTP_TEST: on + OPENDAL_TEST: http OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080 OPENDAL_DISABLE_RANDOM_ROOT: true @@ -97,8 +97,8 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run http + run: cargo nextest run behavior env: - OPENDAL_HTTP_TEST: on + OPENDAL_TEST: http OPENDAL_HTTP_ENDPOINT: http://127.0.0.1:8080 OPENDAL_DISABLE_RANDOM_ROOT: true diff --git a/.github/workflows/service_test_ipfs.yml b/.github/workflows/service_test_ipfs.yml index 37c94f0509ac..486f6ce6ee64 100644 --- a/.github/workflows/service_test_ipfs.yml +++ b/.github/workflows/service_test_ipfs.yml @@ -58,9 +58,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run ipfs --features services-ipfs + run: cargo nextest run behavior --features services-ipfs env: - OPENDAL_IPFS_TEST: on + OPENDAL_TEST: ipfs OPENDAL_IPFS_ROOT: /ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/ OPENDAL_IPFS_ENDPOINT: "http://127.0.0.1:8080" OPENDAL_DISABLE_RANDOM_ROOT: true @@ -72,10 +72,10 @@ jobs: # - uses: actions/checkout@v4 # - name: Test # shell: bash -# run: cargo test ipfs --features services-ipfs -- --show-output +# run: cargo test behavior --features services-ipfs -- --show-output # env: # RUST_BACKTRACE: full # RUST_LOG: debug -# OPENDAL_IPFS_TEST: on +# OPENDAL_TEST: ipfs # OPENDAL_IPFS_ROOT: /ipfs/QmPpCt1aYGb9JWJRmXRUnmJtVgeFFTJGzWFYEEX7bo9zGJ/ # OPENDAL_IPFS_ENDPOINT: "https://ipfs.io" diff --git a/.github/workflows/service_test_ipmfs.yml b/.github/workflows/service_test_ipmfs.yml index 214e1b130be3..9f33330863c3 100644 --- a/.github/workflows/service_test_ipmfs.yml +++ b/.github/workflows/service_test_ipmfs.yml @@ -53,8 +53,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run ipmfs + run: cargo nextest run behavior env: - OPENDAL_IPFS_TEST: on - OPENDAL_IPFS_ROOT: /opendal/ - OPENDAL_IPFS_ENDPOINT: "http://127.0.0.1:5001" + # TODO: enable test when we are ready. + # OPENDAL_TEST: ipmfs + OPENDAL_IPMFS_ROOT: /opendal/ + OPENDAL_IPMFS_ENDPOINT: "http://127.0.0.1:5001" diff --git a/.github/workflows/service_test_libsql.yml b/.github/workflows/service_test_libsql.yml index 68ac2124c6ca..b61de44a783f 100644 --- a/.github/workflows/service_test_libsql.yml +++ b/.github/workflows/service_test_libsql.yml @@ -75,9 +75,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run libsql --features services-libsql + run: cargo nextest run behavior --features services-libsql env: - OPENDAL_LIBSQL_TEST: on + OPENDAL_TEST: libsql OPENDAL_LIBSQL_CONNECTION_STRING: http://127.0.0.1:8080 OPENDAL_LIBSQL_TABLE: data OPENDAL_LIBSQL_KEY_FIELD: key @@ -122,9 +122,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run libsql --features services-libsql + run: cargo nextest run behavior --features services-libsql env: - OPENDAL_LIBSQL_TEST: on + OPENDAL_TEST: libsql OPENDAL_LIBSQL_CONNECTION_STRING: http://127.0.0.1:8080 OPENDAL_LIBSQL_AUTH_TOKEN: eyJhbGciOiJFZERTQSIsInR5cCI6IkpXVCJ9.eyJleHAiOjc5ODg0ODM4Mjd9.MatB2aLnPFusagqH2RMoVExP37o2GFLmaJbmd52OdLtAehRNeqeJZPrefP1t2GBFidApUTLlaBRL6poKq_s3CQ OPENDAL_LIBSQL_TABLE: data diff --git a/.github/workflows/service_test_memcached.yml b/.github/workflows/service_test_memcached.yml index 3fdabbf2690c..785265059ac7 100644 --- a/.github/workflows/service_test_memcached.yml +++ b/.github/workflows/service_test_memcached.yml @@ -62,8 +62,8 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run memcached --features services-memcached + run: cargo nextest run behavior --features services-memcached env: - OPENDAL_MEMCACHED_TEST: on + OPENDAL_TEST: memcached OPENDAL_MEMCACHED_ENDPOINT: tcp://127.0.0.1:11211 OPENDAL_MEMCACHED_ROOT: / diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml index a955040b1038..f2fa0a11697d 100644 --- a/.github/workflows/service_test_memory.yml +++ b/.github/workflows/service_test_memory.yml @@ -48,9 +48,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run memory + run: cargo nextest run behavior env: - OPENDAL_MEMORY_TEST: on + OPENDAL_TEST: memory python: runs-on: ubuntu-latest diff --git a/.github/workflows/service_test_mini_moka.yml b/.github/workflows/service_test_mini_moka.yml index bc0dd81d2813..0774361e13de 100644 --- a/.github/workflows/service_test_mini_moka.yml +++ b/.github/workflows/service_test_mini_moka.yml @@ -48,6 +48,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run mini_moka --features services-mini-moka + run: cargo nextest run behavior --features services-mini-moka env: - OPENDAL_MINI_MOKA_TEST: on + OPENDAL_TEST: mini_moka diff --git a/.github/workflows/service_test_moka.yml b/.github/workflows/service_test_moka.yml index ccf9e862f339..a3fde34590f7 100644 --- a/.github/workflows/service_test_moka.yml +++ b/.github/workflows/service_test_moka.yml @@ -48,6 +48,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run moka --features services-moka + run: cargo nextest run behavior --features services-moka env: - OPENDAL_MOKA_TEST: on + OPENDAL_TEST: moka diff --git a/.github/workflows/service_test_mysql.yml b/.github/workflows/service_test_mysql.yml index 4a8066f8e23c..a57ba8296221 100644 --- a/.github/workflows/service_test_mysql.yml +++ b/.github/workflows/service_test_mysql.yml @@ -56,9 +56,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run mysql --features services-mysql + run: cargo nextest run behavior --features services-mysql env: - OPENDAL_MYSQL_TEST: on + OPENDAL_TEST: mysql OPENDAL_MYSQL_CONNECTION_STRING: mysql://root:password@localhost:3306/testdb OPENDAL_MYSQL_TABLE: data OPENDAL_MYSQL_KEY_FIELD: key diff --git a/.github/workflows/service_test_obs.yml b/.github/workflows/service_test_obs.yml index 76e7fe4e1d36..4be47a5146e0 100644 --- a/.github/workflows/service_test_obs.yml +++ b/.github/workflows/service_test_obs.yml @@ -54,7 +54,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_OBS_TEST: op://services/obs/test OPENDAL_OBS_BUCKET: op://services/obs/bucket OPENDAL_OBS_ENDPOINT: op://services/obs/endpoint OPENDAL_OBS_ACCESS_KEY_ID: op://services/obs/access_key_id @@ -63,4 +62,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run obs + run: cargo nextest run behavior + env: + OPENDAL_TEST: obs diff --git a/.github/workflows/service_test_oss.yml b/.github/workflows/service_test_oss.yml index 287cd6f80f6a..e548a5f1386c 100644 --- a/.github/workflows/service_test_oss.yml +++ b/.github/workflows/service_test_oss.yml @@ -49,9 +49,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run oss + run: cargo nextest run behavior env: - OPENDAL_OSS_TEST: ${{ secrets.OPENDAL_OSS_TEST }} + OPENDAL_TEST: oss OPENDAL_OSS_BUCKET: ${{ secrets.OPENDAL_OSS_BUCKET }} OPENDAL_OSS_ENDPOINT: ${{ secrets.OPENDAL_OSS_ENDPOINT }} OPENDAL_OSS_ACCESS_KEY_ID: ${{ secrets.OPENDAL_OSS_ACCESS_KEY_ID }} diff --git a/.github/workflows/service_test_persy.yml b/.github/workflows/service_test_persy.yml index 87ecb6050a42..1640cdf253ca 100644 --- a/.github/workflows/service_test_persy.yml +++ b/.github/workflows/service_test_persy.yml @@ -46,11 +46,11 @@ jobs: - name: Test persy shell: bash working-directory: core - run: cargo test persy --features services-persy + run: cargo test behavior --features services-persy env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_PERSY_TEST: on + OPENDAL_TEST: persy OPENDAL_PERSY_DATAFILE: ./test.persy OPENDAL_PERSY_SEGMENT: data OPENDAL_PERSY_INDEX: index diff --git a/.github/workflows/service_test_postgresql.yml b/.github/workflows/service_test_postgresql.yml index ff75fb39612d..6fdaa0b025eb 100644 --- a/.github/workflows/service_test_postgresql.yml +++ b/.github/workflows/service_test_postgresql.yml @@ -56,9 +56,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run postgresql --features services-postgresql + run: cargo nextest run behavior --features services-postgresql env: - OPENDAL_POSTGRESQL_TEST: on + OPENDAL_TEST: postgresql OPENDAL_POSTGRESQL_CONNECTION_STRING: postgresql://user:password@localhost:5432/testdb OPENDAL_POSTGRESQL_TABLE: data OPENDAL_POSTGRESQL_KEY_FIELD: key diff --git a/.github/workflows/service_test_redb.yml b/.github/workflows/service_test_redb.yml index ffe8f2dfdac5..919578030820 100644 --- a/.github/workflows/service_test_redb.yml +++ b/.github/workflows/service_test_redb.yml @@ -52,11 +52,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo test redb --features services-redb -j=1 + run: cargo test behavior --features services-redb -j=1 env: - RUST_BACKTRACE: full - RUST_LOG: debug - OPENDAL_REDB_TEST: on + OPENDAL_TEST: redb OPENDAL_REDB_ROOT: / OPENDAL_REDB_DATADIR: /tmp/opendal/redb OPENDAL_REDB_TABLE: redb-table diff --git a/.github/workflows/service_test_redis.yml b/.github/workflows/service_test_redis.yml index 70daf3cbbfa5..1f738eac3144 100644 --- a/.github/workflows/service_test_redis.yml +++ b/.github/workflows/service_test_redis.yml @@ -54,9 +54,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis + run: cargo nextest run behavior --features services-redis env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 @@ -84,9 +84,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis-rustls + run: cargo nextest run behavior --features services-redis-rustls env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_ENDPOINT: rediss://localhost:6379 OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 @@ -107,9 +107,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis + run: cargo nextest run behavior --features services-redis env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_CLUSTER_ENDPOINTS: redis://127.0.0.1:6380/,redis://127.0.0.1:6381/,redis://127.0.0.1:6382/,redis://127.0.0.1:6383/,redis://127.0.0.1:6384/,redis://127.0.0.1:6385/ OPENDAL_REDIS_ROOT: /test/opendal OPENDAL_REDIS_DB: 0 @@ -140,9 +140,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis-rustls + run: cargo nextest run behavior --features services-redis-rustls env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_CLUSTER_ENDPOINTS: rediss://127.0.0.1:6380/,rediss://127.0.0.1:6381/,rediss://127.0.0.1:6382/,rediss://127.0.0.1:6383/,rediss://127.0.0.1:6384/,rediss://127.0.0.1:6385/ OPENDAL_REDIS_PASSWORD: opendal OPENDAL_REDIS_ROOT: /test/opendal @@ -164,9 +164,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis + run: cargo nextest run behavior --features services-redis env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 @@ -187,9 +187,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run redis --features services-redis + run: cargo nextest run behavior --features services-redis env: - OPENDAL_REDIS_TEST: on + OPENDAL_TEST: redis OPENDAL_REDIS_ENDPOINT: tcp://127.0.0.1:6379 OPENDAL_REDIS_ROOT: / OPENDAL_REDIS_DB: 0 diff --git a/.github/workflows/service_test_rocksdb.yml b/.github/workflows/service_test_rocksdb.yml index ad190d2f1469..87453bc1a622 100644 --- a/.github/workflows/service_test_rocksdb.yml +++ b/.github/workflows/service_test_rocksdb.yml @@ -51,9 +51,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo test rocksdb --features services-rocksdb -j=1 + run: cargo test behavior --features services-rocksdb -j=1 env: LD_LIBRARY_PATH: /tmp/rocksdb/lib:${{ env.LD_LIBRARY_PATH }} - OPENDAL_ROCKSDB_TEST: on + OPENDAL_TEST: rocksdb OPENDAL_ROCKSDB_ROOT: / OPENDAL_ROCKSDB_DATADIR: /tmp/opendal/rocksdb/ diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index a0a6f943322d..fcceeed7ce01 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -51,9 +51,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: - OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} + OPENDAL_TEST: s3 OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} @@ -74,9 +74,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: - OPENDAL_S3_TEST: ${{ secrets.OPENDAL_S3_TEST }} + OPENDAL_TEST: s3 OPENDAL_S3_ROOT: ${{ secrets.OPENDAL_S3_ROOT }} OPENDAL_S3_BUCKET: ${{ secrets.OPENDAL_S3_BUCKET }} OPENDAL_S3_ENDPOINT: ${{ secrets.OPENDAL_S3_ENDPOINT }} @@ -109,11 +109,11 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: AWS_WEB_IDENTITY_TOKEN_FILE: tests/data/web_identity_token AWS_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing-assume - OPENDAL_S3_TEST: on + OPENDAL_TEST: s3 OPENDAL_S3_ROOT: CI/ OPENDAL_S3_BUCKET: opendal-testing OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing @@ -140,9 +140,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: - OPENDAL_S3_TEST: on + OPENDAL_TEST: s3 OPENDAL_S3_BUCKET: test OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" OPENDAL_S3_ACCESS_KEY_ID: minioadmin @@ -178,9 +178,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: - OPENDAL_S3_TEST: on + OPENDAL_TEST: s3 OPENDAL_S3_BUCKET: test OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" OPENDAL_S3_ALLOW_ANONYMOUS: on @@ -203,7 +203,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_S3_TEST: op://services/r2/test OPENDAL_S3_BUCKET: op://services/r2/bucket OPENDAL_S3_ENDPOINT: op://services/r2/endpoint OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id @@ -212,8 +211,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run s3 + run: cargo nextest run behavior env: + OPENDAL_TEST: s3 OPENDAL_S3_REGION: auto # This is the R2's limitation # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information diff --git a/.github/workflows/service_test_sftp.yml b/.github/workflows/service_test_sftp.yml index 2f7528fbd4bc..7458757039c3 100644 --- a/.github/workflows/service_test_sftp.yml +++ b/.github/workflows/service_test_sftp.yml @@ -58,9 +58,9 @@ jobs: shell: bash timeout-minutes: 10 working-directory: core - run: cargo nextest run sftp --features services-sftp + run: cargo nextest run behavior --features services-sftp env: - OPENDAL_SFTP_TEST: on + OPENDAL_TEST: sftp OPENDAL_SFTP_ENDPOINT: ssh://127.0.0.1:2222 OPENDAL_SFTP_ROOT: /upload/sftp_test/ OPENDAL_SFTP_USER: foo @@ -92,11 +92,11 @@ jobs: shell: bash timeout-minutes: 10 working-directory: core - run: cargo nextest run sftp --features services-sftp + run: cargo nextest run behavior --features services-sftp env: - OPENDAL_SFTP_TEST: on + OPENDAL_TEST: sftp OPENDAL_SFTP_ENDPOINT: ssh://127.0.0.1:2222 OPENDAL_SFTP_USER: foo OPENDAL_SFTP_KEY: ${{ github.workspace }}/target/ssh/id_rsa OPENDAL_SFTP_KNOWN_HOSTS_STRATEGY: accept - OPENDAL_DISABLE_RANDOM_ROOT: true \ No newline at end of file + OPENDAL_DISABLE_RANDOM_ROOT: true diff --git a/.github/workflows/service_test_sled.yml b/.github/workflows/service_test_sled.yml index 10f316be808d..30c1c8023806 100644 --- a/.github/workflows/service_test_sled.yml +++ b/.github/workflows/service_test_sled.yml @@ -46,11 +46,11 @@ jobs: - name: Test sled shell: bash working-directory: core - run: cargo test sled --features services-sled -j=1 + run: cargo test behavior --features services-sled -j=1 env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_SLED_TEST: on + OPENDAL_TEST: sled OPENDAL_SLED_ROOT: / OPENDAL_SLED_DATADIR: /tmp/opendal/sled/ sled-with-tree: @@ -62,11 +62,11 @@ jobs: - name: Test sled with tree shell: bash working-directory: core - run: cargo test sled --features services-sled -j=1 + run: cargo test behavior --features services-sled -j=1 env: RUST_BACKTRACE: full RUST_LOG: debug - OPENDAL_SLED_TEST: on + OPENDAL_TEST: sled OPENDAL_SLED_ROOT: / OPENDAL_SLED_DATADIR: /tmp/opendal/sled/ OPENDAL_SLED_TREE: sled-tree diff --git a/.github/workflows/service_test_sqlite.yml b/.github/workflows/service_test_sqlite.yml index c9c042ae3a65..0069ccd75b2c 100644 --- a/.github/workflows/service_test_sqlite.yml +++ b/.github/workflows/service_test_sqlite.yml @@ -56,9 +56,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run sqlite --features services-sqlite + run: cargo nextest run behavior --features services-sqlite env: - OPENDAL_SQLITE_TEST: on + OPENDAL_TEST: sqlite OPENDAL_SQLITE_CONNECTION_STRING: file:///tmp/opendal/test.db OPENDAL_SQLITE_TABLE: data OPENDAL_SQLITE_KEY_FIELD: key diff --git a/.github/workflows/service_test_supabase.yml b/.github/workflows/service_test_supabase.yml index 7ddabaca42ad..c772f7605319 100644 --- a/.github/workflows/service_test_supabase.yml +++ b/.github/workflows/service_test_supabase.yml @@ -52,7 +52,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_SUPABASE_TEST: op://services/supabase/test OPENDAL_SUPABASE_BUCKET: op://services/supabase/bucket OPENDAL_SUPABASE_ENDPOINT: op://services/supabase/endpoint OPENDAL_SUPABASE_KEY: op://services/supabase/key @@ -60,4 +59,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo test supabase --features=services-supabase + run: cargo test behavior --features=services-supabase + env: + OPENDAL_TEST: supabase diff --git a/.github/workflows/service_test_tikv.yml b/.github/workflows/service_test_tikv.yml index bb27fb259465..ef099eee24e3 100644 --- a/.github/workflows/service_test_tikv.yml +++ b/.github/workflows/service_test_tikv.yml @@ -66,14 +66,12 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run tikv --features services-tikv + run: cargo nextest run behavior --features services-tikv env: - RUST_BACKTRACE: full - RUST_LOG: debug - OPENDAL_TIKV_TEST: on + OPENDAL_TEST: tikv OPENDAL_TIKV_ENDPOINTS: "127.0.0.1:2379" OPENDAL_TIKV_INSECURE: true - + tikv-tls: runs-on: ubuntu-latest steps: @@ -129,11 +127,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run tikv --features services-tikv + run: cargo nextest run behavior --features services-tikv env: - RUST_BACKTRACE: full - RUST_LOG: debug - OPENDAL_TIKV_TEST: on + OPENDAL_TEST: tikv OPENDAL_TIKV_ENDPOINTS: "https://127.0.0.1:2379" OPENDAL_TIKV_INSECURE: false OPENDAL_TIKV_CA_PATH: "/tmp/tikv/ssl/ca.pem" diff --git a/.github/workflows/service_test_wasabi.yml b/.github/workflows/service_test_wasabi.yml index 6892e414294d..765d06b9b66e 100644 --- a/.github/workflows/service_test_wasabi.yml +++ b/.github/workflows/service_test_wasabi.yml @@ -54,7 +54,6 @@ jobs: export-env: true env: OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} - OPENDAL_WASABI_TEST: op://services/wasabi/test OPENDAL_WASABI_REGION: op://services/wasabi/region OPENDAL_WASABI_BUCKET: op://services/wasabi/bucket OPENDAL_WASABI_ENDPOINT: op://services/wasabi/endpoint @@ -64,4 +63,6 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run wasabi --features=services-wasabi + run: cargo nextest run behavior --features=services-wasabi + env: + OPENDAL_TEST: wasabi diff --git a/.github/workflows/service_test_webdav.yml b/.github/workflows/service_test_webdav.yml index 114357210077..36ec07c5a3d8 100644 --- a/.github/workflows/service_test_webdav.yml +++ b/.github/workflows/service_test_webdav.yml @@ -55,9 +55,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo nextest run webdav + run: cargo nextest run behavior env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080 nginx_with_password: @@ -77,9 +77,9 @@ jobs: - name: Test with password shell: bash working-directory: core - run: cargo nextest run webdav + run: cargo nextest run behavior env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080 OPENDAL_WEBDAV_USERNAME: bar OPENDAL_WEBDAV_PASSWORD: bar @@ -101,9 +101,9 @@ jobs: - name: Test empty password shell: bash working-directory: core - run: cargo nextest run webdav + run: cargo nextest run behavior env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080 OPENDAL_WEBDAV_USERNAME: foo @@ -125,9 +125,9 @@ jobs: shell: bash working-directory: core run: | - cargo nextest run webdav + cargo nextest run behavior env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8081 nextcloud: @@ -168,9 +168,9 @@ jobs: shell: bash working-directory: core run: | - cargo test webdav -j=1 + cargo test behavior -j=1 env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080/remote.php/webdav/ OPENDAL_WEBDAV_USERNAME: admin OPENDAL_WEBDAV_PASSWORD: admin @@ -216,9 +216,9 @@ jobs: shell: bash working-directory: core run: | - cargo test webdav -j=1 + cargo test behavior -j=1 env: - OPENDAL_WEBDAV_TEST: on + OPENDAL_TEST: webdav OPENDAL_WEBDAV_ENDPOINT: http://127.0.0.1:8080/remote.php/webdav/ OPENDAL_WEBDAV_USERNAME: admin OPENDAL_WEBDAV_PASSWORD: admin diff --git a/.github/workflows/service_test_webhdfs.yml b/.github/workflows/service_test_webhdfs.yml index b8057cd80e89..f6b7170b3736 100644 --- a/.github/workflows/service_test_webhdfs.yml +++ b/.github/workflows/service_test_webhdfs.yml @@ -70,9 +70,9 @@ jobs: - name: Test shell: bash working-directory: core - run: cargo test webhdfs + run: cargo test behavior env: - OPENDAL_WEBHDFS_TEST: on + OPENDAL_TEST: webhdfs OPENDAL_WEBHDFS_ROOT: / OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870 @@ -109,9 +109,9 @@ jobs: - name: Test with disable_list_batch shell: bash working-directory: core - run: cargo test webhdfs + run: cargo test behavior env: - OPENDAL_WEBHDFS_TEST: on + OPENDAL_TEST: webhdfs OPENDAL_WEBHDFS_ROOT: / OPENDAL_WEBHDFS_ENDPOINT: http://127.0.0.1:9870 OPENDAL_WEBHDFS_DISABLE_LIST_BATCH: true diff --git a/core/src/services/azdls/writer.rs b/core/src/services/azdls/writer.rs index bc0201fbf4d4..e9e57cd973da 100644 --- a/core/src/services/azdls/writer.rs +++ b/core/src/services/azdls/writer.rs @@ -106,6 +106,28 @@ impl oio::AppendObjectWrite for AzdlsWriter { } async fn append(&self, offset: u64, size: u64, body: AsyncBody) -> Result<()> { + if offset == 0 { + let mut req = + self.core + .azdls_create_request(&self.path, "file", &self.op, AsyncBody::Empty)?; + + self.core.sign(&mut req).await?; + + let resp = self.core.send(req).await?; + + let status = resp.status(); + match status { + StatusCode::CREATED | StatusCode::OK => { + resp.into_body().consume().await?; + } + _ => { + return Err(parse_error(resp) + .await? + .with_operation("Backend::azdls_create_request")); + } + } + } + let mut req = self .core .azdls_update_request(&self.path, Some(size), offset, body)?; diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index f9b70d2e6837..b7ccf14f72a7 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -157,7 +157,7 @@ impl Operator { Scheme::Atomicserver => Self::from_map::(map)?.finish(), #[cfg(feature = "services-azblob")] Scheme::Azblob => Self::from_map::(map)?.finish(), - #[cfg(feature = "services-Azdls")] + #[cfg(feature = "services-azdls")] Scheme::Azdls => Self::from_map::(map)?.finish(), #[cfg(feature = "services-cacache")] Scheme::Cacache => Self::from_map::(map)?.finish(), @@ -165,6 +165,8 @@ impl Operator { Scheme::Cos => Self::from_map::(map)?.finish(), #[cfg(feature = "services-dashmap")] Scheme::Dashmap => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-dropbox")] + Scheme::Dropbox => Self::from_map::(map)?.finish(), #[cfg(feature = "services-etcd")] Scheme::Etcd => Self::from_map::(map)?.finish(), #[cfg(feature = "services-foundationdb")] @@ -185,6 +187,8 @@ impl Operator { Scheme::Ipfs => Self::from_map::(map)?.finish(), #[cfg(feature = "services-ipmfs")] Scheme::Ipmfs => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-libsql")] + Scheme::Libsql => Self::from_map::(map)?.finish(), #[cfg(feature = "services-memcached")] Scheme::Memcached => Self::from_map::(map)?.finish(), #[cfg(feature = "services-memory")] @@ -193,10 +197,14 @@ impl Operator { Scheme::MiniMoka => Self::from_map::(map)?.finish(), #[cfg(feature = "services-moka")] Scheme::Moka => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-mysql")] + Scheme::Mysql => Self::from_map::(map)?.finish(), #[cfg(feature = "services-obs")] Scheme::Obs => Self::from_map::(map)?.finish(), #[cfg(feature = "services-onedrive")] Scheme::Onedrive => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-postgresql")] + Scheme::Postgresql => Self::from_map::(map)?.finish(), #[cfg(feature = "services-gdrive")] Scheme::Gdrive => Self::from_map::(map)?.finish(), #[cfg(feature = "services-oss")] @@ -213,8 +221,12 @@ impl Operator { Scheme::Sftp => Self::from_map::(map)?.finish(), #[cfg(feature = "services-sled")] Scheme::Sled => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-sqlite")] + Scheme::Sqlite => Self::from_map::(map)?.finish(), #[cfg(feature = "services-supabase")] Scheme::Supabase => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-tikv")] + Scheme::Tikv => Self::from_map::(map)?.finish(), #[cfg(feature = "services-vercel-artifacts")] Scheme::VercelArtifacts => Self::from_map::(map)?.finish(), #[cfg(feature = "services-wasabi")] diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 16692e53c404..b627d596a5b1 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -211,7 +211,7 @@ impl From for &'static str { match v { Scheme::Atomicserver => "atomicserver", Scheme::Azblob => "azblob", - Scheme::Azdls => "Azdls", + Scheme::Azdls => "azdls", Scheme::Cacache => "cacache", Scheme::Cos => "cos", Scheme::D1 => "d1", diff --git a/core/tests/behavior/README.md b/core/tests/behavior/README.md index dce39907cdb9..408aa4cb5a9c 100644 --- a/core/tests/behavior/README.md +++ b/core/tests/behavior/README.md @@ -11,14 +11,12 @@ To run the behavior tests, please copy the `.env.example`, which is at project r Take `fs` for example, we need to change to enable behavior test on `fs` on `/tmp`. ```dotenv -OPENDAL_FS_TEST=false OPENDAL_FS_ROOT=/path/to/dir ``` into ```dotenv -OPENDAL_FS_TEST=on OPENDAL_FS_ROOT=/tmp ``` @@ -26,34 +24,26 @@ Notice: If the env variables are not set, all behavior tests will be skipped by ## Run -Test all available backends. +Use `OPENDAL_TEST` to control which service to test: ```shell -cargo test +OPENDAL_TEST=fs cargo test behavior ``` -Test specific backend(such as `fs`). +To run certain types of tests(such as `write`), we use `behavior::test_write`. ```shell -cargo test services_fs -``` - -As `cargo test` only run tests containing the following string in their names, we use `services_fs` to run all tests under `services::fs`. - -To run certain types of tests(such as `write`) for `fs`, we use `services_fs::test_write`. - -```shell -cargo test services_fs::test_write +OPENDAL_TEST=fs cargo test behavior::test_write ``` You can also run specific test(such as `test_stat_dir`) for specific backend. ```shell -cargo test services_fs::test_stat_dir +OPENDAL_TEST=fs cargo test behavior::test_stat_dir ``` ## Debug -To debug a behavior test, you can use `RUST_LOG=debug RUST_BACKTRACE=full cargo test -- --show-output` to print the log with backtrace. +To debug a behavior test, you can use `RUST_LOG=debug RUST_BACKTRACE=full cargo test behavior -- --show-output` to print the log with backtrace. For more details, please visit [cargo test](https://doc.rust-lang.org/cargo/commands/cargo-test.html) or run the command `cargo test --help`. diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs index 3d7cc54d37de..ab4e900e126d 100644 --- a/core/tests/behavior/main.rs +++ b/core/tests/behavior/main.rs @@ -68,127 +68,33 @@ static RUNTIME: Lazy = Lazy::new(|| { .unwrap() }); -fn behavior_test() -> Vec { - let operator = match init_service::() { - Some(op) => op, - None => return Vec::new(), - }; - - let mut trials = vec![]; - // Blocking tests - trials.extend(behavior_blocking_append_tests(&operator)); - trials.extend(behavior_blocking_copy_tests(&operator)); - trials.extend(behavior_blocking_list_tests(&operator)); - trials.extend(behavior_blocking_read_only_tests(&operator)); - trials.extend(behavior_blocking_rename_tests(&operator)); - trials.extend(behavior_blocking_write_tests(&operator)); - // Async tests - trials.extend(behavior_append_tests(&operator)); - trials.extend(behavior_copy_tests(&operator)); - trials.extend(behavior_list_only_tests(&operator)); - trials.extend(behavior_list_tests(&operator)); - trials.extend(behavior_presign_tests(&operator)); - trials.extend(behavior_read_only_tests(&operator)); - trials.extend(behavior_rename_tests(&operator)); - trials.extend(behavior_write_tests(&operator)); - trials.extend(behavior_fuzz_tests(&operator)); - - trials -} - fn main() -> anyhow::Result<()> { let args = Arguments::from_args(); - let mut tests = Vec::new(); + let op = if let Some(op) = init_service()? { + op + } else { + return Ok(()); + }; - #[cfg(feature = "services-atomicserver")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-azblob")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-azdls")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-cacache")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-cos")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-dashmap")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-etcd")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-foundationdb")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-fs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-ftp")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-gcs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-ghac")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-hdfs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-http")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-ipfs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-ipmfs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-libsql")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-memcached")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-memory")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-mini-moka")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-moka")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-obs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-onedrive")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-postgresql")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-gdrive")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-dropbox")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-oss")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-persy")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-redis")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-rocksdb")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-s3")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-sftp")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-sled")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-supabase")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-vercel-artifacts")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-wasabi")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-webdav")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-webhdfs")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-redb")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-tikv")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-mysql")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-sqlite")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-d1")] - tests.extend(behavior_test::()); - #[cfg(feature = "services-azfile")] - tests.extend(behavior_test::()); + let mut tests = Vec::new(); + // Blocking tests + tests.extend(behavior_blocking_append_tests(&op)); + tests.extend(behavior_blocking_copy_tests(&op)); + tests.extend(behavior_blocking_list_tests(&op)); + tests.extend(behavior_blocking_read_only_tests(&op)); + tests.extend(behavior_blocking_rename_tests(&op)); + tests.extend(behavior_blocking_write_tests(&op)); + // Async tests + tests.extend(behavior_append_tests(&op)); + tests.extend(behavior_copy_tests(&op)); + tests.extend(behavior_list_only_tests(&op)); + tests.extend(behavior_list_tests(&op)); + tests.extend(behavior_presign_tests(&op)); + tests.extend(behavior_read_only_tests(&op)); + tests.extend(behavior_rename_tests(&op)); + tests.extend(behavior_write_tests(&op)); + tests.extend(behavior_fuzz_tests(&op)); // Don't init logging while building operator which may break cargo // nextest output diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs index 7cdd032a1b80..d1baf9ac5de1 100644 --- a/core/tests/behavior/utils.rs +++ b/core/tests/behavior/utils.rs @@ -21,6 +21,7 @@ use std::fmt; use std::fmt::Debug; use std::fmt::Formatter; use std::io::SeekFrom; +use std::str::FromStr; use std::usize; use bytes::Bytes; @@ -42,12 +43,20 @@ use crate::RUNTIME; /// Init a service with given scheme. /// -/// - If `opendal_{schema}_test` is on, construct a new Operator with given root. +/// - Load scheme from `OPENDAL_TEST` +/// - Construct a new Operator with given root. /// - Else, returns a `None` to represent no valid config for operator. -pub fn init_service() -> Option { +pub fn init_service() -> anyhow::Result> { let _ = dotenvy::dotenv(); - let prefix = format!("opendal_{}_", B::SCHEME); + let scheme = if let Ok(v) = env::var("OPENDAL_TEST") { + v + } else { + return Ok(None); + }; + let scheme = Scheme::from_str(&scheme).unwrap(); + + let prefix = format!("opendal_{scheme}_"); let mut cfg = env::vars() .filter_map(|(k, v)| { @@ -57,12 +66,6 @@ pub fn init_service() -> Option { }) .collect::>(); - let turn_on_test = cfg.get("test").cloned().unwrap_or_default(); - - if turn_on_test != "on" && turn_on_test != "true" { - return None; - } - // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true. let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true"; if !disable_random_root { @@ -74,7 +77,7 @@ pub fn init_service() -> Option { cfg.insert("root".to_string(), root); } - let op = Operator::from_map::(cfg).expect("must succeed"); + let op = Operator::via_map(scheme, cfg).expect("must succeed"); #[cfg(feature = "layers-chaos")] let op = { @@ -85,15 +88,15 @@ pub fn init_service() -> Option { let mut op = op .layer(LoggingLayer::default().with_backtrace_output(true)) .layer(TimeoutLayer::new()) - .layer(RetryLayer::new().with_max_times(4)) - .finish(); + .layer(RetryLayer::new().with_max_times(4)); + // Enable blocking layer if needed. if !op.info().full_capability().blocking { let _guard = RUNTIME.enter(); op = op.layer(BlockingLayer::create().expect("blocking layer must be created")) } - Some(op) + Ok(Some(op)) } pub fn gen_bytes_with_range(range: impl SampleRange) -> (Vec, usize) { @@ -132,11 +135,10 @@ where F: FnOnce(Operator) -> Fut + Send + 'static, Fut: Future>, { - let name = format!("services_{}::{}", op.info().scheme(), name); let handle = RUNTIME.handle().clone(); let op = op.clone(); - Trial::test(name, move || { + Trial::test(format!("behavior::{name}"), move || { handle .block_on(f(op)) .map_err(|err| Failed::from(err.to_string())) @@ -157,10 +159,9 @@ pub fn build_blocking_trial(name: &str, op: &Operator, f: F) -> Trial where F: FnOnce(BlockingOperator) -> anyhow::Result<()> + Send + 'static, { - let name = format!("services_{}::{}", op.info().scheme(), name); let op = op.blocking(); - Trial::test(name, move || { + Trial::test(format!("behavior::{name}"), move || { f(op).map_err(|err| Failed::from(err.to_string())) }) }