From c0e9640ed4a6083dc03c815bfb2e882097b0c81b Mon Sep 17 00:00:00 2001 From: Nadeshiko Manju Date: Mon, 30 Oct 2023 23:44:35 +0800 Subject: [PATCH] feat(test): Enable new test workflow planner for python binding (#3397) * feat(test): Enable new test workflow planner for python binding Signed-off-by: Manjusaka * Update code Signed-off-by: Manjusaka * Update code Signed-off-by: Manjusaka * Update code Signed-off-by: Manjusaka * Update code Signed-off-by: Manjusaka * update code Signed-off-by: Manjusaka * Update code Signed-off-by: Manjusaka * update code Signed-off-by: Manjusaka --------- Signed-off-by: Manjusaka --- .../behavior_test_binding_java/action.yaml | 2 +- .../behavior_test_binding_python/action.yaml | 51 ++++ .github/scripts/behavior_test/plan.py | 51 +++- .github/services/mysql/mysql/action.yml | 8 +- .github/workflows/behavior_test.yml | 12 + .../behavior_test_binding_python.yml | 71 ++++++ .github/workflows/bindings_python.yml | 19 -- .github/workflows/service_test_memory.yml | 61 ----- .github/workflows/service_test_s3.yml | 42 +--- bindings/python/Cargo.toml | 100 ++++++++ bindings/python/tests/conftest.py | 60 +++++ bindings/python/tests/test_capability.py | 31 +++ bindings/python/tests/test_read.py | 97 ++++++++ bindings/python/tests/test_services.py | 224 ------------------ bindings/python/tests/test_write.py | 115 +++++++++ 15 files changed, 592 insertions(+), 352 deletions(-) create mode 100644 .github/actions/behavior_test_binding_python/action.yaml create mode 100644 .github/workflows/behavior_test_binding_python.yml delete mode 100644 .github/workflows/service_test_memory.yml create mode 100644 bindings/python/tests/test_capability.py create mode 100644 bindings/python/tests/test_read.py delete mode 100644 bindings/python/tests/test_services.py create mode 100644 bindings/python/tests/test_write.py diff --git a/.github/actions/behavior_test_binding_java/action.yaml b/.github/actions/behavior_test_binding_java/action.yaml index cb3724a81d2f..d9252b1c324f 100644 --- a/.github/actions/behavior_test_binding_java/action.yaml +++ b/.github/actions/behavior_test_binding_java/action.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -name: Test Core +name: Test Binding Java description: 'Test Core with given setup and service' inputs: setup: diff --git a/.github/actions/behavior_test_binding_python/action.yaml b/.github/actions/behavior_test_binding_python/action.yaml new file mode 100644 index 000000000000..72437b08fb34 --- /dev/null +++ b/.github/actions/behavior_test_binding_python/action.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Test Binding Python +description: 'Test Core with given setup and service' +inputs: + setup: + description: "The setup action for test" + service: + description: "The service to test" + feature: + description: "The feature to test" + +runs: + using: "composite" + steps: + - name: Setup + shell: bash + run: | + mkdir -p ./dynamic_test_binding_python && + cat <./dynamic_test_binding_python/action.yml + runs: + using: composite + steps: + - name: Setup Test + uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }} + - name: Run Test Binding Python + shell: bash + working-directory: bindings/python + run: | + source venv/bin/activate + pytest -v tests + env: + OPENDAL_TEST: ${{ inputs.service }} + EOF + - name: Run + uses: ./dynamic_test_binding_python diff --git a/.github/scripts/behavior_test/plan.py b/.github/scripts/behavior_test/plan.py index 69397f67f817..3e0e5bdd66fd 100755 --- a/.github/scripts/behavior_test/plan.py +++ b/.github/scripts/behavior_test/plan.py @@ -22,6 +22,7 @@ import re from pathlib import Path from dataclasses import dataclass, field +from typing import Any # The path for current script. SCRIPT_PATH = Path(__file__).parent.absolute() @@ -31,7 +32,7 @@ PROJECT_DIR = GITHUB_DIR.parent -def provided_cases(): +def provided_cases() -> list[dict[str, str]]: root_dir = f"{GITHUB_DIR}/services" cases = [ @@ -74,6 +75,8 @@ class Hint: core: bool = field(default=False, init=False) # Is binding java affected? binding_java: bool = field(default=False, init=False) + # Is binding python affected? + binding_python: bool = field(default=False, init=False) # Should we run all services test? all_service: bool = field(default=False, init=False) @@ -81,7 +84,7 @@ class Hint: services: set = field(default_factory=set, init=False) -def calculate_hint(changed_files): +def calculate_hint(changed_files: list[str]) -> Hint: hint = Hint() # Remove all files that ends with `.md` @@ -95,12 +98,14 @@ def calculate_hint(changed_files): if p == ".github/workflows/behavior_test.yml": hint.core = True hint.binding_java = True + hint.binding_python = True hint.all_service = True if p == ".github/workflows/behavior_test_core.yml": hint.core = True hint.all_service = True if p == ".github/workflows/behavior_test_binding_java.yml": hint.binding_java = True + hint.binding_python = True hint.all_service = True # core affected @@ -113,11 +118,13 @@ def calculate_hint(changed_files): ): hint.core = True hint.binding_java = True + hint.binding_python = True hint.all_service = True # binding java affected. if p.startswith("bindings/java/"): hint.binding_java = True + hint.binding_python = True hint.all_service = True # core service affected @@ -125,6 +132,7 @@ def calculate_hint(changed_files): if match: hint.core = True hint.binding_java = True + hint.binding_python = True hint.services.add(match.group(1)) # core test affected @@ -132,6 +140,7 @@ def calculate_hint(changed_files): if match: hint.core = True hint.binding_java = True + hint.binding_python = True hint.services.add(match.group(1)) return hint @@ -154,7 +163,9 @@ def unique_cases(cases): return list(ucases.values()) -def generate_core_cases(cases, hint): +def generate_core_cases( + cases: list[dict[str, str]], hint: Hint +) -> list[dict[str, str]]: # Always run all tests if it is a push event. if os.getenv("GITHUB_IS_PUSH") == "true": return cases @@ -172,7 +183,9 @@ def generate_core_cases(cases, hint): return cases -def generate_binding_java_cases(cases, hint): +def generate_binding_java_cases( + cases: list[dict[str, str]], hint: Hint +) -> list[dict[str, str]]: cases = unique_cases(cases) # Always run all tests if it is a push event. @@ -192,20 +205,43 @@ def generate_binding_java_cases(cases, hint): return cases -def plan(changed_files): +def generate_binding_python_cases( + cases: list[dict[str, str]], hint: Hint +) -> list[dict[str, str]]: + cases = unique_cases(cases) + if os.getenv("GITHUB_IS_PUSH") == "true": + return cases + + # Return empty if core is False + if not hint.binding_python: + return [] + + # Return all services if all_service is True + if hint.all_service: + return cases + + # Filter all cases that not shown un in changed files + cases = [v for v in cases if v["service"] in hint.services] + return cases + + +def plan(changed_files: list[str]) -> dict[str, Any]: cases = provided_cases() hint = calculate_hint(changed_files) core_cases = generate_core_cases(cases, hint) binding_java_cases = generate_binding_java_cases(cases, hint) + binding_python_cases = generate_binding_python_cases(cases, hint) jobs = { "components": { "core": False, "binding_java": False, + "binding_python": False, }, "core": [], "binding_java": [], + "binding_python": [], } if len(core_cases) > 0: @@ -228,6 +264,11 @@ def plan(changed_files): jobs["binding_java"].append( {"os": "ubuntu-latest", "cases": binding_java_cases} ) + if len(binding_python_cases) > 0: + jobs["components"]["binding_python"] = True + jobs["binding_python"].append( + {"os": "ubuntu-latest", "cases": binding_python_cases} + ) return jobs diff --git a/.github/services/mysql/mysql/action.yml b/.github/services/mysql/mysql/action.yml index 2f2b332e5ee9..a41e7d2be761 100644 --- a/.github/services/mysql/mysql/action.yml +++ b/.github/services/mysql/mysql/action.yml @@ -24,7 +24,13 @@ runs: - name: Setup MySQL Server shell: bash working-directory: fixtures/mysql - run: docker-compose -f docker-compose.yml up -d + run: | + apt update && apt install -y mysql-client + docker-compose -f docker-compose.yml up -d + while ! mysql -h localhost -P 3306 --protocol=tcp -u root -p'password' -e "SELECT 1"; do + echo "Waiting for MySQL..." + sleep 1 + done - name: Setup shell: bash run: | diff --git a/.github/workflows/behavior_test.yml b/.github/workflows/behavior_test.yml index 364eba63b67a..861a7f981900 100644 --- a/.github/workflows/behavior_test.yml +++ b/.github/workflows/behavior_test.yml @@ -102,3 +102,15 @@ jobs: with: os: ${{ matrix.os }} cases: ${{ toJson(matrix.cases) }} + test_binding_python: + name: binding_python / ${{ matrix.os }} + needs: [plan] + if: fromJson(needs.plan.outputs.plan).components.binding_python + secrets: inherit + strategy: + matrix: + include: ${{ fromJson(needs.plan.outputs.plan).binding_python }} + uses: ./.github/workflows/behavior_test_binding_python.yml + with: + os: ${{ matrix.os }} + cases: ${{ toJson(matrix.cases) }} diff --git a/.github/workflows/behavior_test_binding_python.yml b/.github/workflows/behavior_test_binding_python.yml new file mode 100644 index 000000000000..ed6b099d20f2 --- /dev/null +++ b/.github/workflows/behavior_test_binding_python.yml @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Behavior Test Binding Python + +on: + workflow_call: + inputs: + os: + required: true + type: string + cases: + required: true + type: string + +jobs: + test: + name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }} + runs-on: ${{ inputs.os }} + strategy: + matrix: + cases: ${{ fromJson(inputs.cases) }} + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + need-nextest: true + need-protoc: true + need-rocksdb: true + github-token: ${{ secrets.GITHUB_TOKEN }} + + # TODO: 1password is only supported on linux + # + # Waiting for https://github.com/1Password/load-secrets-action/issues/46 + - name: Setup service account token for 1password + if: runner.os == 'Linux' + shell: bash + run: echo "OP_SERVICE_ACCOUNT_TOKEN=${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}" >> $GITHUB_ENV + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Build with maturin + shell: bash + working-directory: "bindings/python" + run: | + python -m venv venv + source venv/bin/activate + pip install maturin[patchelf] + maturin develop -E test -F ${{ matrix.cases.feature }} + + - name: Test Core + uses: ./.github/actions/behavior_test_binding_python + with: + setup: ${{ matrix.cases.setup }} + service: ${{ matrix.cases.service }} + feature: ${{ matrix.cases.feature }} diff --git a/.github/workflows/bindings_python.yml b/.github/workflows/bindings_python.yml index 56bfbea68ea4..d3ba0bcd2616 100644 --- a/.github/workflows/bindings_python.yml +++ b/.github/workflows/bindings_python.yml @@ -39,25 +39,6 @@ permissions: contents: read jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - name: Build with maturin - working-directory: "bindings/python" - run: | - python -m pip install -e .[test] - - name: Run pytest - working-directory: "bindings/python" - env: - OPENDAL_MEMORY_TEST: on - run: | - pytest -vk TestMemory sdist: runs-on: ubuntu-latest diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml deleted file mode 100644 index 3277a4a67799..000000000000 --- a/.github/workflows/service_test_memory.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Service Test Memory - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths: - - "core/src/**" - - "core/tests/**" - - "!core/src/docs/**" - - "!core/src/services/**" - - "core/src/services/memory/**" - - ".github/workflows/service_test_memory.yml" - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true - -jobs: - python: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Build with maturin - working-directory: "bindings/python" - run: | - python -m pip install -e .[test] - - - name: Test - shell: bash - working-directory: bindings/python - run: pytest -vk TestMemory - env: - OPENDAL_MEMORY_TEST: on diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index ed2f5a387165..1a255585a1c7 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -69,44 +69,4 @@ jobs: OPENDAL_S3_ROOT: CI/ OPENDAL_S3_BUCKET: opendal-testing OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing - OPENDAL_S3_REGION: ap-northeast-1 - - python: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Setup MinIO Server - shell: bash - working-directory: fixtures/s3 - run: docker-compose -f docker-compose-minio.yml up -d - - - name: Setup test bucket - env: - AWS_ACCESS_KEY_ID: "minioadmin" - AWS_SECRET_ACCESS_KEY: "minioadmin" - AWS_EC2_METADATA_DISABLED: "true" - run: aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test - - - name: Setup Rust toolchain - uses: ./.github/actions/setup - - - name: Build with maturin - working-directory: "bindings/python" - run: | - python -m pip install -e .[test] - - - name: Test - shell: bash - working-directory: bindings/python - run: pytest -vk TestS3 - env: - OPENDAL_S3_TEST: on - OPENDAL_S3_BUCKET: test - OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000" - OPENDAL_S3_ACCESS_KEY_ID: minioadmin - OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin - OPENDAL_S3_REGION: us-east-1 + OPENDAL_S3_REGION: ap-northeast-1 \ No newline at end of file diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 4bf502e8e187..a2ef50b5bf81 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -27,6 +27,106 @@ repository.workspace = true rust-version.workspace = true version.workspace = true + +[features] +# Enable all opendal default feature by default. +default = [ + "services-azblob", + "services-azdls", + "services-cos", + "services-fs", + "services-gcs", + "services-ghac", + "services-http", + "services-ipmfs", + "services-memory", + "services-obs", + "services-oss", + "services-s3", + "services-webdav", + "services-webhdfs", +] + +services-all = [ + "default", + + "services-cacache", + "services-dashmap", + "services-dropbox", + "services-etcd", + # FIXME this requires a preinstalled fdb library + # "services-foundationdb", + "services-ftp", + "services-gdrive", + # FIXME how to support HDFS services in bindings? + # "services-hdfs", + "services-ipfs", + "services-memcached", + "services-mini-moka", + "services-moka", + "services-onedrive", + "services-persy", + "services-postgresql", + "services-mysql", + "services-redb", + "services-redis", + "services-redis-rustls", + "services-rocksdb", + "services-sled", + "services-supabase", + "services-tikv", + "services-vercel-artifacts", + "services-wasabi", + "services-mongodb", + "services-sqlite", +] + +# Default services provided by opendal. +services-azblob = [ "opendal/services-azblob" ] +services-azdls = [ "opendal/services-azdls" ] +services-cos = [ "opendal/services-cos" ] +services-fs = [ "opendal/services-fs" ] +services-gcs = [ "opendal/services-gcs" ] +services-ghac = [ "opendal/services-ghac" ] +services-http = [ "opendal/services-http" ] +services-ipmfs = [ "opendal/services-ipmfs" ] +services-memory = [ "opendal/services-memory" ] +services-obs = [ "opendal/services-obs" ] +services-oss = [ "opendal/services-oss" ] +services-s3 = [ "opendal/services-s3" ] +services-webdav = [ "opendal/services-webdav" ] +services-webhdfs = [ "opendal/services-webhdfs" ] + +# Optional services provided by opendal. +services-cacache = ["opendal/services-cacache"] +services-dashmap = ["opendal/services-dashmap"] +services-dropbox = ["opendal/services-dropbox"] +services-etcd = ["opendal/services-etcd"] +services-foundationdb = ["opendal/services-foundationdb"] +services-ftp = ["opendal/services-ftp"] +services-gdrive = ["opendal/services-gdrive"] +services-hdfs = ["opendal/services-hdfs"] +services-ipfs = ["opendal/services-ipfs"] +services-memcached = ["opendal/services-memcached"] +services-mini-moka = ["opendal/services-mini-moka"] +services-moka = ["opendal/services-moka"] +services-onedrive = ["opendal/services-onedrive"] +services-persy = ["opendal/services-persy"] +services-postgresql = ["opendal/services-postgresql"] +services-redb = ["opendal/services-redb"] +services-redis = ["opendal/services-redis"] +services-redis-rustls = ["opendal/services-redis-rustls"] +services-rocksdb = ["opendal/services-rocksdb"] +services-sftp = ["opendal/services-sftp"] +services-sled = ["opendal/services-sled"] +services-supabase = ["opendal/services-supabase"] +services-tikv = ["opendal/services-tikv"] +services-vercel-artifacts = ["opendal/services-vercel-artifacts"] +services-wasabi = ["opendal/services-wasabi"] +services-mysql = ["opendal/services-mysql"] +services-mongodb = ["opendal/services-mongodb"] +services-sqlite = ["opendal/services-sqlite"] + [lib] crate-type = ["cdylib"] doc = false diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index 9db7dbe92b8f..6c1a518dab57 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -15,9 +15,69 @@ # specific language governing permissions and limitations # under the License. +import os + from dotenv import load_dotenv import pytest +import opendal load_dotenv() pytest_plugins = ("pytest_asyncio",) + + +def pytest_configure(config): + # register an additional marker + config.addinivalue_line( + "markers", + "need_capability(*capability): mark test to run only on named capability", + ) + + +@pytest.fixture() +def service_name(): + service_name = os.environ.get("OPENDAL_TEST") + if service_name is None: + pytest.skip("OPENDAL_TEST not set") + return service_name + + +@pytest.fixture() +def setup_config(service_name): + # Read arguments from envs. + prefix = f"opendal_{service_name}_" + config = {} + for key in os.environ.keys(): + if key.lower().startswith(prefix): + config[key[len(prefix) :].lower()] = os.environ.get(key) + + return config + + +@pytest.fixture() +def operator(service_name, setup_config): + return opendal.Operator(service_name, **setup_config) + + +@pytest.fixture() +def async_operator(service_name, setup_config): + return opendal.AsyncOperator(service_name, **setup_config) + + +@pytest.fixture(autouse=True) +def check_capability(request, operator, async_operator): + if request.node.get_closest_marker("need_capability"): + if request.node.get_closest_marker("need_capability").args: + if not all( + [ + getattr(operator.capability(), x) + for x in request.node.get_closest_marker("need_capability").args + ] + + [ + getattr(async_operator.capability(), x) + for x in request.node.get_closest_marker("need_capability").args + ] + ): + pytest.skip( + f"skip because {request.node.get_closest_marker('need_capability').args} not supported" + ) diff --git a/bindings/python/tests/test_capability.py b/bindings/python/tests/test_capability.py new file mode 100644 index 000000000000..da2e9c14f854 --- /dev/null +++ b/bindings/python/tests/test_capability.py @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + + +def test_capability(service_name, operator): + cap = operator.capability() + assert cap is not None + assert cap.read is not None + + +def test_capability_exception(service_name, operator): + cap = operator.capability() + assert cap is not None + with pytest.raises(AttributeError) as e_info: + cap.read_demo diff --git a/bindings/python/tests/test_read.py b/bindings/python/tests/test_read.py new file mode 100644 index 000000000000..7c89ac9fdb38 --- /dev/null +++ b/bindings/python/tests/test_read.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from uuid import uuid4 +from random import randint + +import pytest + + +@pytest.mark.need_capability("read", "write", "delete") +def test_sync_read(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"random_file_{str(uuid4())}" + content = os.urandom(size) + operator.write(filename, content) + + read_content = operator.read(filename) + assert read_content is not None + assert read_content == content + + operator.delete(filename) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("read", "write", "delete") +async def test_async_read(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"random_file_{str(uuid4())}" + content = os.urandom(size) + await async_operator.write(filename, content) + + read_content = await async_operator.read(filename) + assert read_content is not None + assert read_content == content + + await async_operator.delete(filename) + + +@pytest.mark.need_capability("read", "write", "delete", "stat") +def test_sync_read_stat(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"random_file_{str(uuid4())}" + content = os.urandom(size) + operator.write(filename, content) + + metadata = operator.stat(filename) + assert metadata is not None + assert metadata.content_length == len(content) + assert metadata.mode.is_file() + + operator.delete(filename) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("read", "write", "delete", "stat") +async def test_async_read_stat(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"random_file_{str(uuid4())}" + content = os.urandom(size) + await async_operator.write(filename, content) + + metadata = await async_operator.stat(filename) + assert metadata is not None + assert metadata.content_length == len(content) + assert metadata.mode.is_file() + + await async_operator.delete(filename) + + operator.delete(filename) + + +@pytest.mark.need_capability("read") +def test_sync_read_not_exists(service_name, operator, async_operator): + with pytest.raises(FileNotFoundError): + operator.read(str(uuid4())) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("read") +async def test_async_read_not_exists(service_name, operator, async_operator): + with pytest.raises(FileNotFoundError): + await async_operator.read(str(uuid4())) diff --git a/bindings/python/tests/test_services.py b/bindings/python/tests/test_services.py deleted file mode 100644 index 1d23cefc2e93..000000000000 --- a/bindings/python/tests/test_services.py +++ /dev/null @@ -1,224 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -from abc import ABC -from uuid import uuid4 -from random import randint - -import opendal -import pytest - - -class AbstractTestSuite(ABC): - service_name = "" - - def setup_method(self): - # Read arguments from envs. - prefix = f"opendal_{self.service_name}_" - self.config = {} - for key in os.environ.keys(): - if key.lower().startswith(prefix): - self.config[key[len(prefix) :].lower()] = os.environ.get(key) - - # Check if current test be enabled. - test_flag = self.config.get("test", "") - if test_flag != "on" and test_flag != "true": - raise ValueError(f"Service {self.service_name} test is not enabled.") - - self.operator = opendal.Operator(self.service_name, **self.config) - self.async_operator = opendal.AsyncOperator(self.service_name, **self.config) - - def test_sync_read(self): - size = randint(1, 1024) - filename = f"random_file_{str(uuid4())}" - content = os.urandom(size) - self.operator.write(filename, content) - - read_content = self.operator.read(filename) - assert read_content is not None - assert read_content == content - - self.operator.delete(filename) - - @pytest.mark.asyncio - async def test_async_read(self): - size = randint(1, 1024) - filename = f"random_file_{str(uuid4())}" - content = os.urandom(size) - await self.async_operator.write(filename, content) - - read_content = await self.async_operator.read(filename) - assert read_content is not None - assert read_content == content - - await self.async_operator.delete(filename) - - def test_sync_read_stat(self): - size = randint(1, 1024) - filename = f"random_file_{str(uuid4())}" - content = os.urandom(size) - self.operator.write(filename, content) - - metadata = self.operator.stat(filename) - assert metadata is not None - assert metadata.content_length == len(content) - assert metadata.mode.is_file() - - self.operator.delete(filename) - - @pytest.mark.asyncio - async def test_async_read_stat(self): - size = randint(1, 1024) - filename = f"random_file_{str(uuid4())}" - content = os.urandom(size) - await self.async_operator.write(filename, content) - - metadata = await self.async_operator.stat(filename) - assert metadata is not None - assert metadata.content_length == len(content) - assert metadata.mode.is_file() - - await self.async_operator.delete(filename) - - self.operator.delete(filename) - - def test_sync_read_not_exists(self): - with pytest.raises(FileNotFoundError): - self.operator.read(str(uuid4())) - - @pytest.mark.asyncio - async def test_async_read_not_exists(self): - with pytest.raises(FileNotFoundError): - await self.async_operator.read(str(uuid4())) - - def test_sync_write(self): - size = randint(1, 1024) - filename = f"test_file_{str(uuid4())}.txt" - content = os.urandom(size) - size = len(content) - self.operator.write(filename, content) - metadata = self.operator.stat(filename) - assert metadata is not None - assert metadata.mode.is_file() - assert metadata.content_length == size - - self.operator.delete(filename) - - @pytest.mark.asyncio - async def test_async_write(self): - size = randint(1, 1024) - filename = f"test_file_{str(uuid4())}.txt" - content = os.urandom(size) - size = len(content) - await self.async_operator.write(filename, content) - metadata = await self.async_operator.stat(filename) - assert metadata is not None - assert metadata.mode.is_file() - assert metadata.content_length == size - - await self.async_operator.delete(filename) - - def test_sync_write_with_non_ascii_name(self): - size = randint(1, 1024) - filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test" - content = os.urandom(size) - size = len(content) - self.operator.write(filename, content) - metadata = self.operator.stat(filename) - assert metadata is not None - assert metadata.mode.is_file() - assert metadata.content_length == size - - self.operator.delete(filename) - - @pytest.mark.asyncio - async def test_async_write_with_non_ascii_name(self): - size = randint(1, 1024) - filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test" - content = os.urandom(size) - size = len(content) - await self.async_operator.write(filename, content) - metadata = await self.async_operator.stat(filename) - assert metadata is not None - assert metadata.mode.is_file() - assert metadata.content_length == size - - await self.async_operator.delete(filename) - - def test_sync_create_dir(self): - path = f"test_dir_{str(uuid4())}/" - self.operator.create_dir(path) - metadata = self.operator.stat(path) - assert metadata is not None - assert metadata.mode.is_dir() - - self.operator.delete(path) - - @pytest.mark.asyncio - async def test_async_create_dir(self): - path = f"test_dir_{str(uuid4())}/" - await self.async_operator.create_dir(path) - metadata = await self.async_operator.stat(path) - assert metadata is not None - assert metadata.mode.is_dir() - - await self.async_operator.delete(path) - - def test_sync_delete(self): - size = randint(1, 1024) - filename = f"test_file_{str(uuid4())}.txt" - content = os.urandom(size) - size = len(content) - self.operator.write(filename, content) - self.operator.delete(filename) - with pytest.raises(FileNotFoundError): - self.operator.stat(filename) - - @pytest.mark.asyncio - async def test_async_delete(self): - size = randint(1, 1024) - filename = f"test_file_{str(uuid4())}.txt" - content = os.urandom(size) - size = len(content) - await self.async_operator.write(filename, content) - await self.async_operator.delete(filename) - with pytest.raises(FileNotFoundError): - await self.operator.stat(filename) - - def test_capability(self): - cap = self.operator.capability() - assert cap is not None - assert cap.read is not None - - def test_capability_exception(self): - cap = self.operator.capability() - assert cap is not None - with pytest.raises(AttributeError) as e_info: - cap.read_demo - - -class TestS3(AbstractTestSuite): - service_name = "s3" - - -class TestFS(AbstractTestSuite): - service_name = "fs" - - -class TestMemory(AbstractTestSuite): - service_name = "memory" diff --git a/bindings/python/tests/test_write.py b/bindings/python/tests/test_write.py new file mode 100644 index 000000000000..12e173ed98b6 --- /dev/null +++ b/bindings/python/tests/test_write.py @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from uuid import uuid4 +from random import randint + +import pytest + + +@pytest.mark.need_capability("write", "delete", "stat") +def test_sync_write(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"test_file_{str(uuid4())}.txt" + content = os.urandom(size) + size = len(content) + operator.write(filename, content) + metadata = operator.stat(filename) + assert metadata is not None + assert metadata.mode.is_file() + assert metadata.content_length == size + + operator.delete(filename) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("write", "delete", "stat") +async def test_async_write(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"test_file_{str(uuid4())}.txt" + content = os.urandom(size) + size = len(content) + await async_operator.write(filename, content) + metadata = await async_operator.stat(filename) + assert metadata is not None + assert metadata.mode.is_file() + assert metadata.content_length == size + + await async_operator.delete(filename) + + +@pytest.mark.need_capability("write", "delete", "stat") +def test_sync_write_with_non_ascii_name(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"βŒπŸ˜±δΈ­ζ–‡_{str(uuid4())}.test" + content = os.urandom(size) + size = len(content) + operator.write(filename, content) + metadata = operator.stat(filename) + assert metadata is not None + assert metadata.mode.is_file() + assert metadata.content_length == size + + operator.delete(filename) + +@pytest.mark.need_capability("create_dir", "stat") +def test_sync_create_dir(service_name, operator, async_operator): + path = f"test_dir_{str(uuid4())}/" + operator.create_dir(path) + metadata = operator.stat(path) + assert metadata is not None + assert metadata.mode.is_dir() + + operator.delete(path) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("create_dir", "stat") +async def test_async_create_dir(service_name, operator, async_operator): + path = f"test_dir_{str(uuid4())}/" + await async_operator.create_dir(path) + metadata = await async_operator.stat(path) + assert metadata is not None + assert metadata.mode.is_dir() + + await async_operator.delete(path) + + +@pytest.mark.need_capability("delete", "stat") +def test_sync_delete(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"test_file_{str(uuid4())}.txt" + content = os.urandom(size) + size = len(content) + operator.write(filename, content) + operator.delete(filename) + with pytest.raises(FileNotFoundError): + operator.stat(filename) + + +@pytest.mark.asyncio +@pytest.mark.need_capability("delete", "stat") +async def test_async_delete(service_name, operator, async_operator): + size = randint(1, 1024) + filename = f"test_file_{str(uuid4())}.txt" + content = os.urandom(size) + size = len(content) + await async_operator.write(filename, content) + await async_operator.delete(filename) + with pytest.raises(FileNotFoundError): + await operator.stat(filename)