diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 787054df..5f2efd51 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -36,12 +36,14 @@ jobs: python get-pip.py --user fi - name: Setup mock server + shell: bash -el {0} run: | - conda create -n mock-server python=3.10 + conda create -y -n mock-server python=3.10 + conda init conda activate mock-server + python3 --version nohup python3 ./tests/mock-server/main.py --port 9000 > py-mock-server.log & echo $! > mock-server.pid - conda deactivate - name: Install dependencies shell: bash -l {0} @@ -49,7 +51,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -I -e ".[dev]" - name: Run cases - shell: bash -l {0} + shell: bash -el {0} env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} @@ -59,8 +61,13 @@ jobs: MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" PYTHONPATH: "$PYTHONPATH:." run: | - set -e flake8 --show-source --max-line-length=160 . + python -m pytest --collect-only ./test_qiniu.py ./tests/cases coverage run -m pytest ./test_qiniu.py ./tests/cases ocular --data-file .coverage codecov + cat mock-server.pid | xargs kill + - name: Print mock server log + if: ${{ failure() }} + run: | + cat py-mock-server.log diff --git a/qiniu/compat.py b/qiniu/compat.py index 4ab4ce78..6a418c99 100644 --- a/qiniu/compat.py +++ b/qiniu/compat.py @@ -33,6 +33,7 @@ # --------- if is_py2: + from urllib import urlencode # noqa from urlparse import urlparse # noqa import StringIO StringIO = BytesIO = StringIO.StringIO @@ -60,7 +61,7 @@ def is_seekable(data): return False elif is_py3: - from urllib.parse import urlparse # noqa + from urllib.parse import urlparse, urlencode # noqa import io StringIO = io.StringIO BytesIO = io.BytesIO diff --git a/setup.py b/setup.py index fdf0b413..154722b8 100644 --- a/setup.py +++ b/setup.py @@ -67,14 +67,6 @@ def find_version(*file_paths): 'freezegun', 'scrutinizer-ocular', 'codecov' - ], - 'dev: python_version <= "3.4"': [ - 'pytest~=4.6', - 'coverage~=5.5' - ], - 'dev: python_version < "3.8"': [ - 'pytest~=5.4', - 'coverage~=7.1' ] }, diff --git a/tests/cases/conftest.py b/tests/cases/conftest.py index 15b205f6..56792f20 100644 --- a/tests/cases/conftest.py +++ b/tests/cases/conftest.py @@ -1,7 +1,10 @@ +# -*- coding: utf-8 -*- import os import pytest +from qiniu import config as qn_config +from qiniu import region from qiniu import Auth @@ -32,3 +35,42 @@ def is_travis(): seems useless. """ yield os.getenv('QINIU_TEST_ENV') == 'travis' + + +@pytest.fixture(scope='function') +def set_conf_default(request): + if hasattr(request, 'param'): + qn_config.set_default(**request.param) + yield + qn_config._config = { + 'default_zone': region.Region(), + 'default_rs_host': qn_config.RS_HOST, + 'default_rsf_host': qn_config.RSF_HOST, + 'default_api_host': qn_config.API_HOST, + 'default_uc_host': qn_config.UC_HOST, + 'default_query_region_host': qn_config.QUERY_REGION_HOST, + 'default_query_region_backup_hosts': [ + 'uc.qbox.me', + 'api.qiniu.com' + ], + 'default_backup_hosts_retry_times': 2, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 2 * qn_config._BLOCK_SIZE # put_file上传方式的临界默认值 + } + + _is_customized_default = { + 'default_zone': False, + 'default_rs_host': False, + 'default_rsf_host': False, + 'default_api_host': False, + 'default_uc_host': False, + 'default_query_region_host': False, + 'default_query_region_backup_hosts': False, + 'default_backup_hosts_retry_times': False, + 'connection_timeout': False, + 'connection_retries': False, + 'connection_pool': False, + 'default_upload_threshold': False + } diff --git a/tests/cases/test_http/test_qiniu_conf.py b/tests/cases/test_http/test_qiniu_conf.py new file mode 100644 index 00000000..14ca13cd --- /dev/null +++ b/tests/cases/test_http/test_qiniu_conf.py @@ -0,0 +1,122 @@ +import pytest +import requests + +from qiniu.compat import urlencode +import qiniu.http as qiniu_http + + +@pytest.fixture(scope='function') +def retry_id(request, mock_server_addr): + success_times = [] + failure_times = [] + delay = None + if hasattr(request, 'param'): + success_times = request.param.get('success_times', success_times) + failure_times = request.param.get('failure_times', failure_times) + delay = request.param.get('delay', None) + query_dict = { + 's': success_times, + 'f': failure_times, + 'd': delay + } + if not query_dict['d']: + del query_dict['d'] + query_params = urlencode( + query_dict, + doseq=True + ) + request_url = '{scheme}://{host}/retry_me/__mgr__?{query_params}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + query_params=query_params + ) + resp = requests.put(request_url) + resp.raise_for_status() + record_id = resp.text + yield record_id + request_url = '{scheme}://{host}/retry_me/__mgr__?id={id}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + id=record_id + ) + resp = requests.delete(request_url) + resp.raise_for_status() + + +@pytest.fixture(scope='function') +def reset_session(): + qiniu_http._session = None + yield + + +class TestQiniuConf: + @pytest.mark.usefixtures('reset_session') + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'connection_timeout': 0.3, + 'connection_retries': 0 + } + ], + indirect=True + ) + @pytest.mark.parametrize( + 'method,opts', + [ + ('get', {}), + ('put', {'data': None, 'files': None}), + ('post', {'data': None, 'files': None}), + ('delete', {'params': None}) + ], + ids=lambda v: v if type(v) is str else 'opts' + ) + def test_timeout_conf(self, mock_server_addr, method, opts, set_conf_default): + request_url = '{scheme}://{host}/timeout?delay=0.5'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc + ) + send = getattr(qiniu_http.qn_http_client, method) + _ret, resp = send(request_url, **opts) + assert 'Read timed out' in str(resp.exception) + + @pytest.mark.usefixtures('reset_session') + @pytest.mark.parametrize( + 'retry_id', + [ + { + 'success_times': [0, 1], + 'failure_times': [5, 0], + }, + ], + indirect=True + ) + @pytest.mark.parametrize( + 'set_conf_default', + [ + { + 'connection_retries': 5 + } + ], + indirect=True + ) + @pytest.mark.parametrize( + 'method,opts', + [ + # post not retry default, see + # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS + ('get', {}), + ('put', {'data': None, 'files': None}), + ('delete', {'params': None}) + ], + ids=lambda v: v if type(v) is str else 'opts' + ) + def test_retry_times(self, retry_id, mock_server_addr, method, opts, set_conf_default): + request_url = '{scheme}://{host}/retry_me?id={id}'.format( + scheme=mock_server_addr.scheme, + host=mock_server_addr.netloc, + id=retry_id + ) + send = getattr(qiniu_http.qn_http_client, method) + _ret, resp = send(request_url, **opts) + assert resp.status_code == 200 diff --git a/tests/cases/test_services/test_storage/test_uploader.py b/tests/cases/test_services/test_storage/test_uploader.py index 396b2d1a..b1cdf51d 100644 --- a/tests/cases/test_services/test_storage/test_uploader.py +++ b/tests/cases/test_services/test_storage/test_uploader.py @@ -90,7 +90,7 @@ def temp_file(request): try: os.remove(tmp_file_path) - except FileNotFoundError: + except Exception: pass diff --git a/tests/mock_server/routes/__init__.py b/tests/mock_server/routes/__init__.py index 121a03ea..7eba32f1 100644 --- a/tests/mock_server/routes/__init__.py +++ b/tests/mock_server/routes/__init__.py @@ -1,7 +1,10 @@ -from .timeout import handle_timeout -from .echo import handle_echo +from .timeout import * +from .echo import * +from .retry_me import * routes = { '/timeout': handle_timeout, - '/echo': handle_echo + '/echo': handle_echo, + '/retry_me': handle_retry_me, + '/retry_me/__mgr__': handle_mgr_retry_me, } diff --git a/tests/mock_server/routes/retry_me.py b/tests/mock_server/routes/retry_me.py new file mode 100644 index 00000000..af4458f7 --- /dev/null +++ b/tests/mock_server/routes/retry_me.py @@ -0,0 +1,145 @@ +import http +import random +import string + +from urllib.parse import parse_qs + +__failure_record = {} + + +def should_fail_by_times(success_times=None, failure_times=None): + """ + Parameters + ---------- + success_times: list[int], default=[1] + failure_times: list[int], default=[0] + + Returns + ------- + Generator[bool, None, None] + + Examples + -------- + + should_fail_by_times([2], [3]) + will succeed 2 times and failed 3 times, and loop + + should_fail_by_times([2, 4], [3]) + will succeed 2 times and failed 3 times, + then succeeded 4 times and failed 3 time, and loop + """ + if not success_times: + success_times = [1] + if not failure_times: + failure_times = [0] + + def success_times_gen(): + while True: + for i in success_times: + yield i + + def failure_times_gen(): + while True: + for i in failure_times: + yield i + + success_times_iter = success_times_gen() + fail_times_iter = failure_times_gen() + + while True: + success = next(success_times_iter) + fail = next(fail_times_iter) + for _ in range(success): + yield False + for _ in range(fail): + yield True + + +def handle_mgr_retry_me(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in ['PUT', 'DELETE']: + request_handler.send_response(http.HTTPStatus.METHOD_NOT_ALLOWED) + return + match method: + case 'PUT': + # s for success + success_times = parse_qs(parsed_uri.query).get('s', []) + # f for failure + failure_times = parse_qs(parsed_uri.query).get('f', []) + + record_id = ''.join(random.choices(string.ascii_letters, k=16)) + + __failure_record[record_id] = should_fail_by_times( + success_times=[int(n) for n in success_times], + failure_times=[int(n) for n in failure_times] + ) + + request_handler.send_response(http.HTTPStatus.OK) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + request_handler.wfile.write(record_id.encode('utf-8')) + case 'DELETE': + record_id = parse_qs(parsed_uri.query).get('id') + if not record_id or not record_id[0]: + request_handler.send_response(http.HTTPStatus.BAD_REQUEST) + return + record_id = record_id[0] + + if record_id in __failure_record: + del __failure_record[record_id] + + request_handler.send_response(http.HTTPStatus.NO_CONTENT) + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + +def handle_retry_me(method, parsed_uri, request_handler): + """ + Parameters + ---------- + method: str + HTTP method + parsed_uri: urllib.parse.ParseResult + parsed URI + request_handler: http.server.BaseHTTPRequestHandler + request handler + """ + if method not in []: + # all method allowed + pass + record_id = parse_qs(parsed_uri.query).get('id') + if not record_id or not record_id[0]: + request_handler.send_response(http.HTTPStatus.BAD_REQUEST) + return + record_id = record_id[0] + + should_fail = next(__failure_record[record_id]) + + if should_fail: + request_handler.send_response(-1) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + resp_body = 'service unavailable' + request_handler.wfile.write(resp_body.encode('utf-8')) + return + + request_handler.send_response(http.HTTPStatus.OK) + request_handler.send_header('Content-Type', 'text/plain') + request_handler.send_header('X-Reqid', record_id) + request_handler.end_headers() + + resp_body = 'ok' + request_handler.wfile.write(resp_body.encode('utf-8'))