add test cases for timeout and retries

.github/workflows/ci-test.yml

@@ -36,20 +36,21 @@ jobs:
             python get-pip.py --user
           fi
       - name: Setup mock server
+        shell: bash -el {0}
         run: |
-          conda create -n mock-server python=3.10
+          conda create -y -n mock-server python=3.10
           conda activate mock-server
-          nohup python3 ./tests/mock-server/main.py --port 9000 > py-mock-server.log &
+          python3 --version
+          nohup python3 tests/mock_server/main.py --port 9000 > py-mock-server.log &
           echo $! > mock-server.pid
-          
           conda deactivate
       - name: Install dependencies
         shell: bash -l {0}
         run: |
           python -m pip install --upgrade pip
           python -m pip install -I -e ".[dev]"
       - name: Run cases
-        shell: bash -l {0}
+        shell: bash -el {0}
         env:
           QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
           QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
@@ -59,8 +60,12 @@ jobs:
           MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
           PYTHONPATH: "$PYTHONPATH:."
         run: |
-          set -e
-          flake8 --show-source --max-line-length=160 .
+          flake8 --show-source --max-line-length=160 ./qiniu
           coverage run -m pytest ./test_qiniu.py ./tests/cases
           ocular --data-file .coverage
           codecov
+          cat mock-server.pid | xargs kill
+      - name: Print mock server log
+        if: ${{ failure() }}
+        run: |
+          cat py-mock-server.log

qiniu/compat.py

@@ -33,6 +33,7 @@
 # ---------
 
 if is_py2:
+    from urllib import urlencode  # noqa
     from urlparse import urlparse  # noqa
     import StringIO
     StringIO = BytesIO = StringIO.StringIO
@@ -60,7 +61,7 @@ def is_seekable(data):
             return False
 
 elif is_py3:
-    from urllib.parse import urlparse  # noqa
+    from urllib.parse import urlparse, urlencode  # noqa
     import io
     StringIO = io.StringIO
     BytesIO = io.BytesIO

setup.py

@@ -60,21 +60,13 @@ def find_version(*file_paths):
     ],
     extras_require={
         'dev': [
-            'coverage',
+            'coverage<7.2',
             'flake8',
             'pytest',
             'pytest-cov',
             'freezegun',
             'scrutinizer-ocular',
             'codecov'
-        ],
-        'dev: python_version <= "3.4"': [
-            'pytest~=4.6',
-            'coverage~=5.5'
-        ],
-        'dev: python_version < "3.8"': [
-            'pytest~=5.4',
-            'coverage~=7.1'
         ]
     },
 

tests/cases/conftest.py

@@ -1,7 +1,10 @@
+# -*- coding: utf-8 -*-
 import os
 
 import pytest
 
+from qiniu import config as qn_config
+from qiniu import region
 from qiniu import Auth
 
 
@@ -32,3 +35,42 @@ def is_travis():
     seems useless.
     """
     yield os.getenv('QINIU_TEST_ENV') == 'travis'
+
+
+@pytest.fixture(scope='function')
+def set_conf_default(request):
+    if hasattr(request, 'param'):
+        qn_config.set_default(**request.param)
+    yield
+    qn_config._config = {
+        'default_zone': region.Region(),
+        'default_rs_host': qn_config.RS_HOST,
+        'default_rsf_host': qn_config.RSF_HOST,
+        'default_api_host': qn_config.API_HOST,
+        'default_uc_host': qn_config.UC_HOST,
+        'default_query_region_host': qn_config.QUERY_REGION_HOST,
+        'default_query_region_backup_hosts': [
+            'uc.qbox.me',
+            'api.qiniu.com'
+        ],
+        'default_backup_hosts_retry_times': 2,
+        'connection_timeout': 30,  # 链接超时为时间为30s
+        'connection_retries': 3,  # 链接重试次数为3次
+        'connection_pool': 10,  # 链接池个数为10
+        'default_upload_threshold': 2 * qn_config._BLOCK_SIZE  # put_file上传方式的临界默认值
+    }
+
+    _is_customized_default = {
+        'default_zone': False,
+        'default_rs_host': False,
+        'default_rsf_host': False,
+        'default_api_host': False,
+        'default_uc_host': False,
+        'default_query_region_host': False,
+        'default_query_region_backup_hosts': False,
+        'default_backup_hosts_retry_times': False,
+        'connection_timeout': False,
+        'connection_retries': False,
+        'connection_pool': False,
+        'default_upload_threshold': False
+    }

tests/cases/test_http/test_qiniu_conf.py

@@ -0,0 +1,122 @@
+import pytest
+import requests
+
+from qiniu.compat import urlencode
+import qiniu.http as qiniu_http
+
+
+@pytest.fixture(scope='function')
+def retry_id(request, mock_server_addr):
+    success_times = []
+    failure_times = []
+    delay = None
+    if hasattr(request, 'param'):
+        success_times = request.param.get('success_times', success_times)
+        failure_times = request.param.get('failure_times', failure_times)
+        delay = request.param.get('delay', None)
+    query_dict = {
+        's': success_times,
+        'f': failure_times,
+        'd': delay
+    }
+    if not query_dict['d']:
+        del query_dict['d']
+    query_params = urlencode(
+        query_dict,
+        doseq=True
+    )
+    request_url = '{scheme}://{host}/retry_me/__mgr__?{query_params}'.format(
+        scheme=mock_server_addr.scheme,
+        host=mock_server_addr.netloc,
+        query_params=query_params
+    )
+    resp = requests.put(request_url)
+    resp.raise_for_status()
+    record_id = resp.text
+    yield record_id
+    request_url = '{scheme}://{host}/retry_me/__mgr__?id={id}'.format(
+        scheme=mock_server_addr.scheme,
+        host=mock_server_addr.netloc,
+        id=record_id
+    )
+    resp = requests.delete(request_url)
+    resp.raise_for_status()
+
+
+@pytest.fixture(scope='function')
+def reset_session():
+    qiniu_http._session = None
+    yield
+
+
+class TestQiniuConf:
+    @pytest.mark.usefixtures('reset_session')
+    @pytest.mark.parametrize(
+        'set_conf_default',
+        [
+            {
+                'connection_timeout': 0.3,
+                'connection_retries': 0
+            }
+        ],
+        indirect=True
+    )
+    @pytest.mark.parametrize(
+        'method,opts',
+        [
+            ('get', {}),
+            ('put', {'data': None, 'files': None}),
+            ('post', {'data': None, 'files': None}),
+            ('delete', {'params': None})
+        ],
+        ids=lambda v: v if type(v) is str else 'opts'
+    )
+    def test_timeout_conf(self, mock_server_addr, method, opts, set_conf_default):
+        request_url = '{scheme}://{host}/timeout?delay=0.5'.format(
+            scheme=mock_server_addr.scheme,
+            host=mock_server_addr.netloc
+        )
+        send = getattr(qiniu_http.qn_http_client, method)
+        _ret, resp = send(request_url, **opts)
+        assert 'Read timed out' in str(resp.exception)
+
+    @pytest.mark.usefixtures('reset_session')
+    @pytest.mark.parametrize(
+        'retry_id',
+        [
+            {
+                'success_times': [0, 1],
+                'failure_times': [5, 0],
+            },
+        ],
+        indirect=True
+    )
+    @pytest.mark.parametrize(
+        'set_conf_default',
+        [
+            {
+                'connection_retries': 5
+            }
+        ],
+        indirect=True
+    )
+    @pytest.mark.parametrize(
+        'method,opts',
+        [
+            # post not retry default, see
+            # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS
+            ('get', {}),
+            ('put', {'data': None, 'files': None}),
+            ('delete', {'params': None})
+        ],
+        ids=lambda v: v if type(v) is str else 'opts'
+    )
+    def test_retry_times(self, retry_id, mock_server_addr, method, opts, set_conf_default):
+        request_url = '{scheme}://{host}/retry_me?id={id}'.format(
+            scheme=mock_server_addr.scheme,
+            host=mock_server_addr.netloc,
+            id=retry_id
+        )
+        send = getattr(qiniu_http.qn_http_client, method)
+        _ret, resp = send(request_url, **opts)
+        assert resp.status_code == 200

tests/cases/test_services/test_storage/test_uploader.py

@@ -4,7 +4,7 @@
 import tempfile
 import pytest
 
-from qiniu.compat import json
+from qiniu.compat import json, is_py2
 from qiniu import (
     Zone,
     etag,
@@ -90,7 +90,7 @@ def temp_file(request):
 
     try:
         os.remove(tmp_file_path)
-    except FileNotFoundError:
+    except Exception:
         pass
 
 
@@ -522,3 +522,58 @@ def test_put_stream_v2_with_metadata(
         assert 'x-qn-meta' in ret
         assert ret['x-qn-meta']['name'] == 'qiniu'
         assert ret['x-qn-meta']['age'] == '18'
+
+    @pytest.mark.parametrize('temp_file', [30 * MB], indirect=True)
+    @pytest.mark.parametrize('version', ['v1', 'v2'])
+    def test_resume_upload(self, bucket_name, qn_auth, temp_file, version):
+        key = 'test_resume_upload_{}'.format(version)
+        size = os.stat(temp_file).st_size
+        part_size = 4 * MB
+
+        def mock_fail(uploaded_size, _total_size):
+            if uploaded_size > 10 * MB:
+                raise Exception('Mock Fail')
+
+        try:
+            with open(temp_file, 'rb') as input_stream:
+                token = qn_auth.upload_token(bucket_name, key)
+                try:
+                    _ret, _into = put_stream(
+                        up_token=token,
+                        key=key,
+                        input_stream=input_stream,
+                        file_name=os.path.basename(temp_file),
+                        data_size=size,
+                        hostscache_dir=None,
+                        part_size=part_size,
+                        version=version,
+                        bucket_name=bucket_name,
+                        progress_handler=mock_fail
+                    )
+                except Exception as e:
+                    if 'Mock Fail' not in str(e):
+                        raise e
+        except IOError:
+            if is_py2:
+                # https://github.com/pytest-dev/pytest/issues/2370
+                # https://github.com/pytest-dev/pytest/pull/3305
+                pass
+
+        def should_start_from_resume(uploaded_size, _total_size):
+            assert uploaded_size // part_size >= 1
+
+        with open(temp_file, 'rb') as input_stream:
+            token = qn_auth.upload_token(bucket_name, key)
+            ret, into = put_stream(
+                up_token=token,
+                key=key,
+                input_stream=input_stream,
+                file_name=os.path.basename(temp_file),
+                data_size=size,
+                hostscache_dir=None,
+                part_size=part_size,
+                version=version,
+                bucket_name=bucket_name,
+                progress_handler=should_start_from_resume
+            )
+            assert ret['key'] == key

tests/mock_server/routes/__init__.py

@@ -1,7 +1,10 @@
-from .timeout import handle_timeout
-from .echo import handle_echo
+from .timeout import *
+from .echo import *
+from .retry_me import *
 
 routes = {
     '/timeout': handle_timeout,
-    '/echo': handle_echo
+    '/echo': handle_echo,
+    '/retry_me': handle_retry_me,
+    '/retry_me/__mgr__': handle_mgr_retry_me,
 }