add test cases for timeout and retries

qiniu/compat.py

@@ -33,7 +33,7 @@
 # ---------
 
 if is_py2:
-    from urlparse import urlparse  # noqa
+    from urlparse import urlparse, urlencode  # noqa
     import StringIO
     StringIO = BytesIO = StringIO.StringIO
 
@@ -60,7 +60,7 @@ def is_seekable(data):
             return False
 
 elif is_py3:
-    from urllib.parse import urlparse  # noqa
+    from urllib.parse import urlparse, urlencode  # noqa
     import io
     StringIO = io.StringIO
     BytesIO = io.BytesIO

tests/cases/test_http/test_qiniu_conf.py

@@ -0,0 +1,111 @@
+import pytest
+import requests
+
+from qiniu.compat import urlencode
+from qiniu import config as qn_config
+import qiniu.http as qiniu_http
+
+
+@pytest.fixture(scope='function')
+def retry_id(request, mock_server_addr):
+    success_times = []
+    failure_times = []
+    delay = None
+    if hasattr(request, 'param'):
+        success_times = request.param.get('success_times', success_times)
+        failure_times = request.param.get('failure_times', failure_times)
+        delay = request.param.get('delay', None)
+    query_dict = {
+        's': success_times,
+        'f': failure_times,
+        'd': delay
+    }
+    if not query_dict['d']:
+        del query_dict['d']
+    query_params = urlencode(
+        query_dict,
+        doseq=True
+    )
+    request_url = '{scheme}://{host}/retry_me/__mgr__?{query_params}'.format(
+        scheme=mock_server_addr.scheme,
+        host=mock_server_addr.netloc,
+        query_params=query_params
+    )
+    resp = requests.put(request_url)
+    resp.raise_for_status()
+    record_id = resp.text
+    yield record_id
+    request_url = '{scheme}://{host}/retry_me/__mgr__?id={id}'.format(
+        scheme=mock_server_addr.scheme,
+        host=mock_server_addr.netloc,
+        id=record_id
+    )
+    resp = requests.delete(request_url)
+    resp.raise_for_status()
+
+
+@pytest.fixture(scope='function')
+def reset_session():
+    qiniu_http._session = None
+    yield
+
+
+class TestQiniuConf:
+    @pytest.mark.usefixtures('reset_session')
+    @pytest.mark.parametrize(
+        'method,opts',
+        [
+            ('get', {}),
+            ('put', {'data': None, 'files': None}),
+            ('post', {'data': None, 'files': None}),
+            ('delete', {'params': None})
+        ],
+        ids=lambda v: v if type(v) is str else 'opts'
+    )
+    def test_timeout_conf(self, mock_server_addr, method, opts):
+        qn_config.set_default(
+            connection_timeout=0.3,
+            connection_retries=0
+        )
+        request_url = '{scheme}://{host}/timeout?delay=0.5'.format(
+            scheme=mock_server_addr.scheme,
+            host=mock_server_addr.netloc
+        )
+        send = getattr(qiniu_http.qn_http_client, method)
+        _ret, resp = send(request_url, **opts)
+        assert 'Read timed out' in str(resp.exception)
+
+    @pytest.mark.usefixtures('reset_session')
+    @pytest.mark.parametrize(
+        'retry_id',
+        [
+            {
+                'success_times': [0, 1],
+                'failure_times': [5, 0],
+            },
+        ],
+        indirect=True
+    )
+    @pytest.mark.parametrize(
+        'method,opts',
+        [
+            # post not retry default, see
+            # https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.Retry.DEFAULT_ALLOWED_METHODS
+            ('get', {}),
+            ('put', {'data': None, 'files': None}),
+            ('delete', {'params': None})
+        ],
+        ids=lambda v: v if type(v) is str else 'opts'
+    )
+    def test_retry_times(self, retry_id, mock_server_addr, method, opts):
+        qn_config.set_default(
+            connection_retries=5
+        )
+        request_url = '{scheme}://{host}/retry_me?id={id}'.format(
+            scheme=mock_server_addr.scheme,
+            host=mock_server_addr.netloc,
+            id=retry_id
+        )
+        send = getattr(qiniu_http.qn_http_client, method)
+        _ret, resp = send(request_url, **opts)
+        assert resp.status_code == 200

tests/mock_server/routes/__init__.py

@@ -1,7 +1,10 @@
-from .timeout import handle_timeout
-from .echo import handle_echo
+from .timeout import *
+from .echo import *
+from .retry_me import *
 
 routes = {
     '/timeout': handle_timeout,
-    '/echo': handle_echo
+    '/echo': handle_echo,
+    '/retry_me': handle_retry_me,
+    '/retry_me/__mgr__': handle_mgr_retry_me,
 }

tests/mock_server/routes/retry_me.py

@@ -0,0 +1,145 @@
+import http
+import random
+import string
+
+from urllib.parse import parse_qs
+
+__failure_record = {}
+
+
+def should_fail_by_times(success_times=None, failure_times=None):
+    """
+    Parameters
+    ----------
+    success_times: list[int], default=[1]
+    failure_times: list[int], default=[0]
+
+    Returns
+    -------
+    Generator[bool, None, None]
+
+    Examples
+    --------
+
+    should_fail_by_times([2], [3])
+        will succeed 2 times and failed 3 times, and loop
+
+    should_fail_by_times([2, 4], [3])
+        will succeed 2 times and failed 3 times,
+        then succeeded 4 times and failed 3 time, and loop
+    """
+    if not success_times:
+        success_times = [1]
+    if not failure_times:
+        failure_times = [0]
+
+    def success_times_gen():
+        while True:
+            for i in success_times:
+                yield i
+
+    def failure_times_gen():
+        while True:
+            for i in failure_times:
+                yield i
+
+    success_times_iter = success_times_gen()
+    fail_times_iter = failure_times_gen()
+
+    while True:
+        success = next(success_times_iter)
+        fail = next(fail_times_iter)
+        for _ in range(success):
+            yield False
+        for _ in range(fail):
+            yield True
+
+
+def handle_mgr_retry_me(method, parsed_uri, request_handler):
+    """
+    Parameters
+    ----------
+    method: str
+        HTTP method
+    parsed_uri: urllib.parse.ParseResult
+        parsed URI
+    request_handler: http.server.BaseHTTPRequestHandler
+        request handler
+    """
+    if method not in ['PUT', 'DELETE']:
+        request_handler.send_response(http.HTTPStatus.METHOD_NOT_ALLOWED)
+        return
+    match method:
+        case 'PUT':
+            # s for success
+            success_times = parse_qs(parsed_uri.query).get('s', [])
+            # f for failure
+            failure_times = parse_qs(parsed_uri.query).get('f', [])
+
+            record_id = ''.join(random.choices(string.ascii_letters, k=16))
+
+            __failure_record[record_id] = should_fail_by_times(
+                success_times=[int(n) for n in success_times],
+                failure_times=[int(n) for n in failure_times]
+            )
+
+            request_handler.send_response(http.HTTPStatus.OK)
+            request_handler.send_header('Content-Type', 'text/plain')
+            request_handler.send_header('X-Reqid', record_id)
+            request_handler.end_headers()
+
+            request_handler.wfile.write(record_id.encode('utf-8'))
+        case 'DELETE':
+            record_id = parse_qs(parsed_uri.query).get('id')
+            if not record_id or not record_id[0]:
+                request_handler.send_response(http.HTTPStatus.BAD_REQUEST)
+                return
+            record_id = record_id[0]
+
+            if record_id in __failure_record:
+                del __failure_record[record_id]
+
+            request_handler.send_response(http.HTTPStatus.NO_CONTENT)
+            request_handler.send_header('X-Reqid', record_id)
+            request_handler.end_headers()
+
+
+def handle_retry_me(method, parsed_uri, request_handler):
+    """
+    Parameters
+    ----------
+    method: str
+        HTTP method
+    parsed_uri: urllib.parse.ParseResult
+        parsed URI
+    request_handler: http.server.BaseHTTPRequestHandler
+        request handler
+    """
+    if method not in []:
+        # all method allowed
+        pass
+    record_id = parse_qs(parsed_uri.query).get('id')
+    if not record_id or not record_id[0]:
+        request_handler.send_response(http.HTTPStatus.BAD_REQUEST)
+        return
+    record_id = record_id[0]
+
+    should_fail = next(__failure_record[record_id])
+
+    if should_fail:
+        request_handler.send_response(-1)
+        request_handler.send_header('Content-Type', 'text/plain')
+        request_handler.send_header('X-Reqid', record_id)
+        request_handler.end_headers()
+
+        resp_body = 'service unavailable'
+        request_handler.wfile.write(resp_body.encode('utf-8'))
+        return
+
+    request_handler.send_response(http.HTTPStatus.OK)
+    request_handler.send_header('Content-Type', 'text/plain')
+    request_handler.send_header('X-Reqid', record_id)
+    request_handler.end_headers()
+
+    resp_body = 'ok'
+    request_handler.wfile.write(resp_body.encode('utf-8'))