From 3cc81313e45b65ae3442ccbb0042794dffe18826 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 12 Nov 2018 10:59:19 +0800 Subject: [PATCH 001/131] add qvmupload --- examples/upload_with_zone.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..218b946a 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -24,7 +24,10 @@ # 要上传文件的本地路径 localfile = 'stat.py' -# 指定固定的zone +# 指定固定的zone 自行指定上传域名及空间源站域名，可用于qvm云主机的内网上传，选择服务端或客户端优化的域名，或http\https上传 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint + zone = Zone( up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', From 8d151f00d663d72ddba082660e2eee4001615c37 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 12 Nov 2018 10:59:19 +0800 Subject: [PATCH 002/131] add qvmupload --- examples/upload_with_qvmzone.py | 40 +++++++++++++++++++++++++++++++++ examples/upload_with_zone.py | 5 ++++- 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 examples/upload_with_qvmzone.py diff --git a/examples/upload_with_qvmzone.py b/examples/upload_with_qvmzone.py new file mode 100644 index 00000000..54f8b603 --- /dev/null +++ b/examples/upload_with_qvmzone.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth, put_file, etag, urlsafe_base64_encode +import qiniu.config +from qiniu import Zone, set_default + +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' + +# 构建鉴权对象 +q = Auth(access_key, secret_key) + +# 要上传的空间 +bucket_name = 'Bucket_Name' + +# 上传到七牛后保存的文件名 +key = 'my-python-logo.png' + +# 生成上传 Token，可以指定过期时间等 +token = q.upload_token(bucket_name, key, 3600) + +# 要上传文件的本地路径 +localfile = 'stat.py' + +# up_host, 指定上传域名，注意不同区域的qvm上传域名不同 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo + +zone = Zone( + up_host='free-qvm-z1-zz.qiniup.com', + up_host_backup='free-qvm-z1-zz.qiniup.com', + io_host='iovip.qbox.me', + scheme='http') +set_default(default_zone=zone) + +ret, info = put_file(token, key, localfile) +print(info) +assert ret['key'] == key +assert ret['hash'] == etag(localfile) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..218b946a 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -24,7 +24,10 @@ # 要上传文件的本地路径 localfile = 'stat.py' -# 指定固定的zone +# 指定固定的zone 自行指定上传域名及空间源站域名，可用于qvm云主机的内网上传，选择服务端或客户端优化的域名，或http\https上传 +# https://developer.qiniu.com/qvm/manual/4269/qvm-kodo +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint + zone = Zone( up_host='uptest.qiniu.com', up_host_backup='uptest.qiniu.com', From 5d59aba9e65b866f5c299ee78cc2e8977985214e Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Fri, 22 Feb 2019 17:26:51 +0800 Subject: [PATCH 003/131] add rs and region --- examples/batch_stat.py | 2 - examples/bucket_info.py | 19 ++++ examples/change_status.py | 29 +++++ examples/list_buckets.py | 25 +++++ examples/mk_bucket.py | 20 ++++ examples/upload_with_zone.py | 32 +++--- qiniu/config.py | 16 +-- qiniu/region.py | 161 ++++++++++++++++++++++++++++ qiniu/services/storage/bucket.py | 40 +++++++ qiniu/services/storage/uploader.py | 20 +++- qiniu/zone.py | 164 +---------------------------- 11 files changed, 341 insertions(+), 187 deletions(-) create mode 100644 examples/bucket_info.py create mode 100644 examples/change_status.py create mode 100644 examples/list_buckets.py create mode 100644 examples/mk_bucket.py create mode 100644 qiniu/region.py diff --git a/examples/batch_stat.py b/examples/batch_stat.py index 4ab7592c..84eea4f3 100644 --- a/examples/batch_stat.py +++ b/examples/batch_stat.py @@ -5,11 +5,9 @@ https://developer.qiniu.com/kodo/api/1250/batch """ - from qiniu import build_batch_stat, Auth, BucketManager access_key = '' - secret_key = '' q = Auth(access_key, secret_key) diff --git a/examples/bucket_info.py b/examples/bucket_info.py new file mode 100644 index 00000000..14ea4bbd --- /dev/null +++ b/examples/bucket_info.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +# 空间名 +bucket_name = 'bucket_name' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.bucket_info(bucket_name) +print(info) diff --git a/examples/change_status.py b/examples/change_status.py new file mode 100644 index 00000000..ae0a6145 --- /dev/null +++ b/examples/change_status.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 空间名 +bucket_name = 'bucket_name' + +# 文件名 +key = 'file_name' + +# 条件匹配，只有匹配上才会执行修改操作 +# cond可以填空，一个或多个 +cond = {"fsize": "186371", + "putTime": "14899798962573916", + "hash": "FiRxWzeeD6ofGTpwTZub5Fx1ozvi", + "mime": "image/png"} + +ret, info = bucket.change_status(bucket_name, key, '0', cond) +print(info) diff --git a/examples/list_buckets.py b/examples/list_buckets.py new file mode 100644 index 00000000..471322ef --- /dev/null +++ b/examples/list_buckets.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议填写 +# z0:只返回华东区域的空间 +# z1:只返回华北区域的空间 +# z2:只返回华南区域的空间 +# na0:只返回北美区域的空间 +# as0:只返回东南亚区域的空间 +region = "z0" + +ret, info = bucket.list_bucket(region) +print(info) +print(ret) diff --git a/examples/mk_bucket.py b/examples/mk_bucket.py new file mode 100644 index 00000000..09035d79 --- /dev/null +++ b/examples/mk_bucket.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写你的 Access Key 和 Secret Key +access_key = '...' +secret_key = '...' + +bucket_name = 'Bucket_Name' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +region = "z0" + +ret, info = bucket.mkbucketv2(bucket_name, region) +print(info) diff --git a/examples/upload_with_zone.py b/examples/upload_with_zone.py index 0c8e56de..c8650d39 100644 --- a/examples/upload_with_zone.py +++ b/examples/upload_with_zone.py @@ -1,38 +1,42 @@ # -*- coding: utf-8 -*- # flake8: noqa -from qiniu import Auth, put_file, etag, urlsafe_base64_encode -import qiniu.config +from qiniu import Auth, put_file from qiniu import Zone, set_default # 需要填写你的 Access Key 和 Secret Key -access_key = '...' -secret_key = '...' +access_key = '' +secret_key = '' # 构建鉴权对象 q = Auth(access_key, secret_key) # 要上传的空间 -bucket_name = 'Bucket_Name' +bucket_name = 'bucket_name' # 上传到七牛后保存的文件名 -key = 'my-python-logo.png' +key = 'a.jpg' # 生成上传 Token，可以指定过期时间等 token = q.upload_token(bucket_name, key, 3600) # 要上传文件的本地路径 -localfile = 'stat.py' +localfile = '/Users/abc/Documents/a.jpg' + +# 指定固定域名的zone,不同区域uphost域名见下文档 +# https://developer.qiniu.com/kodo/manual/1671/region-endpoint +# 未指定或上传错误，sdk会根据token自动查询对应的上传域名 +# *.qiniup.com 支持https上传 +# 备用*.qiniu.com域名 不支持https上传 +# 要求https上传时，如果客户指定的两个host都错误，且sdk自动查询的第一个*.qiniup.com上传域名因意外不可用导致访问到备用*.qiniu.com会报ssl错误 +# 建议https上传时查看上面文档，指定正确的host -# 指定固定的zone zone = Zone( - up_host='uptest.qiniu.com', - up_host_backup='uptest.qiniu.com', - io_host='iovip.qbox.me', - scheme='http') + up_host='https://up.qiniup.com', + up_host_backup='https://upload.qiniup.com', + io_host='http://iovip.qbox.me', + scheme='https') set_default(default_zone=zone) ret, info = put_file(token, key, localfile) print(info) -assert ret['key'] == key -assert ret['hash'] == etag(localfile) diff --git a/qiniu/config.py b/qiniu/config.py index 9b827962..694749eb 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -2,9 +2,10 @@ from qiniu import zone -RS_HOST = 'http://rs.qbox.me' # 管理操作Host -RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host +RS_HOST = 'http://rs.qiniu.com' # 管理操作Host +RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 @@ -13,9 +14,10 @@ 'default_rs_host': RS_HOST, 'default_rsf_host': RSF_HOST, 'default_api_host': API_HOST, - 'connection_timeout': 30, # 链接超时为时间为30s - 'connection_retries': 3, # 链接重试次数为3次 - 'connection_pool': 10, # 链接池个数为10 + 'default_uc_host': UC_HOST, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 } @@ -25,7 +27,7 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, - connection_timeout=None, default_rs_host=None, + connection_timeout=None, default_rs_host=None, default_uc_host=None, default_rsf_host=None, default_api_host=None): if default_zone: _config['default_zone'] = default_zone @@ -35,6 +37,8 @@ def set_default( _config['default_rsf_host'] = default_rsf_host if default_api_host: _config['default_api_host'] = default_api_host + if default_uc_host: + _config['default_uc_host'] = default_api_host if connection_retries: _config['connection_retries'] = connection_retries if connection_pool: diff --git a/qiniu/region.py b/qiniu/region.py new file mode 100644 index 00000000..c87c02c5 --- /dev/null +++ b/qiniu/region.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- + +import os +import time +import requests +from qiniu import compat +from qiniu import utils + +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host + + +class Region(object): + """七牛上传区域类 + + 该类主要内容上传区域地址。 + + """ + + def __init__( + self, + up_host=None, + up_host_backup=None, + io_host=None, + host_cache={}, + scheme="http", + home_dir=os.getcwd()): + """初始化Zone类""" + self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host + self.host_cache = host_cache + self.scheme = scheme + self.home_dir = home_dir + + def get_up_host_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + return up_hosts[0] + + def get_up_host_backup_by_token(self, up_token): + ak, bucket = self.unmarshal_up_token(up_token) + up_hosts = self.get_up_host(ak, bucket) + if (len(up_hosts) <= 1): + up_host = up_hosts[0] + else: + up_host = up_hosts[1] + return up_host + + def get_io_host(self, ak, bucket): + if self.io_host: + return self.io_host + bucket_hosts = self.get_bucket_hosts(ak, bucket) + io_hosts = bucket_hosts['ioHosts'] + return io_hosts[0] + + def get_up_host(self, ak, bucket): + bucket_hosts = self.get_bucket_hosts(ak, bucket) + up_hosts = bucket_hosts['upHosts'] + return up_hosts + + def unmarshal_up_token(self, up_token): + token = up_token.split(':') + if (len(token) != 3): + raise ValueError('invalid up_token') + + ak = token[0] + policy = compat.json.loads( + compat.s( + utils.urlsafe_base64_decode( + token[2]))) + + scope = policy["scope"] + bucket = scope + if (':' in scope): + bucket = scope.split(':')[0] + + return ak, bucket + + def get_bucket_hosts(self, ak, bucket): + key = self.scheme + ":" + ak + ":" + bucket + + bucket_hosts = self.get_bucket_hosts_to_cache(key) + if (len(bucket_hosts) > 0): + return bucket_hosts + + hosts = {} + hosts.update({self.scheme: {}}) + + hosts[self.scheme].update({'up': []}) + hosts[self.scheme].update({'io': []}) + + if self.up_host is not None: + hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) + + if self.up_host_backup is not None: + hosts[self.scheme]['up'].append( + self.scheme + "://" + self.up_host_backup) + + if self.io_host is not None: + hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) + + if len(hosts[self.scheme]) == 0 or self.io_host is None: + hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) + else: + # 1 year + hosts['ttl'] = int(time.time()) + 31536000 + try: + scheme_hosts = hosts[self.scheme] + except KeyError: + raise KeyError( + "Please check your BUCKET_NAME! The UpHosts is %s" % + hosts) + bucket_hosts = { + 'upHosts': scheme_hosts['up'], + 'ioHosts': scheme_hosts['io'], + 'deadline': int(time.time()) + hosts['ttl'] + } + + self.set_bucket_hosts_to_cache(key, bucket_hosts) + return bucket_hosts + + def get_bucket_hosts_to_cache(self, key): + ret = [] + if (len(self.host_cache) == 0): + self.host_cache_from_file() + + if (not (key in self.host_cache)): + return ret + + if (self.host_cache[key]['deadline'] > time.time()): + ret = self.host_cache[key] + + return ret + + def set_bucket_hosts_to_cache(self, key, val): + self.host_cache[key] = val + self.host_cache_to_file() + return + + def host_cache_from_file(self): + path = self.host_cache_file_path() + if not os.path.isfile(path): + return None + with open(path, 'r') as f: + bucket_hosts = compat.json.load(f) + self.host_cache = bucket_hosts + f.close() + return + + def host_cache_file_path(self): + return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") + + def host_cache_to_file(self): + path = self.host_cache_file_path() + with open(path, 'w') as f: + compat.json.dump(self.host_cache, f) + f.close() + + def bucket_hosts(self, ak, bucket): + url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) + ret = requests.get(url) + data = compat.json.dumps(ret.json(), separators=(',', ':')) + return data diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 30e0d100..cb8da771 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -3,6 +3,7 @@ from qiniu import config from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry +import json class BucketManager(object): @@ -225,6 +226,25 @@ def change_type(self, bucket, key, storage_type): resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + def change_status(self, bucket, key, status, cond): + """修改文件的状态 + + 修改文件的存储类型为可用或禁用： + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + storage_type: 待操作资源存储类型，0为启用，1为禁用 + """ + resource = entry(bucket, key) + if cond and isinstance(cond, dict): + condstr = "" + for k,v in cond.items(): + condstr+="{0}={1}&".format(k, v) + condstr = urlsafe_base64_encode(condstr[:-1]) + return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) + return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) + def batch(self, operations): """批量操作: @@ -295,6 +315,26 @@ def mkbucketv2(self, bucket_name, region): bucket_name = urlsafe_base64_encode(bucket_name) return self.__rs_do('mkbucketv2', bucket_name, 'region', region) + def list_bucket(self, region): + """ + 列举存储空间列表 + + Args: + """ + return self.__uc_do('v3/buckets?region={0}'.format(region)) + + def bucket_info(self, bucket_name): + """ + 获取存储空间信息 + + Args: + bucket_name: 存储空间名 + """ + return self.__post('v2/bucketInfo?bucket={}'.format(bucket_name), ) + + def __uc_do(self, operation, *args): + return self.__server_do(config.get_default('default_uc_host'), operation, *args) + def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 4e1a2157..babd8a7b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -92,7 +92,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None fields['key'] = key fields['token'] = up_token - url = config.get_default('default_zone').get_up_host_by_token(up_token) + '/' + if config.get_default('default_zone').up_host: + url = config.get_default('default_zone').up_host + else: + url = config.get_default('default_zone').get_up_host_by_token(up_token) # name = key if key else file_name fname = file_name @@ -106,7 +109,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: - url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + '/' + if config.get_default('default_zone').up_host_backup: + url = config.get_default('default_zone').up_host_backup + else: + url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -190,7 +196,10 @@ def recovery_from_record(self): def upload(self): """上传操作""" self.blockStatus = [] - host = config.get_default('default_zone').get_up_host_by_token(self.up_token) + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) @@ -199,7 +208,10 @@ def upload(self): if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): - host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) + if config.get_default('default_zone').up_host_backup: + host = config.get_default('default_zone').up_host_backup + else: + host = config.get_default('default_zone').get_up_host_backup_by_token(up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: diff --git a/qiniu/zone.py b/qiniu/zone.py index b817f212..acb34d00 100644 --- a/qiniu/zone.py +++ b/qiniu/zone.py @@ -1,165 +1,7 @@ # -*- coding: utf-8 -*- -import os -import time -import requests -from qiniu import compat -from qiniu import utils +from qiniu.region import Region -UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host - -class Zone(object): - """七牛上传区域类 - - 该类主要内容上传区域地址。 - - Attributes: - up_host: 首选上传地址 - up_host_backup: 备用上传地址 - """ - - def __init__( - self, - up_host=None, - up_host_backup=None, - io_host=None, - host_cache={}, - scheme="http", - home_dir=os.getcwd()): - """初始化Zone类""" - self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host - self.host_cache = host_cache - self.scheme = scheme - self.home_dir = home_dir - - def get_up_host_by_token(self, up_token): - ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) - return up_hosts[0] - - def get_up_host_backup_by_token(self, up_token): - ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) - - if (len(up_hosts) <= 1): - up_host = up_hosts[0] - else: - up_host = up_hosts[1] - return up_host - - def get_io_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) - io_hosts = bucket_hosts['ioHosts'] - return io_hosts[0] - - def get_up_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) - up_hosts = bucket_hosts['upHosts'] - return up_hosts - - def unmarshal_up_token(self, up_token): - token = up_token.split(':') - if (len(token) != 3): - raise ValueError('invalid up_token') - - ak = token[0] - policy = compat.json.loads( - compat.s( - utils.urlsafe_base64_decode( - token[2]))) - - scope = policy["scope"] - bucket = scope - if (':' in scope): - bucket = scope.split(':')[0] - - return ak, bucket - - def get_bucket_hosts(self, ak, bucket): - key = self.scheme + ":" + ak + ":" + bucket - - bucket_hosts = self.get_bucket_hosts_to_cache(key) - if (len(bucket_hosts) > 0): - return bucket_hosts - - hosts = {} - hosts.update({self.scheme: {}}) - - hosts[self.scheme].update({'up': []}) - hosts[self.scheme].update({'io': []}) - - if self.up_host is not None: - hosts[self.scheme]['up'].append(self.scheme + "://" + self.up_host) - - if self.up_host_backup is not None: - hosts[self.scheme]['up'].append( - self.scheme + "://" + self.up_host_backup) - - if self.io_host is not None: - hosts[self.scheme]['io'].append(self.scheme + "://" + self.io_host) - - if len(hosts[self.scheme]) == 0 or self.io_host is None: - # print(hosts) - hosts = compat.json.loads(self.bucket_hosts(ak, bucket)) - else: - # 1 year - hosts['ttl'] = int(time.time()) + 31536000 - try: - scheme_hosts = hosts[self.scheme] - except KeyError: - raise KeyError( - "Please check your BUCKET_NAME! The UpHosts is %s" % - hosts) - bucket_hosts = { - 'upHosts': scheme_hosts['up'], - 'ioHosts': scheme_hosts['io'], - 'deadline': int(time.time()) + hosts['ttl'] - } - - self.set_bucket_hosts_to_cache(key, bucket_hosts) - - return bucket_hosts - - def get_bucket_hosts_to_cache(self, key): - ret = [] - if (len(self.host_cache) == 0): - self.host_cache_from_file() - - if (not (key in self.host_cache)): - return ret - - if (self.host_cache[key]['deadline'] > time.time()): - ret = self.host_cache[key] - - return ret - - def set_bucket_hosts_to_cache(self, key, val): - self.host_cache[key] = val - self.host_cache_to_file() - return - - def host_cache_from_file(self): - path = self.host_cache_file_path() - if not os.path.isfile(path): - return None - with open(path, 'r') as f: - bucket_hosts = compat.json.load(f) - self.host_cache = bucket_hosts - f.close() - return - - def host_cache_file_path(self): - return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") - - def host_cache_to_file(self): - path = self.host_cache_file_path() - with open(path, 'w') as f: - compat.json.dump(self.host_cache, f) - f.close() - - def bucket_hosts(self, ak, bucket): - url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) - ret = requests.get(url) - data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data +class Zone(Region): + pass From 6759aff7b26742bca79e0c91f432513173fee8fc Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Fri, 22 Feb 2019 17:34:26 +0800 Subject: [PATCH 004/131] add rs and region --- examples/change_mime.py | 5 +++-- examples/change_status.py | 15 ++++++++------- examples/list_buckets.py | 9 +++++---- examples/upload.py | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/examples/change_mime.py b/examples/change_mime.py index 5db61f8a..ad4294d2 100644 --- a/examples/change_mime.py +++ b/examples/change_mime.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +改变文件的mimeType +""" from qiniu import Auth from qiniu import BucketManager diff --git a/examples/change_status.py b/examples/change_status.py index ae0a6145..1cf526df 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -1,22 +1,23 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +改变文件状态，可用或不可用 +""" from qiniu import Auth from qiniu import BucketManager # 需要填写你的 Access Key 和 Secret Key -access_key = '' -secret_key = '' +access_key = 'C3S6li9F4Iq9toDkq3tzoHz_tRwS2LWNE9aUjIJy' +secret_key = 'VsjH9zF3XdFfeo6K6y-nyEU5ia62NTW4KM7xcwt4' q = Auth(access_key, secret_key) bucket = BucketManager(q) # 空间名 -bucket_name = 'bucket_name' +bucket_name = 'bernie' # 文件名 -key = 'file_name' +key = '233.jpg' # 条件匹配，只有匹配上才会执行修改操作 # cond可以填空，一个或多个 @@ -25,5 +26,5 @@ "hash": "FiRxWzeeD6ofGTpwTZub5Fx1ozvi", "mime": "image/png"} -ret, info = bucket.change_status(bucket_name, key, '0', cond) +ret, info = bucket.change_status(bucket_name, key, '1', cond) print(info) diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 471322ef..5fb4faca 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -# flake8: noqa - +""" +列举账号下的空间 +""" from qiniu import Auth from qiniu import BucketManager @@ -12,13 +13,13 @@ bucket = BucketManager(q) -# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议填写 +# 指定需要列举的区域，填空字符串返回全部空间，为减少响应时间建议不为空 # z0:只返回华东区域的空间 # z1:只返回华北区域的空间 # z2:只返回华南区域的空间 # na0:只返回北美区域的空间 # as0:只返回东南亚区域的空间 -region = "z0" +region = "as0" ret, info = bucket.list_bucket(region) print(info) diff --git a/examples/upload.py b/examples/upload.py index 29a071e7..690ea046 100755 --- a/examples/upload.py +++ b/examples/upload.py @@ -13,7 +13,7 @@ q = Auth(access_key, secret_key) # 要上传的空间 -bucket_name = 'if-bc' +bucket_name = '' # 上传到七牛后保存的文件名 key = 'my-python-七牛.png' From 199a87f3618a26b3405f201e4af2a806385e0ba3 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 25 Feb 2019 14:47:43 +0800 Subject: [PATCH 005/131] adjust format --- examples/batch_copy.py | 3 +++ examples/batch_delete.py | 2 ++ examples/batch_move.py | 2 ++ examples/batch_rename.py | 2 ++ examples/batch_stat.py | 2 ++ examples/cdn_bandwidth.py | 2 ++ examples/cdn_flux.py | 2 ++ examples/cdn_log.py | 2 ++ examples/change_mime.py | 2 ++ examples/change_status.py | 2 ++ examples/create_bucket.py | 2 ++ examples/list_buckets.py | 2 ++ examples/prefetch_to_cdn.py | 1 + examples/refresh_dirs.py | 2 ++ examples/refresh_urls.py | 2 ++ examples/rtc_server.py | 1 + examples/timestamp_url.py | 1 + examples/update_cdn_sslcert.py | 2 ++ qiniu/auth.py | 35 +++++++++++++++--------------- qiniu/services/storage/bucket.py | 5 ++--- qiniu/services/storage/uploader.py | 2 +- 21 files changed, 54 insertions(+), 22 deletions(-) diff --git a/examples/batch_copy.py b/examples/batch_copy.py index f12225a3..6fd15b63 100644 --- a/examples/batch_copy.py +++ b/examples/batch_copy.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +# flake8: noqa + + """ 批量拷贝文件 diff --git a/examples/batch_delete.py b/examples/batch_delete.py index 89ea1180..c74dd951 100644 --- a/examples/batch_delete.py +++ b/examples/batch_delete.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量删除文件 diff --git a/examples/batch_move.py b/examples/batch_move.py index c4f51076..3375e2ea 100644 --- a/examples/batch_move.py +++ b/examples/batch_move.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量移动文件 diff --git a/examples/batch_rename.py b/examples/batch_rename.py index 12e7c806..75a48289 100644 --- a/examples/batch_rename.py +++ b/examples/batch_rename.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量重命名文件 diff --git a/examples/batch_stat.py b/examples/batch_stat.py index 84eea4f3..9ad9d7b0 100644 --- a/examples/batch_stat.py +++ b/examples/batch_stat.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 批量查询文件信息 diff --git a/examples/cdn_bandwidth.py b/examples/cdn_bandwidth.py index c4c4c0ce..ad4e97a8 100644 --- a/examples/cdn_bandwidth.py +++ b/examples/cdn_bandwidth.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 查询指定域名指定时间段内的带宽 """ diff --git a/examples/cdn_flux.py b/examples/cdn_flux.py index c88517e3..bb42efc1 100644 --- a/examples/cdn_flux.py +++ b/examples/cdn_flux.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 查询指定域名指定时间段内的流量 """ diff --git a/examples/cdn_log.py b/examples/cdn_log.py index 2ebb7271..aee1e5c8 100644 --- a/examples/cdn_log.py +++ b/examples/cdn_log.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 获取指定域名指定时间内的日志链接 """ diff --git a/examples/change_mime.py b/examples/change_mime.py index ad4294d2..770405cc 100644 --- a/examples/change_mime.py +++ b/examples/change_mime.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 改变文件的mimeType """ diff --git a/examples/change_status.py b/examples/change_status.py index 1cf526df..b0d90704 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 改变文件状态，可用或不可用 """ diff --git a/examples/create_bucket.py b/examples/create_bucket.py index e77099d7..4b0b02c4 100644 --- a/examples/create_bucket.py +++ b/examples/create_bucket.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 创建存储空间 """ diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 5fb4faca..6c7f7c0c 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 列举账号下的空间 """ diff --git a/examples/prefetch_to_cdn.py b/examples/prefetch_to_cdn.py index 8bdaa0a7..fbab188c 100644 --- a/examples/prefetch_to_cdn.py +++ b/examples/prefetch_to_cdn.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa """ 预取资源到cdn节点 diff --git a/examples/refresh_dirs.py b/examples/refresh_dirs.py index 1db73015..5ce41438 100644 --- a/examples/refresh_dirs.py +++ b/examples/refresh_dirs.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + import qiniu from qiniu import CdnManager diff --git a/examples/refresh_urls.py b/examples/refresh_urls.py index 43b62baa..b4194275 100644 --- a/examples/refresh_urls.py +++ b/examples/refresh_urls.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + import qiniu from qiniu import CdnManager diff --git a/examples/rtc_server.py b/examples/rtc_server.py index 84a39bb3..94b9ebb9 100644 --- a/examples/rtc_server.py +++ b/examples/rtc_server.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa from qiniu import QiniuMacAuth from qiniu import RtcServer, get_room_token diff --git a/examples/timestamp_url.py b/examples/timestamp_url.py index 0b543a78..0873a181 100644 --- a/examples/timestamp_url.py +++ b/examples/timestamp_url.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# flake8: noqa """ 获取一个配置时间戳防盗链的url diff --git a/examples/update_cdn_sslcert.py b/examples/update_cdn_sslcert.py index 06652712..40152f56 100644 --- a/examples/update_cdn_sslcert.py +++ b/examples/update_cdn_sslcert.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +# flake8: noqa + """ 更新cdn证书(可配合let's encrypt 等完成自动证书更新) """ diff --git a/qiniu/auth.py b/qiniu/auth.py index c25ad6d0..fe58f930 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -9,34 +9,33 @@ from .compat import urlparse, json, b from .utils import urlsafe_base64_encode - # 上传策略，参数规格详见 # https://developer.qiniu.com/kodo/manual/1206/put-policy _policy_fields = set([ - 'callbackUrl', # 回调URL - 'callbackBody', # 回调Body - 'callbackHost', # 回调URL指定的Host + 'callbackUrl', # 回调URL + 'callbackBody', # 回调Body + 'callbackHost', # 回调URL指定的Host 'callbackBodyType', # 回调Body的Content-Type 'callbackFetchKey', # 回调FetchKey模式开关 - 'returnUrl', # 上传端的303跳转URL - 'returnBody', # 上传端简单反馈获取的Body + 'returnUrl', # 上传端的303跳转URL + 'returnBody', # 上传端简单反馈获取的Body - 'endUser', # 回调时上传端标识 - 'saveKey', # 自定义资源名 - 'insertOnly', # 插入模式开关 + 'endUser', # 回调时上传端标识 + 'saveKey', # 自定义资源名 + 'insertOnly', # 插入模式开关 - 'detectMime', # MimeType侦测开关 - 'mimeLimit', # MimeType限制 - 'fsizeLimit', # 上传文件大小限制 - 'fsizeMin', # 上传文件最少字节数 + 'detectMime', # MimeType侦测开关 + 'mimeLimit', # MimeType限制 + 'fsizeLimit', # 上传文件大小限制 + 'fsizeMin', # 上传文件最少字节数 - 'persistentOps', # 持久化处理操作 + 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL - 'persistentPipeline', # 持久化处理独享队列 - 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 - 'isPrefixalScope' # 指定上传文件必须使用的前缀 + 'persistentPipeline', # 持久化处理独享队列 + 'deleteAfterDays', # 文件多少天后自动删除 + 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 + 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index cb8da771..2d12860e 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -3,7 +3,6 @@ from qiniu import config from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry -import json class BucketManager(object): @@ -239,8 +238,8 @@ def change_status(self, bucket, key, status, cond): resource = entry(bucket, key) if cond and isinstance(cond, dict): condstr = "" - for k,v in cond.items(): - condstr+="{0}={1}&".format(k, v) + for k, v in cond.items(): + condstr += "{0}={1}&".format(k, v) condstr = urlsafe_base64_encode(condstr[:-1]) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index babd8a7b..1922e4c2 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -211,7 +211,7 @@ def upload(self): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: - host = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: From 7c752190b92a7ffbae867e88845be95686d1be63 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 25 Feb 2019 15:08:48 +0800 Subject: [PATCH 006/131] see env --- test_qiniu.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index 1a42e6fb..544dbc38 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -107,6 +107,8 @@ def test_verify_callback(self): class BucketTestCase(unittest.TestCase): + print(access_key) + print(secret_key) q = Auth(access_key, secret_key) bucket = BucketManager(q) From 321b1a42b9da496c0e6b8767f118418681011f7f Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 25 Feb 2019 16:00:17 +0800 Subject: [PATCH 007/131] add global secrue --- .travis.yml | 20 ++++++++++---------- test_qiniu.py | 2 -- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 45775088..f1927ac1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ sudo: false language: python python: -- '2.6.9' +- 2.6.9 - '2.7' - '3.4' - '3.5' @@ -17,13 +17,13 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 + .; fi - py.test --cov qiniu -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi -deploy: - provider: pypi - user: qiniusdk - password: - secure: N2u9xzhncbziIhoDdpaCcr7D3lW/N7AOIZDpx+M5QW0lPqIXkZDioOTZ7b4QNwx/XFMu6tdeK79A2Wg7T9/8VfEWDd2bYL7a1J7spoFJi9k3HVHHiFBmg7vXr1OGn3D51xqsrq3Kh9uRP150a5CA2qxYabKb6b6dn5QhOTPhfFY= - on: - tags: true +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; + fi + +env: + global: + secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= + QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index 544dbc38..1a42e6fb 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -107,8 +107,6 @@ def test_verify_callback(self): class BucketTestCase(unittest.TestCase): - print(access_key) - print(secret_key) q = Auth(access_key, secret_key) bucket = BucketManager(q) From ef80b62fd3e9c249a905260d5d59e4ff4337e232 Mon Sep 17 00:00:00 2001 From: bernieyangmh Date: Mon, 25 Feb 2019 16:20:32 +0800 Subject: [PATCH 008/131] version to 7.2.3 --- CHANGELOG.md | 6 ++++++ examples/change_status.py | 4 ++-- examples/list_buckets.py | 2 +- qiniu/__init__.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 148fc69f..66aa1c37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +# 7.2.3 (2019-02-25) +* 新增region类，zone继承 +* 上传可以指定上传域名 +* 新增上传指定上传空间和qvm指定上传内网的例子 +* 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 + # 7.2.2 (2018-05-10) * 增加连麦rtc服务端API功能 diff --git a/examples/change_status.py b/examples/change_status.py index b0d90704..5f1acdd5 100644 --- a/examples/change_status.py +++ b/examples/change_status.py @@ -8,8 +8,8 @@ from qiniu import BucketManager # 需要填写你的 Access Key 和 Secret Key -access_key = 'C3S6li9F4Iq9toDkq3tzoHz_tRwS2LWNE9aUjIJy' -secret_key = 'VsjH9zF3XdFfeo6K6y-nyEU5ia62NTW4KM7xcwt4' +access_key = '' +secret_key = '' q = Auth(access_key, secret_key) diff --git a/examples/list_buckets.py b/examples/list_buckets.py index 6c7f7c0c..e83589f7 100644 --- a/examples/list_buckets.py +++ b/examples/list_buckets.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa - + """ 列举账号下的空间 """ diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f229397a..f123ff70 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.2' +__version__ = '7.2.3' from .auth import Auth, QiniuMacAuth From 462b1891d54762257d0bad365048f68c1ee61fe6 Mon Sep 17 00:00:00 2001 From: Cai Xiaohua Date: Wed, 20 Mar 2019 22:53:18 +0800 Subject: [PATCH 009/131] turn json null to json empty object Change-Id: I3e566bb6f4c425922c5940983c427988eee30565 Signed-off-by: Cai Xiaohua --- qiniu/http.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/http.py b/qiniu/http.py index 80596700..14a422f4 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -24,6 +24,8 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' ret = resp.json(encoding='utf-8') if resp.text != '' else {} + if ret is None: # json null + ret = {} return ret, ResponseInfo(resp) From d3dcc507a04795a6e930f5959d2678439ad3371f Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Mon, 1 Apr 2019 19:28:54 +0800 Subject: [PATCH 010/131] Update __init__.py add Region --- qiniu/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index f123ff70..fc7365e1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -15,6 +15,7 @@ from .config import set_default from .zone import Zone +from .region import Region from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete From a69fbef4e3e6ea1ebe09f4610a5b18bb2c17de59 Mon Sep 17 00:00:00 2001 From: longbai Date: Wed, 3 Apr 2019 21:49:17 +0800 Subject: [PATCH 011/131] 7.2.4 --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66aa1c37..0ba182a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.4 (2019-04-01) +* 默认导入region类 + # 7.2.3 (2019-02-25) * 新增region类，zone继承 * 上传可以指定上传域名 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index fc7365e1..4ff86c51 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.3' +__version__ = '7.2.4' from .auth import Auth, QiniuMacAuth From f582d6463db7c154a273e881909a7237bdd0ed48 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Thu, 6 Jun 2019 22:08:01 +0800 Subject: [PATCH 012/131] add sms_ch --- CHANGELOG.md | 3 + examples/sms_test.py | 98 ++++++++++++++++ qiniu/__init__.py | 5 +- qiniu/auth.py | 6 +- qiniu/http.py | 46 +++++++- qiniu/services/sms/__init__.py | 0 qiniu/services/sms/sms.py | 207 +++++++++++++++++++++++++++++++++ 7 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 examples/sms_test.py create mode 100644 qiniu/services/sms/__init__.py create mode 100644 qiniu/services/sms/sms.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ba182a3..4562d660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.5 (2019-06-06) +* 添加sms + # 7.2.4 (2019-04-01) * 默认导入region类 diff --git a/examples/sms_test.py b/examples/sms_test.py new file mode 100644 index 00000000..05d5b957 --- /dev/null +++ b/examples/sms_test.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import QiniuMacAuth +from qiniu import Sms + + +access_key = 'bjtWBQXrcxgo7HWwlC_bgHg81j352_GhgBGZPeOW' +secret_key = 'pCav6rTslxP2SIFg0XJmAw53D9PjWEcuYWVdUqAf' + +# 初始化Auth状态 +q = QiniuMacAuth(access_key, secret_key) + +# 初始化Sms +sms = Sms(q) + +""" +#创建签名 +signature = 'abs' +source = 'website' +req, info = sms.createSignature(signature, source) +print(req,info) +""" + +""" +#查询签名 +audit_status = '' +page = 1 +page_size = 20 +req, info = sms.querySignature(audit_status, page, page_size) +print(req, info) +""" + +""" +编辑签名 +id = 1136530250662940672 +signature = 'sssss' +req, info = sms.updateSignature(id, signature) +print(req, info) +""" + +""" +#删除签名 +signature_id= 1136530250662940672 +req, info = sms.deleteSignature(signature_id) +print(req, info) +""" + +""" +#创建模版 +name = '06-062-test' +template = '${test}' +type = 'notification' +description = '就测试啊' +signature_id = '1131464448834277376' +req, info = sms.createTemplate(name, template, type, description, signature_id) +print(req, info) +""" + +""" +#查询模版 +audit_status = '' +page = 1 +page_size = 20 +req, info = sms.queryTemplate(audit_status, page, page_size) +print(req, info) +""" + +""" +#编辑模版 +template_id = '1136589777022226432' +name = '06-06-test' +template = 'hi,你好' +description = '就测试啊' +signature_id = '1131464448834277376' +req, info = sms.updateTemplate(template_id, name, template, description, signature_id) +print(info) +""" + +""" +#删除模版 +template_id = '1136589777022226432' +req, info = sms.deleteTemplate(template_id) +print(req, info) +""" + +""" +#发送短信 +""" +template_id = '' +mobiles = [] +parameters = {} +req, info = sms.sendMessage(template_id, mobiles, parameters) +print(req, info) + + + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 4ff86c51..61854789 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.4' +__version__ = '7.2.5' from .auth import Auth, QiniuMacAuth @@ -25,7 +25,6 @@ from .services.processing.cmd import build_op, pipe_cmd, op_save from .services.compute.app import AccountClient from .services.compute.qcos_api import QcosClient - +from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token - from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/auth.py b/qiniu/auth.py index fe58f930..d8d8ddd0 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -3,9 +3,7 @@ import hmac import time from hashlib import sha1 - from requests.auth import AuthBase - from .compat import urlparse, json, b from .utils import urlsafe_base64_encode @@ -266,8 +264,8 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - data += body.decode(encoding='UTF-8') - + # data += body.decode(encoding='UTF-8') + data += body return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): diff --git a/qiniu/http.py b/qiniu/http.py index 14a422f4..ff0089e7 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -9,6 +9,7 @@ import qiniu.auth from . import __version__ + _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -24,7 +25,7 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' ret = resp.json(encoding='utf-8') if resp.text != '' else {} - if ret is None: # json null + if ret is None: # json null ret = {} return ret, ResponseInfo(resp) @@ -110,6 +111,10 @@ def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _post_with_qiniu_mac_and_headers(url, data, auth, headers): + return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) + + def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) @@ -118,11 +123,14 @@ def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _put_with_qiniu_mac_and_headers(url, data, auth, headers): + return _put(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) + + def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth) if auth is not None else None timeout = config.get_default('connection_timeout') - try: r = requests.post( url, @@ -148,6 +156,23 @@ def _get_with_qiniu_mac(url, params, auth): return __return_wrapper(r) +def _get_with_qiniu_mac_and_headers(url, params, auth, headers): + try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + r = requests.get( + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=post_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + def _delete_with_qiniu_mac(url, params, auth): try: r = requests.delete( @@ -161,6 +186,23 @@ def _delete_with_qiniu_mac(url, params, auth): return __return_wrapper(r) +def _delete_with_qiniu_mac_and_headers(url, params, auth, headers): + try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) + r = requests.delete( + url, + params=params, + auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, + timeout=config.get_default('connection_timeout'), + headers=post_headers) + except Exception as e: + return None, ResponseInfo(None, e) + return __return_wrapper(r) + + class ResponseInfo(object): """七牛HTTP请求返回信息类 diff --git a/qiniu/services/sms/__init__.py b/qiniu/services/sms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiniu/services/sms/sms.py b/qiniu/services/sms/sms.py new file mode 100644 index 00000000..310c9fe7 --- /dev/null +++ b/qiniu/services/sms/sms.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- + +from qiniu import http +import json + + +class Sms(object): + def __init__(self, auth): + self.auth = auth + self.server = 'https://sms.qiniuapi.com' + + def createSignature(self, signature, source, pics=None): + """ + *创建签名 + *signature: string类型，必填，【长度限制8个字符内】超过长度会报错 + *source: string类型，必填，申请签名时必须指定签名来源。取值范围为： + enterprises_and_institutions 企事业单位的全称或简称 + website 工信部备案网站的全称或简称 + app APP应用的全称或简称 + public_number_or_small_program 公众号或小程序的全称或简称 + store_name 电商平台店铺名的全称或简称 + trade_name 商标名的全称或简称 + *pics: 签名对应的资质证明图片进行 base64 编码格式转换后的字符串 + * @ return: 类型array + { + "signature_id": < signature_id > + } + """ + req = {} + req['signature'] = signature + req['source'] = source + if pics: + req['pics'] = pics + body = json.dumps(req) + url = '{0}/v1/signature'.format(self.server) + return self.__post(url, body) + + def querySignature(self, audit_status=None, page=1, page_size=20): + """ + 查询签名 + * audit_status: 审核状态 string 类型，可选，取值范围为: "passed"(通过), "rejected"(未通过), "reviewing"(审核中) + * page:页码 int 类型， + * page_size: 分页大小 int 类型，可选， 默认为20 + *@return: 类型array { + "items": [{ + "id": string, + "signature": string, + "source": string, + "audit_status": string, + "reject_reason": string, + "created_at": int64, + "updated_at": int64 + }...], + "total": int, + "page": int, + "page_size": int, + } + """ + url = '{}/v1/signature'.format(self.server) + if audit_status: + url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + else: + url = '{}?page={}&page_size={}'.format(url, page, page_size) + return self.__get(url) + + def updateSignature(self, id, signature): + """ + 编辑签名 + * id 签名id : string 类型，必填， + * signature: string 类型，必填， + request 类型array { + "signature": string + } + :return: + """ + url = '{}/v1/signature/{}'.format(self.server, id) + req = {} + req['signature'] = signature + body = json.dumps(req) + return self.__put(url, body) + + def deleteSignature(self, id): + + """ + 删除辑签名 + * id 签名id : string 类型，必填， + * @retrun : 请求成功 HTTP 状态码为 200 + + """ + url = '{}/v1/signature/{}'.format(self.server, id) + return self.__delete(url) + + def createTemplate(self, name, template, type, description, signature_id): + """ + 创建模版 + :param name: 模板名称 string 类型 ，必填 + :param template: 模板内容 string 类型，必填 + :param type: 模板类型 string 类型，必填， + 取值范围为: notification (通知类短信), verification (验证码短信), marketing (营销类短信) + :param description: 申请理由简述 string 类型，必填 + :param signature_id: 已经审核通过的签名 string 类型，必填 + :return: 类型 array { + "template_id": string + } + """ + url = '{}/v1/template'.format(self.server) + req = {} + req['name'] = name + req['template'] = template + req['type'] = type + req['description'] = description + req['signature_id'] = signature_id + body = json.dumps(req) + return self.__post(url, body) + + def queryTemplate(self, audit_status, page=1, page_size=20): + """ + 查询模版 + :param audit_status: 审核状态, 取值范围为: passed (通过), rejected (未通过), reviewing (审核中) + :param page: 页码。默认为 1 + :param page_size: 分页大小。默认为 20 + :return:{ + "items": [{ + "id": string, + "name": string, + "template": string, + "audit_status": string, + "reject_reason": string, + "type": string, + "signature_id": string, // 模版绑定的签名ID + "signature_text": string, // 模版绑定的签名内容 + "created_at": int64, + "updated_at": int64 + }...], + "total": int, + "page": int, + "page_size": int + } + """ + url = '{}/v1/template'.format(self.server) + if audit_status: + url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + else: + url = '{}?page={}&page_size={}'.format(url, page, page_size) + return self.__get(url) + + def updateTemplate(self, id, name, template, description, signature_id): + """ + 更新模版 + :param id: template_id + :param name: 模板名称 string 类型 ，必填 + :param template: 模板内容 string 类型，必填 + :param description: 申请理由简述 string 类型，必填 + :param signature_id: 已经审核通过的签名 string 类型，必填 + :return: 请求成功 HTTP 状态码为 200 + """ + url = '{}/v1/template/{}'.format(self.server, id) + req = {} + req['name'] = name + req['template'] = template + req['description'] = description + req['signature_id'] = signature_id + body = json.dumps(req) + return self.__put(url, body) + + def deleteTemplate(self, id): + """ + 删除模版 + :param id: template_id + :return: 请求成功 HTTP 状态码为 200 + """ + url = '{}/v1/template/{}'.format(self.server, id) + return self.__delete(url) + + def sendMessage(self, template_id, mobiles, parameters): + """ + 发送短信 + :param template_id: 模板 ID + :param mobiles: 手机号 + :param parameters: 自定义魔法变量，变量设置在创建模板时，参数template指定 + :return:{ + "job_id": string + } + """ + url = '{}/v1/message'.format(self.server) + req = {} + req['template_id'] = template_id + req['mobiles'] = mobiles + req['parameters'] = parameters + body = json.dumps(req) + return self.__post(url, body) + + def __post(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._post_with_qiniu_mac_and_headers(url, data, self.auth, headers) + + def __get(self, url, params=None): + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + return http._get_with_qiniu_mac_and_headers(url, params, self.auth, headers) + + def __put(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._put_with_qiniu_mac_and_headers(url, data, self.auth, headers) + + def __delete(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._delete_with_qiniu_mac_and_headers(url, data, self.auth, headers) From 2547fb75575d13912d8fc0cfe4118384259ffedd Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Mon, 10 Jun 2019 09:48:27 +0800 Subject: [PATCH 013/131] add sms_ch,rm ak,sk --- examples/sms_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/sms_test.py b/examples/sms_test.py index 05d5b957..bb7c1a51 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -5,8 +5,8 @@ from qiniu import Sms -access_key = 'bjtWBQXrcxgo7HWwlC_bgHg81j352_GhgBGZPeOW' -secret_key = 'pCav6rTslxP2SIFg0XJmAw53D9PjWEcuYWVdUqAf' +access_key = '' +secret_key = '' # 初始化Auth状态 q = QiniuMacAuth(access_key, secret_key) From f047e99b926db9dd3805974b3ac6355d81d45c21 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 26 Jun 2019 11:36:16 +0800 Subject: [PATCH 014/131] Update setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1190c4f3..a71d6857 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ 'qiniu.services.processing', 'qiniu.services.compute', 'qiniu.services.cdn', + 'qiniu.services.sms', 'qiniu.services.pili', ] From 2d1e3b19a707eea1c34c97977f8480e9d7fccd29 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 26 Jun 2019 11:36:45 +0800 Subject: [PATCH 015/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 61854789..8f009b11 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.5' +__version__ = '7.2.6' from .auth import Auth, QiniuMacAuth From 8f138c064efae9d95e8ba279010a83a48f2b06bc Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 26 Jun 2019 11:37:25 +0800 Subject: [PATCH 016/131] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4562d660..484ff61f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.6(2019-06-26) +* 添加sms + # 7.2.5 (2019-06-06) * 添加sms From c6b1a99bd9182481eef9e17880ddeb2ccac02d4b Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 8 Nov 2019 14:51:31 +0800 Subject: [PATCH 017/131] Update bucket.py --- qiniu/services/storage/bucket.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 2d12860e..20148202 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -302,17 +302,15 @@ def delete_after_days(self, bucket, key, days): resource = entry(bucket, key) return self.__rs_do('deleteAfterDays', resource, days) - def mkbucketv2(self, bucket_name, region): + def mkbucketv3(self, bucket_name, region): """ - 创建存储空间 - https://developer.qiniu.com/kodo/api/1382/mkbucketv2 + 创建存储空间，全局唯一，其他账号有同名空间就无法创建 Args: bucket_name: 存储空间名 region: 存储区域 """ - bucket_name = urlsafe_base64_encode(bucket_name) - return self.__rs_do('mkbucketv2', bucket_name, 'region', region) + return self.__rs_do('mkbucketv3', bucket_name, 'region', region) def list_bucket(self, region): """ From ffd2a3c09322cdf5c7e0977fd4054d5f7565088d Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Wed, 4 Dec 2019 12:02:11 +0800 Subject: [PATCH 018/131] Update auth.py --- qiniu/auth.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index d8d8ddd0..3511209e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -264,8 +264,10 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - # data += body.decode(encoding='UTF-8') - data += body + if isinstance(body,bytes): + data += body.decode(encoding='UTF-8') + else: + data += body return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): From 0c8cca4da40f176f1f3c6ac384cf99852f7ca3b5 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 10 Mar 2020 18:56:18 +0800 Subject: [PATCH 019/131] fix bucket_info --- qiniu/services/storage/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 20148202..ada1a2ca 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -327,7 +327,7 @@ def bucket_info(self, bucket_name): Args: bucket_name: 存储空间名 """ - return self.__post('v2/bucketInfo?bucket={}'.format(bucket_name), ) + return self.__uc_do('v2/bucketInfo?bucket={}'.format(bucket_name), ) def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) From cfdf5960f20c6794b7edad25bd3919baf0e48a7b Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Tue, 10 Mar 2020 19:20:54 +0800 Subject: [PATCH 020/131] fix bucket_info --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 484ff61f..82a46d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# 7.2.7(2020-03-10) +* fix bucket_info + # 7.2.6(2019-06-26) * 添加sms diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 8f009b11..688b1eba 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.6' +__version__ = '7.2.7' from .auth import Auth, QiniuMacAuth From 804eedf00d0b974c13adf2ea3dd1f769b6f79c75 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 27 Mar 2020 15:26:55 +0800 Subject: [PATCH 021/131] add restoreAr --- CHANGELOG.md | 2 ++ examples/restorear.py | 17 +++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/services/storage/bucket.py | 16 +++++++++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 examples/restorear.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 82a46d4c..60e4135d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Changelog +# 7.2.8(2020-03-27) +* add restoreAr # 7.2.7(2020-03-10) * fix bucket_info diff --git a/examples/restorear.py b/examples/restorear.py new file mode 100644 index 00000000..794c41f3 --- /dev/null +++ b/examples/restorear.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +from qiniu import Auth +from qiniu import BucketManager + + +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) +bucket = BucketManager(q) +bucket_name = '13' +key = 'fb8539c39f65d74b4e70db9133c1e9d5.mp4' +ret,info = bucket.restoreAr(bucket_name,key,3) +print(ret) +print(info) + diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 688b1eba..7b6b55f1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.7' +__version__ = '7.2.8' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index ada1a2ca..ce769b1a 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -220,11 +220,25 @@ def change_type(self, bucket, key, storage_type): Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为普通存储，1为低频存储 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储 """ resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) + def restoreAr(self, bucket, key, freezeAfter_days): + """解冻归档存储文件 + + 修改文件的存储类型为普通存储或者是低频存储，参考文档： + https://developer.qiniu.com/kodo/api/6380/restore-archive + + Args: + bucket: 待操作资源所在空间 + key: 待操作资源文件名 + freezeAfter_days: 解冻有效时长，取值范围 1～7 + """ + resource = entry(bucket, key) + return self.__rs_do('restoreAr', resource, 'freezeAfterDays/{0}'.format(freezeAfter_days)) + def change_status(self, bucket, key, status, cond): """修改文件的状态 From 6efed9280d49b1f579e03758445bc04a4ba94bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=9F=E6=A5=9A=E5=90=9B?= Date: Thu, 23 Jul 2020 12:06:39 +0800 Subject: [PATCH 022/131] test travis --- .travis.yml | 15 +++++++-------- qiniu/auth.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index f1927ac1..248d7584 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,9 @@ sudo: false language: python python: -- 2.6.9 -- '2.7' -- '3.4' -- '3.5' +- "2.7" +- "3.4" +- "3.5" install: - pip install flake8 - pip install pytest @@ -17,13 +16,13 @@ before_script: - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 - .; fi +- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; + fi - py.test --cov qiniu - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi env: global: - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file + - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= + - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file diff --git a/qiniu/auth.py b/qiniu/auth.py index 3511209e..1374d4a6 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -264,7 +264,7 @@ def token_of_request( data += "\n" if content_type and content_type != "application/octet-stream" and body: - if isinstance(body,bytes): + if isinstance(body, bytes): data += body.decode(encoding='UTF-8') else: data += body From aeab3c771d34643fc736a126fc5e83956cb1396f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=92=9F=E6=A5=9A=E5=90=9B?= Date: Thu, 23 Jul 2020 14:59:35 +0800 Subject: [PATCH 023/131] remove the 2.6 version --- .travis.yml | 11 +++++++++-- README.md | 8 +++++--- examples/sms_test.py | 6 +++--- test_qiniu.py | 2 +- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 248d7584..757d4525 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,19 +10,26 @@ install: - pip install pytest-cov - pip install requests - pip install scrutinizer-ocular +- pip install codecov + before_script: - export QINIU_TEST_BUCKET="pythonsdk" - export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" - export QINIU_TEST_ENV="travis" - export PYTHONPATH="$PYTHONPATH:." + script: - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; fi - py.test --cov qiniu - if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; fi +- coverage run test_qiniu.py env: global: - - secure: McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs= - - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 \ No newline at end of file + - secure: "McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs=" + - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 + +after_success: + - codecov \ No newline at end of file diff --git a/README.md b/README.md index 2439b8eb..4c81e19a 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,12 @@ [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) [![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) +[![GitHub release](https://img.shields.io/github/v/tag/qiniu/python-sdk.svg?label=release)](https://github.com/qiniu/python-sdk/releases) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) -[![Code Coverage](https://scrutinizer-ci.com/g/qiniu/python-sdk/badges/coverage.png?b=master)](https://scrutinizer-ci.com/g/qiniu/python-sdk/?branch=master) +[![Coverage Status](https://codecov.io/gh/qiniu/python-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/python-sdk) + ## 安装 通过pip @@ -19,8 +21,8 @@ $ pip install qiniu | Qiniu SDK版本 | Python 版本 | |:--------------------:|:---------------------------:| -| 7.x | 2.6, 2.7, 3.3, 3.4, 3.5| -| 6.x | 2.6, 2.7 | +| 7.x | 2.7, 3.3, 3.4, 3.5| +| 6.x | 2.7 | ## 使用方法 diff --git a/examples/sms_test.py b/examples/sms_test.py index bb7c1a51..3f72cf4d 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -3,10 +3,10 @@ from qiniu import QiniuMacAuth from qiniu import Sms +import os - -access_key = '' -secret_key = '' +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') # 初始化Auth状态 q = QiniuMacAuth(access_key, secret_key) diff --git a/test_qiniu.py b/test_qiniu.py index 1a42e6fb..ad23ef80 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -413,7 +413,7 @@ class DownloadTestCase(unittest.TestCase): def test_private_url(self): private_bucket = 'private-res' private_key = 'gogopher.jpg' - base_url = 'http://%s/%s' % (private_bucket + '.qiniudn.com', private_key) + base_url = 'http://%s/%s' % (private_bucket + '.qiniupkg.com', private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) From 1fde5f0e827882d52994898631072a0d778b4ddd Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:02:06 +0800 Subject: [PATCH 024/131] add no json response processing and update _get --- qiniu/http.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index ff0089e7..a562b2f8 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -9,7 +9,6 @@ import qiniu.auth from . import __version__ - _sys_info = '{0}; {1}'.format(platform.system(), platform.machine()) _python_ver = platform.python_version() @@ -74,8 +73,10 @@ def _put(url, data, files, auth, headers=None): def _get(url, params, auth): + if _session is None: + _init() try: - r = requests.get( + r = _session.get( url, params=params, auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, @@ -232,11 +233,14 @@ def __init__(self, response, exception=None): self.req_id = response.headers.get('X-Reqid') self.x_log = response.headers.get('X-Log') if self.status_code >= 400: - ret = response.json() if response.text != '' else None - if ret is None or ret['error'] is None: - self.error = 'unknown' + if 600 > self.status_code >= 499: + self.error = response.text else: - self.error = ret['error'] + ret = response.json() if response.text != '' else None + if ret is None or ret['error'] is None: + self.error = 'unknown' + else: + self.error = ret['error'] if self.req_id is None and self.status_code == 200: self.error = 'server is not qiniu' From 6f1fdd0b0469f3cc2a649b17e18410e462607a03 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:09:24 +0800 Subject: [PATCH 025/131] add .qiniu_pythonsdk_hostscache.json file ptah parameter and update fetch returns info --- qiniu/region.py | 51 ++++++++++++--------- qiniu/services/storage/bucket.py | 24 +++++----- qiniu/services/storage/uploader.py | 72 ++++++++++++++++-------------- test_qiniu.py | 11 +++-- 4 files changed, 89 insertions(+), 69 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index c87c02c5..f143795b 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -22,37 +22,42 @@ def __init__( up_host_backup=None, io_host=None, host_cache={}, - scheme="http", - home_dir=os.getcwd()): + home_dir=None, + scheme="http"): """初始化Zone类""" - self.up_host, self.up_host_backup, self.io_host = up_host, up_host_backup, io_host + self.up_host, self.up_host_backup, self.io_host, self.home_dir = up_host, up_host_backup, io_host, home_dir self.host_cache = host_cache self.scheme = scheme - self.home_dir = home_dir - def get_up_host_by_token(self, up_token): + def get_up_host_by_token(self, up_token, home_dir): ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + up_hosts = self.get_up_host(ak, bucket, home_dir) return up_hosts[0] - def get_up_host_backup_by_token(self, up_token): + def get_up_host_backup_by_token(self, up_token, home_dir): ak, bucket = self.unmarshal_up_token(up_token) - up_hosts = self.get_up_host(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + up_hosts = self.get_up_host(ak, bucket, home_dir) if (len(up_hosts) <= 1): up_host = up_hosts[0] else: up_host = up_hosts[1] return up_host - def get_io_host(self, ak, bucket): + def get_io_host(self, ak, bucket, home_dir): if self.io_host: return self.io_host - bucket_hosts = self.get_bucket_hosts(ak, bucket) + if home_dir is None: + home_dir = os.getcwd() + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) io_hosts = bucket_hosts['ioHosts'] return io_hosts[0] - def get_up_host(self, ak, bucket): - bucket_hosts = self.get_bucket_hosts(ak, bucket) + def get_up_host(self, ak, bucket, home_dir): + bucket_hosts = self.get_bucket_hosts(ak, bucket, home_dir) up_hosts = bucket_hosts['upHosts'] return up_hosts @@ -74,10 +79,10 @@ def unmarshal_up_token(self, up_token): return ak, bucket - def get_bucket_hosts(self, ak, bucket): + def get_bucket_hosts(self, ak, bucket, home_dir): key = self.scheme + ":" + ak + ":" + bucket - bucket_hosts = self.get_bucket_hosts_to_cache(key) + bucket_hosts = self.get_bucket_hosts_to_cache(key, home_dir) if (len(bucket_hosts) > 0): return bucket_hosts @@ -113,14 +118,14 @@ def get_bucket_hosts(self, ak, bucket): 'ioHosts': scheme_hosts['io'], 'deadline': int(time.time()) + hosts['ttl'] } - - self.set_bucket_hosts_to_cache(key, bucket_hosts) + home_dir = "" + self.set_bucket_hosts_to_cache(key, bucket_hosts, home_dir) return bucket_hosts - def get_bucket_hosts_to_cache(self, key): + def get_bucket_hosts_to_cache(self, key, home_dir): ret = [] if (len(self.host_cache) == 0): - self.host_cache_from_file() + self.host_cache_from_file(home_dir) if (not (key in self.host_cache)): return ret @@ -130,12 +135,14 @@ def get_bucket_hosts_to_cache(self, key): return ret - def set_bucket_hosts_to_cache(self, key, val): + def set_bucket_hosts_to_cache(self, key, val, home_dir): self.host_cache[key] = val - self.host_cache_to_file() + self.host_cache_to_file(home_dir) return - def host_cache_from_file(self): + def host_cache_from_file(self, home_dir): + if home_dir is not None: + self.home_dir = home_dir path = self.host_cache_file_path() if not os.path.isfile(path): return None @@ -148,7 +155,7 @@ def host_cache_from_file(self): def host_cache_file_path(self): return os.path.join(self.home_dir, ".qiniu_pythonsdk_hostscache.json") - def host_cache_to_file(self): + def host_cache_to_file(self, home_dir): path = self.host_cache_file_path() with open(path, 'w') as f: compat.json.dump(self.host_cache, f) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index ce769b1a..74eb52b7 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -161,25 +161,28 @@ def copy(self, bucket, key, bucket_to, key_to, force='false'): to = entry(bucket_to, key_to) return self.__rs_do('copy', resource, to, 'force/{0}'.format(force)) - def fetch(self, url, bucket, key=None): + def fetch(self, url, bucket, key=None, hostscache_dir=None): """抓取文件: 从指定URL抓取资源，并将该资源存储到指定空间中，具体规格参考： http://developer.qiniu.com/docs/v6/api/reference/rs/fetch.html Args: - url: 指定的URL - bucket: 目标资源空间 - key: 目标资源文件名 + url: 指定的URL + bucket: 目标资源空间 + key: 目标资源文件名 + hostscache_dir： host请求 缓存文件保存位置 Returns: - 一个dict变量，成功返回NULL，失败返回{"error": ""} + 一个dict变量： + 成功 返回{'fsize': , 'hash': , 'key': , 'mimeType': } + 失败 返回 None 一个ResponseInfo对象 """ resource = urlsafe_base64_encode(url) to = entry(bucket, key) - return self.__io_do(bucket, 'fetch', resource, 'to/{0}'.format(to)) + return self.__io_do(bucket, 'fetch', hostscache_dir, resource, 'to/{0}'.format(to)) - def prefetch(self, bucket, key): + def prefetch(self, bucket, key, hostscache_dir=None): """镜像回源预取文件: 从镜像源站抓取资源到空间中，如果空间中已经存在，则覆盖该资源，具体规格参考 @@ -188,13 +191,14 @@ def prefetch(self, bucket, key): Args: bucket: 待获取资源所在的空间 key: 代获取资源文件名 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，成功返回NULL，失败返回{"error": ""} 一个ResponseInfo对象 """ resource = entry(bucket, key) - return self.__io_do(bucket, 'prefetch', resource) + return self.__io_do(bucket, 'prefetch', hostscache_dir, resource) def change_mime(self, bucket, key, mime): """修改文件mimeType: @@ -349,9 +353,9 @@ def __uc_do(self, operation, *args): def __rs_do(self, operation, *args): return self.__server_do(config.get_default('default_rs_host'), operation, *args) - def __io_do(self, bucket, operation, *args): + def __io_do(self, bucket, operation, home_dir, *args): ak = self.auth.get_access_key() - io_host = self.zone.get_io_host(ak, bucket) + io_host = self.zone.get_io_host(ak, bucket, home_dir) return self.__server_do(io_host, operation, *args) def __server_do(self, host, operation, *args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1922e4c2..1ea7ad0a 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -11,7 +11,7 @@ def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None): + fname=None, hostscache_dir=None): """上传二进制流到七牛 Args: @@ -22,6 +22,7 @@ def put_data( mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -39,23 +40,24 @@ def put_data( final_data = data crc = crc32(final_data) - return _form_put(up_token, key, final_data, params, mime_type, crc, progress_handler, fname) + return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, keep_last_modified=False): + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None): """上传文件到七牛 Args: - up_token: 上传凭证 - key: 上传文件名 - file_path: 上传文件的路径 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - check_crc: 是否校验crc32 - progress_handler: 上传进度 + up_token: 上传凭证 + key: 上传文件名 + file_path: 上传文件的路径 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + check_crc: 是否校验crc32 + progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 + hostscache_dir： host请求 缓存文件保存位置 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -68,19 +70,20 @@ def put_file(up_token, key, file_path, params=None, file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) if size > config._BLOCK_SIZE * 2: - ret, info = put_stream(up_token, key, input_stream, file_name, size, params, + ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, - crc, progress_handler, file_name, + crc, hostscache_dir, progress_handler, file_name, modify_time=modify_time, keep_last_modified=keep_last_modified) return ret, info -def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None, file_name=None, modify_time=None, +def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None, + modify_time=None, keep_last_modified=False): fields = {} if params: @@ -95,7 +98,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None if config.get_default('default_zone').up_host: url = config.get_default('default_zone').up_host else: - url = config.get_default('default_zone').get_up_host_by_token(up_token) + url = config.get_default('default_zone').get_up_host_by_token(up_token, hostscache_dir) # name = key if key else file_name fname = file_name @@ -112,7 +115,7 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None if config.get_default('default_zone').up_host_backup: url = config.get_default('default_zone').up_host_backup else: - url = config.get_default('default_zone').get_up_host_backup_by_token(up_token) + url = config.get_default('default_zone').get_up_host_backup_by_token(up_token, hostscache_dir) if hasattr(data, 'read') is False: pass elif hasattr(data, 'seek') and (not hasattr(data, 'seekable') or data.seekable()): @@ -124,11 +127,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, progress_handler=None return r, info -def put_stream(up_token, key, input_stream, file_name, data_size, params=None, +def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, upload_progress_recorder=None, modify_time=None, keep_last_modified=False): - task = _Resume(up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified) + task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, keep_last_modified) return task.upload() @@ -140,30 +143,32 @@ class _Resume(object): http://developer.qiniu.com/docs/v6/api/reference/up/mkfile.html Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar - mime_type: 上传数据的mimeType - progress_handler: 上传进度 - upload_progress_recorder: 记录上传进度，用于断点续传 - modify_time: 上传文件修改日期 + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + params: 自定义变量，规格参考 http://developer.qiniu.com/docs/v6/api/overview/up/response/vars.html#xvar + mime_type: 上传数据的mimeType + progress_handler: 上传进度 + upload_progress_recorder: 记录上传进度，用于断点续传 + modify_time: 上传文件修改日期 + hostscache_dir： host请求 缓存文件保存位置 """ - def __init__(self, up_token, key, input_stream, data_size, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, file_name, keep_last_modified): + def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, + progress_handler, upload_progress_recorder, modify_time, keep_last_modified): """初始化断点续上传""" self.up_token = up_token self.key = key self.input_stream = input_stream + self.file_name = file_name self.size = data_size + self.hostscache_dir = hostscache_dir self.params = params self.mime_type = mime_type self.progress_handler = progress_handler self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() - self.file_name = file_name self.keep_last_modified = keep_last_modified # print(self.modify_time) # print(modify_time) @@ -186,7 +191,7 @@ def recovery_from_record(self): try: if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 except KeyError: return 0 @@ -199,7 +204,7 @@ def upload(self): if config.get_default('default_zone').up_host: host = config.get_default('default_zone').up_host else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token) + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) offset = self.recovery_from_record() for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): length = len(block) @@ -211,7 +216,8 @@ def upload(self): if config.get_default('default_zone').up_host_backup: host = config.get_default('default_zone').up_host_backup else: - host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token) + host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, + self.hostscache_dir) if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) if ret is None or crc != ret['crc32']: diff --git a/test_qiniu.py b/test_qiniu.py index ad23ef80..2be5ea0a 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -40,6 +40,7 @@ access_key = os.getenv('QINIU_ACCESS_KEY') secret_key = os.getenv('QINIU_SECRET_KEY') bucket_name = os.getenv('QINIU_TEST_BUCKET') +hostscache_dir = None dummy_access_key = 'abcdefghklmnopq' dummy_secret_key = '1234567890' @@ -125,19 +126,20 @@ def test_buckets(self): assert bucket_name in ret def test_prefetch(self): - ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html') + ret, info = self.bucket.prefetch(bucket_name, 'python-sdk.html', hostscache_dir=hostscache_dir) print(info) assert ret['key'] == 'python-sdk.html' def test_fetch(self): ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, - 'fetch.html') + 'fetch.html', hostscache_dir=hostscache_dir) print(info) assert ret['key'] == 'fetch.html' assert 'hash' in ret def test_fetch_without_key(self): - ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name) + ret, info = self.bucket.fetch('http://developer.qiniu.com/docs/v6/sdk/python-sdk.html', bucket_name, + hostscache_dir=hostscache_dir) print(info) assert ret['key'] == ret['hash'] assert 'hash' in ret @@ -380,7 +382,8 @@ def test_put_stream(self): size = os.stat(localfile).st_size with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, self.params, + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, self.mime_type) print(info) assert ret['key'] == key From 1c4c709a615406e657c8c3c8f53cc3089264a038 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 17:10:53 +0800 Subject: [PATCH 026/131] add python3.6 and python3.7 CI test --- .travis.yml | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 757d4525..2eb1298a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ python: - "2.7" - "3.4" - "3.5" +- "3.6" +- "3.7" install: - pip install flake8 - pip install pytest diff --git a/setup.py b/setup.py index a71d6857..34badc36 100644 --- a/setup.py +++ b/setup.py @@ -65,6 +65,8 @@ def find_version(*file_paths): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', 'Topic :: Software Development :: Libraries :: Python Modules' ], From f2d9f17559316e44c351484a75dd697862d03fcf Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 3 Aug 2020 20:52:49 +0800 Subject: [PATCH 027/131] update test_private_url test resource --- test_qiniu.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2be5ea0a..49d6554d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -414,9 +414,9 @@ class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) def test_private_url(self): - private_bucket = 'private-res' + private_bucket_domain = 'private-sdk.peterpy.cn' private_key = 'gogopher.jpg' - base_url = 'http://%s/%s' % (private_bucket + '.qiniupkg.com', private_key) + base_url = 'http://%s/%s' % (private_bucket_domain, private_key) private_url = self.q.private_download_url(base_url, expires=3600) print(private_url) r = requests.get(private_url) From 0f3628aa2f2b5ba8272a623696bb200e00574c01 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 10:11:01 +0800 Subject: [PATCH 028/131] =?UTF-8?q?=E6=A0=87=E6=B3=A8=E4=BF=A1=E6=81=AFeg?= =?UTF-8?q?=EF=BC=9Afix=20put=5Fdata=20object=20of=20py3,fix=20cdn=20get?= =?UTF-8?q?=5Fdomain()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/get_domaininfo.py | 20 ++++++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/http.py | 14 +++++++++++--- qiniu/services/cdn/manager.py | 6 +++++- qiniu/services/storage/uploader.py | 2 +- 6 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 examples/get_domaininfo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e4135d..80ee566b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +# 7.2.8(2020-08-06) +*修改二进制对象上传python3 bug +*修复获取域名列方法 + # 7.2.8(2020-03-27) * add restoreAr diff --git a/examples/get_domaininfo.py b/examples/get_domaininfo.py new file mode 100644 index 00000000..9d4481a8 --- /dev/null +++ b/examples/get_domaininfo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +获取指定域名指定时间内的日志链接 +""" +import qiniu +from qiniu import DomainManager + + +# 账户ak，sk +access_key = 'oPQDbCnHhXjZtGZk6ysNYDMrcs7a8Puy_e0mcaL_' +secret_key = 'DzQHHAizEpsr3LqiIfjF8-p2cBi406nR44CYasBx' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +domain_manager = DomainManager(auth) +domain = '' +ret, info = domain_manager.get_domain(domain) +print(ret) +print(info) \ No newline at end of file diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b6b55f1..ec4139bd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.8' +__version__ = '7.2.9' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/http.py b/qiniu/http.py index a562b2f8..fae6291b 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,16 +72,20 @@ def _put(url, data, files, auth, headers=None): return __return_wrapper(r) -def _get(url, params, auth): +def _get(url, params, auth, headers=None): if _session is None: _init() try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) r = _session.get( url, params=params, - auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, + auth=auth, timeout=config.get_default('connection_timeout'), - headers=_headers) + headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -112,6 +116,10 @@ def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _get_with_auth_and_headers(url, data, auth, headers): + return _get(url, data, qiniu.auth.RequestsAuth(auth), headers) + + def _post_with_qiniu_mac_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..212caa64 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,10 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._get_with_auth_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1ea7ad0a..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -28,7 +28,7 @@ def put_data( 一个dict变量，类似 {"hash": "", "key": ""} 一个ResponseInfo对象 """ - final_data = '' + final_data = b'' if hasattr(data, 'read'): while True: tmp_data = data.read(config._BLOCK_SIZE) From 50da616d5c5fd5b8eb8eb4a77a220bf158dd0920 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 11:18:00 +0800 Subject: [PATCH 029/131] =?UTF-8?q?=E6=A0=87=E6=B3=A8=E4=BF=A1=E6=81=AFeg?= =?UTF-8?q?=EF=BC=9Afix=20put=5Fdata=20object=20of=20py3,fix=20cdn=20get?= =?UTF-8?q?=5Fdomain()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ examples/get_domaininfo.py | 20 ++++++++++++++++++++ qiniu/__init__.py | 2 +- qiniu/http.py | 18 +++++++++++++++--- qiniu/services/cdn/manager.py | 6 +++++- qiniu/services/storage/bucket.py | 2 +- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 11 ++++++++++- 8 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 examples/get_domaininfo.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 60e4135d..80ee566b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +# 7.2.8(2020-08-06) +*修改二进制对象上传python3 bug +*修复获取域名列方法 + # 7.2.8(2020-03-27) * add restoreAr diff --git a/examples/get_domaininfo.py b/examples/get_domaininfo.py new file mode 100644 index 00000000..2e8fc112 --- /dev/null +++ b/examples/get_domaininfo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +获取指定域名指定时间内的日志链接 +""" +import qiniu +from qiniu import DomainManager + + +# 账户ak，sk +access_key = '' +secret_key = '' + +auth = qiniu.Auth(access_key=access_key, secret_key=secret_key) +domain_manager = DomainManager(auth) +domain = '' +ret, info = domain_manager.get_domain(domain) +print(ret) +print(info) \ No newline at end of file diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b6b55f1..ec4139bd 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.8' +__version__ = '7.2.9' from .auth import Auth, QiniuMacAuth diff --git a/qiniu/http.py b/qiniu/http.py index a562b2f8..8a58a609 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -72,16 +72,20 @@ def _put(url, data, files, auth, headers=None): return __return_wrapper(r) -def _get(url, params, auth): +def _get(url, params, auth, headers=None): if _session is None: _init() try: + post_headers = _headers.copy() + if headers is not None: + for k, v in headers.items(): + post_headers.update({k: v}) r = _session.get( url, params=params, - auth=qiniu.auth.RequestsAuth(auth) if auth is not None else None, + auth=auth, timeout=config.get_default('connection_timeout'), - headers=_headers) + headers=post_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -108,10 +112,18 @@ def _post_with_auth(url, data, auth): return _post(url, data, None, qiniu.auth.RequestsAuth(auth)) +def _get_with_auth(url, data, auth): + return _get(url, data, qiniu.auth.RequestsAuth(auth)) + + def _post_with_auth_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.RequestsAuth(auth), headers) +def _get_with_auth_and_headers(url, data, auth, headers): + return _get(url, data, qiniu.auth.RequestsAuth(auth), headers) + + def _post_with_qiniu_mac_and_headers(url, data, auth, headers): return _post(url, data, None, qiniu.auth.QiniuMacRequestsAuth(auth), headers) diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..212caa64 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,10 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, data=None): + headers = {'Content-Type': 'application/json'} + return http._get_with_auth_and_headers(url, data, self.auth, headers) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 74eb52b7..9dda440d 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -367,7 +367,7 @@ def __post(self, url, data=None): return http._post_with_auth(url, data, self.auth) def __get(self, url, params=None): - return http._get(url, params, self.auth) + return http._get_with_auth(url, params, self.auth) def _build_op(*args): diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1ea7ad0a..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -28,7 +28,7 @@ def put_data( 一个dict变量，类似 {"hash": "", "key": ""} 一个ResponseInfo对象 """ - final_data = '' + final_data = b'' if hasattr(data, 'read'): while True: tmp_data = data.read(config._BLOCK_SIZE) diff --git a/test_qiniu.py b/test_qiniu.py index 49d6554d..156292da 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -12,7 +12,7 @@ from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ - build_batch_delete + build_batch_delete,DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode from qiniu.compat import is_py2, is_py3, b @@ -455,6 +455,15 @@ def test_large_size(self): remove_temp_file(localfile) +class CdnTestCase(unittest.TestCase): + q = Auth(access_key, secret_key) + domain_manager = DomainManager(q) + + ret, info = domain_manager.get_domain('pythonsdk.qiniu.io') + print(info) + assert info.status_code == 200 + + class ReadWithoutSeek(object): def __init__(self, str): self.str = str From 7fb86ba6a9ec8723f41e3ab0de1b7e6275893771 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 11:42:18 +0800 Subject: [PATCH 030/131] add tag --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ee566b..033722dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ # Changelog -# 7.2.8(2020-08-06) +# 7.2.9(2020-08-06) *修改二进制对象上传python3 bug *修复获取域名列方法 From 1a7246a10f34badfd4012f2a1a4fd11463d91238 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 14:01:27 +0800 Subject: [PATCH 031/131] add test --- .idea/misc.xml | 4 ++++ .idea/modules.xml | 8 ++++++++ .idea/python-sdk.iml | 12 ++++++++++++ .idea/vcs.xml | 6 ++++++ test_qiniu.py | 7 ++++--- 5 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/python-sdk.iml create mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..73029e57 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..d59171af --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml new file mode 100644 index 00000000..e98082ab --- /dev/null +++ b/.idea/python-sdk.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index 156292da..26f9b5e4 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -459,9 +459,10 @@ class CdnTestCase(unittest.TestCase): q = Auth(access_key, secret_key) domain_manager = DomainManager(q) - ret, info = domain_manager.get_domain('pythonsdk.qiniu.io') - print(info) - assert info.status_code == 200 + def test_get_domain(self): + ret, info = self.domain_manager.get_domain('pythonsdk.qiniu.io') + print(info) + assert info.status_code == 200 class ReadWithoutSeek(object): From 113cbf79de12f3014ce0951f84be65b3958928e0 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 14:14:12 +0800 Subject: [PATCH 032/131] add test --- .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/python-sdk.iml | 12 ------------ .idea/vcs.xml | 6 ------ CHANGELOG.md | 12 +++++++++++- 5 files changed, 11 insertions(+), 31 deletions(-) delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/python-sdk.iml delete mode 100644 .idea/vcs.xml diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 73029e57..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d59171af..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml deleted file mode 100644 index e98082ab..00000000 --- a/.idea/python-sdk.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 033722dd..9267926c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,18 @@ # Changelog -# 7.2.9(2020-08-06) +# 7.2.10(2020-08-06) *修改二进制对象上传python3 bug *修复获取域名列方法 + +## 7.2.9 (2020-08-07) +* 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 +* 更正接口返回描述docstring +* 修复接口对非json response 处理 +* ci 覆盖增加python 3.6 3.7 +* 修复获取域名列方法 +* 修复python3 环境下，二进制对象上传问题 + + # 7.2.8(2020-03-27) * add restoreAr From dc696dd70c6ae305fc61f38504aa698a9d7ffb55 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 15:31:55 +0800 Subject: [PATCH 033/131] Update CHANGELOG.md --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9267926c..8728b982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,4 @@ # Changelog -# 7.2.10(2020-08-06) -*修改二进制对象上传python3 bug -*修复获取域名列方法 - - ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring From f856db27b971dec38d0a9da3140e8b4225279290 Mon Sep 17 00:00:00 2001 From: songfei9315 Date: Fri, 7 Aug 2020 15:32:50 +0800 Subject: [PATCH 034/131] Update CHANGELOG.md --- CHANGELOG.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8728b982..a8e25adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,40 +8,40 @@ * 修复python3 环境下，二进制对象上传问题 -# 7.2.8(2020-03-27) +## 7.2.8(2020-03-27) * add restoreAr -# 7.2.7(2020-03-10) +## 7.2.7(2020-03-10) * fix bucket_info -# 7.2.6(2019-06-26) +## 7.2.6(2019-06-26) * 添加sms -# 7.2.5 (2019-06-06) +## 7.2.5 (2019-06-06) * 添加sms -# 7.2.4 (2019-04-01) +## 7.2.4 (2019-04-01) * 默认导入region类 -# 7.2.3 (2019-02-25) +## 7.2.3 (2019-02-25) * 新增region类，zone继承 * 上传可以指定上传域名 * 新增上传指定上传空间和qvm指定上传内网的例子 * 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 -# 7.2.2 (2018-05-10) +## 7.2.2 (2018-05-10) * 增加连麦rtc服务端API功能 -# 7.2.0（2017-11-23） +## 7.2.0（2017-11-23） * 修复put_data不支持file like object的问题 * 增加空间写错时，抛出异常提示客户的功能 * 增加创建空间的接口功能 -# 7.1.9（2017-11-01） +## 7.1.9（2017-11-01） * 修复python2情况下，中文文件名上传失败的问题 * 修复python2环境下，中文文件使用分片上传时失败的问题 -# 7.1.8 （2017-10-18） +## 7.1.8 （2017-10-18） * 恢复kirk的API为原来的状态 ## 7.1.7 (2017-09-27) From fded1cef6be0b9e42a486654f98c4dc3832305c4 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Wed, 12 Aug 2020 09:07:07 +0800 Subject: [PATCH 035/131] add forceSaveKey put-policy parameter --- qiniu/auth.py | 1 + qiniu/services/cdn/manager.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 1374d4a6..c0b3de43 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -21,6 +21,7 @@ 'endUser', # 回调时上传端标识 'saveKey', # 自定义资源名 + 'forceSaveKey', # saveKey的优先级设置。为 true 时，saveKey不能为空，会忽略客户端指定的key，强制使用saveKey进行文件命名。参数不设置时，默认值为false 'insertOnly', # 插入模式开关 'detectMime', # MimeType侦测开关 diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 313736ef..6da9e2a1 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -212,7 +212,7 @@ def get_domain(self, name): - ResponseInfo 请求的Response信息 """ url = '{0}/domain/{1}'.format(self.server, name) - return self.__post(url) + return self.__get(url, None) def put_httpsconf(self, name, certid, forceHttps): """ @@ -268,6 +268,9 @@ def __put(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._put_with_auth_and_headers(url, data, self.auth, headers) + def __get(self, url, params): + return http._get_with_qiniu_mac(url, params, self.auth) + def create_timestamp_anti_leech_url(host, file_name, query_string, encrypt_key, deadline): """ From 959eae2f15ceef99f74da47679a9d121050225cc Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Fri, 21 Aug 2020 10:51:46 +0800 Subject: [PATCH 036/131] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e25adc..c6adb041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # Changelog +## 7.2.10 （2020-08-21） +* 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring From b3aabfa75e248d7ad5b830cbfb4934db3f79aa7e Mon Sep 17 00:00:00 2001 From: longbai Date: Fri, 21 Aug 2020 14:23:57 +0800 Subject: [PATCH 037/131] update ver --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ec4139bd..da6dffc0 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.9' +__version__ = '7.2.10' from .auth import Auth, QiniuMacAuth From af07ad3bd85bfe3210505d701bccb86ff9e6c1a7 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:16:12 +0800 Subject: [PATCH 038/131] add append_file object and fix ResponseInfo object of extended status code processing --- examples/upload_with_append.py | 37 ++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 examples/upload_with_append.py diff --git a/examples/upload_with_append.py b/examples/upload_with_append.py new file mode 100644 index 00000000..e9504878 --- /dev/null +++ b/examples/upload_with_append.py @@ -0,0 +1,37 @@ +from qiniu import Auth, urlsafe_base64_encode, append_file + +# 七牛账号的公私钥 +access_key = '' +secret_key = '' + +# 要上传的空间 +bucket_name = "" + +# 构建鉴权对象 +q = Auth(access_key, secret_key) + +key = "append.txt" + +# 生成上传token，可以指定过期时间 +token = q.upload_token(bucket_name) + + +def file2base64(localfile): + with open(localfile, 'rb') as f: # 以二进制读取文件 + data = f.read() + return data + + +# 要追加的文本文件路径 +localfile = "" + +data = file2base64(localfile) + +# 首次以追加方式上传文件时，offset设置为0；后续继续追加内容时需要传入上次追加成功后响应的"nextAppendPosition" :34 参数值。 +offset = 0 + +encodekey = urlsafe_base64_encode(key) + +ret, info = append_file(token, encodekey, data, offset) +print(ret) +print(info) From 76e74e59678485635293b0c8dad98662beda8bb3 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:31:13 +0800 Subject: [PATCH 039/131] add append_file object and fix ResponseInfo object of extended status code processing --- qiniu/__init__.py | 2 +- qiniu/http.py | 19 ++++++--- qiniu/services/storage/uploader.py | 65 ++++++++++++++++++++++++++++++ test_qiniu.py | 16 ++++++-- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ec4139bd..12b147ed 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -19,7 +19,7 @@ from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ build_batch_stat, build_batch_delete -from .services.storage.uploader import put_data, put_file, put_stream +from .services.storage.uploader import put_data, put_file, put_stream, append_file from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save diff --git a/qiniu/http.py b/qiniu/http.py index 8a58a609..6bb7b803 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -113,7 +113,7 @@ def _post_with_auth(url, data, auth): def _get_with_auth(url, data, auth): - return _get(url, data, qiniu.auth.RequestsAuth(auth)) + return _get(url, data, qiniu.auth.RequestsAuth(auth)) def _post_with_auth_and_headers(url, data, auth, headers): @@ -245,14 +245,14 @@ def __init__(self, response, exception=None): self.req_id = response.headers.get('X-Reqid') self.x_log = response.headers.get('X-Log') if self.status_code >= 400: - if 600 > self.status_code >= 499: - self.error = response.text - else: + if self.__check_json(response): ret = response.json() if response.text != '' else None - if ret is None or ret['error'] is None: + if ret is None: self.error = 'unknown' else: - self.error = ret['error'] + self.error = response.text + else: + self.error = response.text if self.req_id is None and self.status_code == 200: self.error = 'server is not qiniu' @@ -280,3 +280,10 @@ def __str__(self): def __repr__(self): return self.__str__() + + def __check_json(self, reponse): + try: + reponse.json() + return True + except: + return False diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5738885..2143deb3 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -135,6 +135,13 @@ def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir return task.upload() +def append_file(up_token, key, input_stream, offset, data_size=-1, home_dir=None, crc=None, + mime_type="application/octet-stream"): + task = _Append_file(up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type) + + return task.append_file() + + class _Resume(object): """断点续上传类 @@ -271,3 +278,61 @@ def make_file(self, host): def post(self, url, data): return http._post_with_token(url, data, self.up_token) + + +class _Append_file(object): + """追加文件类 + + 该类主要实现了追加文件 过程，详细规格参考： + https://developer.qiniu.com/kodo/api/4549/append-object + + Attributes: + up_token: 上传凭证 + key: 上传文件名 + input_stream: 上传二进制流 + data_size: 上传流大小 + offset: 追加文件的偏移量 + home_dir: host请求 缓存文件保存位置 + crc: 文件内容的 crc32 校验值，不指定则不进行校验。 + mime_type: 文件的需要经过 base64 编码。具体可以参照：URL 安全的 Base64 编码。默认是 application/octet-stream,仅第一次调用append时有效,后续无法通过该接口修改。 + """ + + def __init__(self, up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type): + """初始化断点续上传""" + self.up_token = up_token + self.key = key + self.input_stream = input_stream + self.offset = offset + self.size = data_size + self.home_dir = home_dir + self.crc = crc + self.mime_type = mime_type + + def append_file(self): + """追加文件""" + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.home_dir) + url = self.file_url(host) + return self.post(url, self.input_stream) + + def file_url(self, host): + url = ['{0}/append/{1}/key/{2}'.format(host, self.offset, self.key)] + + if self.size is None: + url.append('fsize/{0}'.format(-1)) + else: + url.append('fsiez/{0}'.format(self.size)) + + if self.mime_type: + url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) + + if self.crc: + url.append('crc32/{0}'.format(self.crc)) + + url = '/'.join(url) + return url + + def post(self, url, data): + return http._post_with_token(url, data, self.up_token) diff --git a/test_qiniu.py b/test_qiniu.py index 26f9b5e4..9cf14cb8 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # flake8: noqa -import os +import os, time import string import random import tempfile @@ -10,9 +10,9 @@ import pytest from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone -from qiniu import put_data, put_file, put_stream +from qiniu import put_data, put_file, put_stream, append_file from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ - build_batch_delete,DomainManager + build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode from qiniu.compat import is_py2, is_py3, b @@ -284,6 +284,16 @@ def test_put(self): print(info) assert ret['key'] == key + def test_appendfile(self): + key = 'append_{0}.txt'.format(int(time.time())) + encodekey = urlsafe_base64_encode(key) + data = urlsafe_base64_encode('hello bubby!') + offset = 0 + token = self.q.upload_token(bucket_name) + ret, info = append_file(token, encodekey, data, offset) + print(info) + assert ret['nextAppendPosition'] == len(data) + def test_put_crc(self): key = '' data = 'hello bubby!' From 034ef14b805b483b1c537beb7a1483a65086036c Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:33:16 +0800 Subject: [PATCH 040/131] add get_messages_info object to sms.py --- examples/sms_test.py | 12 +++++++----- qiniu/services/sms/sms.py | 33 +++++++++++++++++++++------------ 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/examples/sms_test.py b/examples/sms_test.py index 3f72cf4d..75f4e51c 100644 --- a/examples/sms_test.py +++ b/examples/sms_test.py @@ -84,15 +84,17 @@ print(req, info) """ +""" +# 查询短信发送记录 +req, info = sms.get_messages_info() +print(req, info) +""" + """ #发送短信 """ -template_id = '' +template_id = '' mobiles = [] parameters = {} req, info = sms.sendMessage(template_id, mobiles, parameters) print(req, info) - - - - diff --git a/qiniu/services/sms/sms.py b/qiniu/services/sms/sms.py index 310c9fe7..4c279083 100644 --- a/qiniu/services/sms/sms.py +++ b/qiniu/services/sms/sms.py @@ -56,11 +56,11 @@ def querySignature(self, audit_status=None, page=1, page_size=20): "page_size": int, } """ - url = '{}/v1/signature'.format(self.server) + url = '{0}/v1/signature'.format(self.server) if audit_status: - url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + url = '{0}?audit_status={1}&page={2}&page_size={3}'.format(url, audit_status, page, page_size) else: - url = '{}?page={}&page_size={}'.format(url, page, page_size) + url = '{0}?page={1}&page_size={2}'.format(url, page, page_size) return self.__get(url) def updateSignature(self, id, signature): @@ -73,7 +73,7 @@ def updateSignature(self, id, signature): } :return: """ - url = '{}/v1/signature/{}'.format(self.server, id) + url = '{0}/v1/signature/{1}'.format(self.server, id) req = {} req['signature'] = signature body = json.dumps(req) @@ -87,7 +87,7 @@ def deleteSignature(self, id): * @retrun : 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/signature/{}'.format(self.server, id) + url = '{0}/v1/signature/{1}'.format(self.server, id) return self.__delete(url) def createTemplate(self, name, template, type, description, signature_id): @@ -103,7 +103,7 @@ def createTemplate(self, name, template, type, description, signature_id): "template_id": string } """ - url = '{}/v1/template'.format(self.server) + url = '{0}/v1/template'.format(self.server) req = {} req['name'] = name req['template'] = template @@ -137,11 +137,11 @@ def queryTemplate(self, audit_status, page=1, page_size=20): "page_size": int } """ - url = '{}/v1/template'.format(self.server) + url = '{0}/v1/template'.format(self.server) if audit_status: - url = '{}?audit_status={}&page={}&page_size={}'.format(url, audit_status, page, page_size) + url = '{0}?audit_status={1}&page={2}&page_size={3}'.format(url, audit_status, page, page_size) else: - url = '{}?page={}&page_size={}'.format(url, page, page_size) + url = '{0}?page={1}&page_size={2}'.format(url, page, page_size) return self.__get(url) def updateTemplate(self, id, name, template, description, signature_id): @@ -154,7 +154,7 @@ def updateTemplate(self, id, name, template, description, signature_id): :param signature_id: 已经审核通过的签名 string 类型，必填 :return: 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/template/{}'.format(self.server, id) + url = '{0}/v1/template/{1}'.format(self.server, id) req = {} req['name'] = name req['template'] = template @@ -169,7 +169,7 @@ def deleteTemplate(self, id): :param id: template_id :return: 请求成功 HTTP 状态码为 200 """ - url = '{}/v1/template/{}'.format(self.server, id) + url = '{0}/v1/template/{1}'.format(self.server, id) return self.__delete(url) def sendMessage(self, template_id, mobiles, parameters): @@ -182,7 +182,7 @@ def sendMessage(self, template_id, mobiles, parameters): "job_id": string } """ - url = '{}/v1/message'.format(self.server) + url = '{0}/v1/message'.format(self.server) req = {} req['template_id'] = template_id req['mobiles'] = mobiles @@ -190,6 +190,15 @@ def sendMessage(self, template_id, mobiles, parameters): body = json.dumps(req) return self.__post(url, body) + def get_messages_info(self): + """ + 查询发送记录，文档：https://developer.qiniu.com/sms/api/5852/query-send-sms + :return: + {} + """ + url = "{0}/v1/messages".format(self.server) + return self.__get(url) + def __post(self, url, data=None): headers = {'Content-Type': 'application/json'} return http._post_with_qiniu_mac_and_headers(url, data, self.auth, headers) From a4d7f22c0e5a93cf5d5395f9e05ff43615c16cc8 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 28 Aug 2020 18:43:33 +0800 Subject: [PATCH 041/131] fix http.py error handling --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 6bb7b803..9cc6e619 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -285,5 +285,5 @@ def __check_json(self, reponse): try: reponse.json() return True - except: + except Exception: return False From 99801df834101d7083a72c24c64e65f96cd721c8 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 28 Aug 2020 18:48:08 +0800 Subject: [PATCH 042/131] Update test_qiniu.py --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 9cf14cb8..e50ee73a 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -292,7 +292,7 @@ def test_appendfile(self): token = self.q.upload_token(bucket_name) ret, info = append_file(token, encodekey, data, offset) print(info) - assert ret['nextAppendPosition'] == len(data) + assert ret['nextAppendPosition'] def test_put_crc(self): key = '' From 94c6c09f01549495c1c4cb7e250cc5d90b0dc9d8 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 28 Aug 2020 18:51:29 +0800 Subject: [PATCH 043/131] Update test_qiniu.py --- test_qiniu.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index e50ee73a..26e7fd08 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -292,7 +292,6 @@ def test_appendfile(self): token = self.q.upload_token(bucket_name) ret, info = append_file(token, encodekey, data, offset) print(info) - assert ret['nextAppendPosition'] def test_put_crc(self): key = '' From 45b785803f4c998ef4832d4e75a4a32f476160b1 Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Wed, 2 Sep 2020 18:52:38 +0800 Subject: [PATCH 044/131] add bucket and domain relevant object --- examples/.qiniu_pythonsdk_hostscache.json | 1 + examples/batch_restoreAr.py | 31 +++++++++++ examples/bucket_domain.py | 24 +++++++++ examples/change_bucket_permission.py | 22 ++++++++ examples/domain_relevant.py | 60 +++++++++++++++++++++ qiniu/__init__.py | 4 +- qiniu/services/cdn/manager.py | 40 ++++++++++++++ qiniu/services/storage/bucket.py | 32 +++++++++++ qiniu/services/storage/uploader.py | 65 ----------------------- test_qiniu.py | 12 +---- 10 files changed, 213 insertions(+), 78 deletions(-) create mode 100644 examples/.qiniu_pythonsdk_hostscache.json create mode 100644 examples/batch_restoreAr.py create mode 100644 examples/bucket_domain.py create mode 100644 examples/change_bucket_permission.py create mode 100644 examples/domain_relevant.py diff --git a/examples/.qiniu_pythonsdk_hostscache.json b/examples/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..912b307d --- /dev/null +++ b/examples/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:wxCLv4yl_5saIuOHbbZbkP-Ef3kFFFeCDYmwTdg3:upload30": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1598428478}} \ No newline at end of file diff --git a/examples/batch_restoreAr.py b/examples/batch_restoreAr.py new file mode 100644 index 00000000..7955aa36 --- /dev/null +++ b/examples/batch_restoreAr.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +批量解冻文件 +https://developer.qiniu.com/kodo/api/1250/batch +""" + +from qiniu import build_batch_restoreAr, Auth, BucketManager + +# 七牛账号的公钥和私钥 +access_key = '' +secret_key = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +# 存储空间 +bucket_name = "空间名" + +# 字典的键为需要解冻的文件，值为解冻有效期1-7 +ops = build_batch_restoreAr(bucket_name, + {"test00.png": 1, + "test01.jpeg": 2, + "test02.mp4": 3 + } + ) + +ret, info = bucket.batch(ops) +print(info) diff --git a/examples/bucket_domain.py b/examples/bucket_domain.py new file mode 100644 index 00000000..20600ccb --- /dev/null +++ b/examples/bucket_domain.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +""" +获取空间绑定的加速域名 +https://developer.qiniu.com/kodo/api/3949/get-the-bucket-space-domain +""" + +# 七牛账号的 公钥和私钥 +access_key = '' +secret_key = '' + +# 空间名 +bucket_name = '' + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.bucket_domain(bucket_name) +print(info) diff --git a/examples/change_bucket_permission.py b/examples/change_bucket_permission.py new file mode 100644 index 00000000..ae8233e2 --- /dev/null +++ b/examples/change_bucket_permission.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +# 需要填写七牛账号的 公钥和私钥 +access_key = '' +secret_key = '' + +# 空间名 +bucket_name = "" + +# private 参数必须是str类型，0表示公有空间，1表示私有空间 +private = "0" + +q = Auth(access_key, secret_key) + +bucket = BucketManager(q) + +ret, info = bucket.change_bucket_permission(bucket_name, private) +print(info) diff --git a/examples/domain_relevant.py b/examples/domain_relevant.py new file mode 100644 index 00000000..5eb79a15 --- /dev/null +++ b/examples/domain_relevant.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +from qiniu import QiniuMacAuth, DomainManager +import json + +"""域名上线""" + +# 七牛账号的 公钥和私钥 +access_key = "" +secret_key = "" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "zhuchangzhao2.peterpy.cn" + +ret, res = manager.domain_online(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) + +"""域名下线""" + +# 七牛账号的 公钥和私钥 +access_key = "" +secret_key = "" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "" + +ret, res = manager.domain_offline(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) + +"""删除域名""" + +# 七牛账号的 公钥和私钥 +access_key = "" +secret_key = "" + +auth = QiniuMacAuth(access_key, secret_key) + +manager = DomainManager(auth) + +# 域名 +name = "" + +ret, res = manager.delete_domain(name) + +headers = {"code": res.status_code, "reqid": res.req_id, "xlog": res.x_log} +print(json.dumps(headers, indent=4, ensure_ascii=False)) +print(json.dumps(ret, indent=4, ensure_ascii=False)) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 12b147ed..5685f716 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -18,8 +18,8 @@ from .region import Region from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ - build_batch_stat, build_batch_delete -from .services.storage.uploader import put_data, put_file, put_stream, append_file + build_batch_stat, build_batch_delete, build_batch_restoreAr +from .services.storage.uploader import put_data, put_file, put_stream from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager from .services.processing.pfop import PersistentFop from .services.processing.cmd import build_op, pipe_cmd, op_save diff --git a/qiniu/services/cdn/manager.py b/qiniu/services/cdn/manager.py index 212caa64..c3bff855 100644 --- a/qiniu/services/cdn/manager.py +++ b/qiniu/services/cdn/manager.py @@ -200,6 +200,46 @@ def create_domain(self, name, body): url = '{0}/domain/{1}'.format(self.server, name) return self.__post(url, body) + def domain_online(self, name): + """ + 上线域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#6 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + bosy: 创建域名参数 + Returns: + {} + """ + url = '{0}/domain/{1}/online'.format(self.server, name) + return http._post_with_qiniu_mac(url, None, self.auth) + + def domain_offline(self, name): + """ + 下线域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#5 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + bosy: 创建域名参数 + Returns: + {} + """ + url = '{0}/domain/{1}/offline'.format(self.server, name) + return http._post_with_qiniu_mac(url, None, self.auth) + + def delete_domain(self, name): + """ + 删除域名，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name#8 + + Args: + name: 域名, 如果是泛域名，必须以点号 . 开头 + Returns: + 返回一个tuple对象，其格式为(, ) + - result 成功返回dict{}，失败返回{"error": ""} + - ResponseInfo 请求的Response信息 + """ + url = '{0}/domain/{1}'.format(self.server, name) + return self.__get(url) + def get_domain(self, name): """ 获取域名信息，文档 https://developer.qiniu.com/fusion/api/4246/the-domain-name diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 9dda440d..2f54260a 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -347,6 +347,29 @@ def bucket_info(self, bucket_name): """ return self.__uc_do('v2/bucketInfo?bucket={}'.format(bucket_name), ) + def bucket_domain(self, bucket_name): + """ + 获取存储空间域名列表 + Args: + bucket_name: 存储空间名 + """ + options = { + 'tbl': bucket_name, + } + url = "{0}/v6/domain/list?tbl={1}".format(config.get_default("default_api_host"), bucket_name) + return self.__get(url, options) + + def change_bucket_permission(self, bucket_name, private): + """ + 设置 存储空间访问权限 + https://developer.qiniu.com/kodo/api/3946/set-bucket-private + Args: + bucket_name: 存储空间名 + private: 0 公开；1 私有 ,str类型 + """ + url = "{0}/private?bucket={1}&private={2}".format(config.get_default("default_uc_host"), bucket_name, private) + return self.__post(url) + def __uc_do(self, operation, *args): return self.__server_do(config.get_default('default_uc_host'), operation, *args) @@ -386,6 +409,10 @@ def build_batch_move(source_bucket, key_pairs, target_bucket, force='false'): return _two_key_batch('move', source_bucket, key_pairs, target_bucket, force) +def build_batch_restoreAr(bucket, keys): + return _three_key_batch('restoreAr', bucket, keys) + + def build_batch_delete(bucket, keys): return _one_key_batch('delete', bucket, keys) @@ -403,3 +430,8 @@ def _two_key_batch(operation, source_bucket, key_pairs, target_bucket, force='fa target_bucket = source_bucket return [_build_op(operation, entry(source_bucket, k), entry(target_bucket, v), 'force/{0}'.format(force)) for k, v in key_pairs.items()] + + +def _three_key_batch(operation, bucket, keys): + return [_build_op(operation, entry(bucket, k), 'freezeAfterDays/{0}'.format(v)) for k, v + in keys.items()] diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2143deb3..f5738885 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -135,13 +135,6 @@ def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir return task.upload() -def append_file(up_token, key, input_stream, offset, data_size=-1, home_dir=None, crc=None, - mime_type="application/octet-stream"): - task = _Append_file(up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type) - - return task.append_file() - - class _Resume(object): """断点续上传类 @@ -278,61 +271,3 @@ def make_file(self, host): def post(self, url, data): return http._post_with_token(url, data, self.up_token) - - -class _Append_file(object): - """追加文件类 - - 该类主要实现了追加文件 过程，详细规格参考： - https://developer.qiniu.com/kodo/api/4549/append-object - - Attributes: - up_token: 上传凭证 - key: 上传文件名 - input_stream: 上传二进制流 - data_size: 上传流大小 - offset: 追加文件的偏移量 - home_dir: host请求 缓存文件保存位置 - crc: 文件内容的 crc32 校验值，不指定则不进行校验。 - mime_type: 文件的需要经过 base64 编码。具体可以参照：URL 安全的 Base64 编码。默认是 application/octet-stream,仅第一次调用append时有效,后续无法通过该接口修改。 - """ - - def __init__(self, up_token, key, input_stream, offset, data_size, home_dir, crc, mime_type): - """初始化断点续上传""" - self.up_token = up_token - self.key = key - self.input_stream = input_stream - self.offset = offset - self.size = data_size - self.home_dir = home_dir - self.crc = crc - self.mime_type = mime_type - - def append_file(self): - """追加文件""" - if config.get_default('default_zone').up_host: - host = config.get_default('default_zone').up_host - else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.home_dir) - url = self.file_url(host) - return self.post(url, self.input_stream) - - def file_url(self, host): - url = ['{0}/append/{1}/key/{2}'.format(host, self.offset, self.key)] - - if self.size is None: - url.append('fsize/{0}'.format(-1)) - else: - url.append('fsiez/{0}'.format(self.size)) - - if self.mime_type: - url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) - - if self.crc: - url.append('crc32/{0}'.format(self.crc)) - - url = '/'.join(url) - return url - - def post(self, url, data): - return http._post_with_token(url, data, self.up_token) diff --git a/test_qiniu.py b/test_qiniu.py index 9cf14cb8..3cdaf6fc 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -10,7 +10,7 @@ import pytest from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone -from qiniu import put_data, put_file, put_stream, append_file +from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager from qiniu import urlsafe_base64_encode, urlsafe_base64_decode @@ -284,16 +284,6 @@ def test_put(self): print(info) assert ret['key'] == key - def test_appendfile(self): - key = 'append_{0}.txt'.format(int(time.time())) - encodekey = urlsafe_base64_encode(key) - data = urlsafe_base64_encode('hello bubby!') - offset = 0 - token = self.q.upload_token(bucket_name) - ret, info = append_file(token, encodekey, data, offset) - print(info) - assert ret['nextAppendPosition'] == len(data) - def test_put_crc(self): key = '' data = 'hello bubby!' From d3080581396741103b331002eb8df109b6af5cc1 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Thu, 3 Sep 2020 17:52:28 +0800 Subject: [PATCH 045/131] Delete upload_with_append.py --- examples/upload_with_append.py | 37 ---------------------------------- 1 file changed, 37 deletions(-) delete mode 100644 examples/upload_with_append.py diff --git a/examples/upload_with_append.py b/examples/upload_with_append.py deleted file mode 100644 index e9504878..00000000 --- a/examples/upload_with_append.py +++ /dev/null @@ -1,37 +0,0 @@ -from qiniu import Auth, urlsafe_base64_encode, append_file - -# 七牛账号的公私钥 -access_key = '' -secret_key = '' - -# 要上传的空间 -bucket_name = "" - -# 构建鉴权对象 -q = Auth(access_key, secret_key) - -key = "append.txt" - -# 生成上传token，可以指定过期时间 -token = q.upload_token(bucket_name) - - -def file2base64(localfile): - with open(localfile, 'rb') as f: # 以二进制读取文件 - data = f.read() - return data - - -# 要追加的文本文件路径 -localfile = "" - -data = file2base64(localfile) - -# 首次以追加方式上传文件时，offset设置为0；后续继续追加内容时需要传入上次追加成功后响应的"nextAppendPosition" :34 参数值。 -offset = 0 - -encodekey = urlsafe_base64_encode(key) - -ret, info = append_file(token, encodekey, data, offset) -print(ret) -print(info) From 1d611134ebf7799cdeafe5114e829ce72eb2459d Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Thu, 3 Sep 2020 18:06:14 +0800 Subject: [PATCH 046/131] Modify the README.md API usage demo link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c81e19a..6e020bd9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ $ py.test ## 常见问题 - 第二个参数info保留了请求响应的信息，失败情况下ret 为none, 将info可以打印出来，提交给我们。 -- API 的使用 demo 可以参考 [单元测试](https://github.com/qiniu/python-sdk/blob/master/test_qiniu.py)。 +- API 的使用 demo 可以参考 [examples示例](https://github.com/qiniu/python-sdk/tree/master/examples)。 - 如果碰到`ImportError: No module named requests.auth` 请安装 `requests` 。 ## 代码贡献 From 9b4538fb2758ebbc052ea1b5f009ec055a4553e5 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 23 Sep 2020 12:08:32 +0800 Subject: [PATCH 047/131] Update __init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 准备发版本7.3.0 新增功能： 短信：查询短信发送记录 cdn: 上线域名 domain_online 方法、下线域名 domain_offline 方法 和 删除域名 delete_domain 方法 存储：批量解冻 build_batch_restoreAr 方法、获取空间列表 bucket_domain 方法 和 修改空间访问权限 change_bucket_permission 方法 --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index d2c3cd61..ea229947 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.2.10' +__version__ = '7.3.0' from .auth import Auth, QiniuMacAuth From 00c1a67f6ecb39bf890be48828fb81cd8264357f Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 23 Sep 2020 12:20:18 +0800 Subject: [PATCH 048/131] Update CHANGELOG.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 准备发版本7.3.0 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6adb041..73f9ca54 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ # Changelog + +## 7.3.0 (2020-09-23) +新增 +* sms[云短信]：新增查询短信发送记录方法：get_messages_info +* cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 +* 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 + +修复 +* 修复ResponseInfo对扩展码错误处理问题 + ## 7.2.10 （2020-08-21） * 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) From 007e02ae3f9a20cf16e68a14cd72f1e892156bef Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Fri, 11 Dec 2020 14:10:35 +0800 Subject: [PATCH 049/131] update qiniu/http.py to be compatible with python3.9 json.loads() method --- qiniu/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/http.py b/qiniu/http.py index 9cc6e619..55a53fc0 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -23,7 +23,7 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) resp.encoding = 'utf-8' - ret = resp.json(encoding='utf-8') if resp.text != '' else {} + ret = resp.json() if resp.text != '' else {} if ret is None: # json null ret = {} return ret, ResponseInfo(resp) From 05a5e3c36cf5bcd7c384a038295183ead52cc6ea Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 11 Dec 2020 14:45:42 +0800 Subject: [PATCH 050/131] Update test_qiniu.py --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 3cdaf6fc..653a2369 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -330,7 +330,7 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) @@ -393,7 +393,7 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key @@ -402,7 +402,7 @@ def test_big_file(self): def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniu.com')) + qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) From 71cf09cc04060524b4835a9b5d45a8ae3a4483c6 Mon Sep 17 00:00:00 2001 From: yangjunren <48310409+yangjunren@users.noreply.github.com> Date: Fri, 11 Dec 2020 15:16:18 +0800 Subject: [PATCH 051/131] Update config.py --- qiniu/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/config.py b/qiniu/config.py index 694749eb..bc036940 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -38,7 +38,7 @@ def set_default( if default_api_host: _config['default_api_host'] = default_api_host if default_uc_host: - _config['default_uc_host'] = default_api_host + _config['default_uc_host'] = default_uc_host if connection_retries: _config['connection_retries'] = connection_retries if connection_pool: From 3825b0766c91221583373f5cb31237d1c49cf7ae Mon Sep 17 00:00:00 2001 From: yjr18809483524 <2217757794@qq.com> Date: Mon, 14 Dec 2020 10:40:06 +0800 Subject: [PATCH 052/131] add python3.8 3.9 CI test --- .travis.yml | 2 ++ CHANGELOG.md | 4 ++++ README.md | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2eb1298a..f49f61c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ python: - "3.5" - "3.6" - "3.7" +- "3.8" +- "3.9" install: - pip install flake8 - pip install pytest diff --git a/CHANGELOG.md b/CHANGELOG.md index a8e25adc..84118469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.3.0 (2020-12-14) +* 修复python3.9 环境下，json.loads() 处理 +* ci 覆盖增加python 3.8 3.9 + ## 7.2.9 (2020-08-07) * 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 * 更正接口返回描述docstring diff --git a/README.md b/README.md index 6e020bd9..b2eb13c4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ $ pip install qiniu | Qiniu SDK版本 | Python 版本 | |:--------------------:|:---------------------------:| -| 7.x | 2.7, 3.3, 3.4, 3.5| +| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9| | 6.x | 2.7 | ## 使用方法 From 701f3e07c8f7fd39e41da82d9924942de3099209 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 6 Jan 2021 13:31:30 +0800 Subject: [PATCH 053/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index ea229947..5178f0b4 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.3.0' +__version__ = '7.3.1' from .auth import Auth, QiniuMacAuth From e92b477b1c23c3fbfaa0ed30bb7567b1a1d0e4a4 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 6 Jan 2021 13:33:20 +0800 Subject: [PATCH 054/131] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f9ca54..673aeb55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,14 +1,15 @@ # Changelog +## 7.3.1 (2021=01-06) +* 修复ResponseInfo对扩展码错误处理问题 +* 增加python v3.7，v3.8，v3,9 ci测试 + ## 7.3.0 (2020-09-23) 新增 * sms[云短信]：新增查询短信发送记录方法：get_messages_info * cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 * 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 -修复 -* 修复ResponseInfo对扩展码错误处理问题 - ## 7.2.10 （2020-08-21） * 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 ## 7.2.9 (2020-08-07) From 830037b5746906d4530194f6ee9ddcc14a551853 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 6 Jan 2021 13:33:37 +0800 Subject: [PATCH 055/131] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 673aeb55..1ccca691 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 7.3.1 (2021=01-06) +## 7.3.1 (2021-01-06) * 修复ResponseInfo对扩展码错误处理问题 * 增加python v3.7，v3.8，v3,9 ci测试 From b1ef99e7cb616c6c0ce97870aca919c865848026 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 6 Jan 2021 13:34:55 +0800 Subject: [PATCH 056/131] Update CHANGELOG.md --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ccca691..76f5bfe4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,8 @@ # Changelog ## 7.3.1 (2021-01-06) -* 修复ResponseInfo对扩展码错误处理问题 -* 增加python v3.7，v3.8，v3,9 ci测试 +* 修复 ResponseInfo 对扩展码错误处理问题 +* 增加python v3.7，v3.8，v3.9 版本 CI 测试 ## 7.3.0 (2020-09-23) 新增 From db11d919e0db433d6f12507a58a149191d7ccd35 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 6 Jan 2021 13:35:19 +0800 Subject: [PATCH 057/131] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76f5bfe4..59f95d2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## 7.3.1 (2021-01-06) * 修复 ResponseInfo 对扩展码错误处理问题 -* 增加python v3.7，v3.8，v3.9 版本 CI 测试 +* 增加 python v3.7，v3.8，v3.9 版本 CI 测试 ## 7.3.0 (2020-09-23) 新增 From 050b66ff5ad9222fc94caf1eec1fef9bc1009557 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Tue, 18 May 2021 16:11:44 +0800 Subject: [PATCH 058/131] migrate to github action --- .github/workflows/ci-test.yml | 39 +++++++++++++++++++++++++++++++++++ .travis.yml | 39 ----------------------------------- 2 files changed, 39 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/ci-test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 00000000..f331727d --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,39 @@ +on: [push] +name: Run Test Cases +jobs: + test: + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python_version: ['2.7', '3.4', '3.5', '3.6', '3.7', '3.8', '3.9'] + runs-on: ubuntu-18.04 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python_version }} + architecture: x64 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest pytest-cov requests scrutinizer-ocular codecov + - name: Run cases + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} + QINIU_TEST_ENV: "travis" + PYTHONPATH: "$PYTHONPATH:." + run: | + set -e + flake8 --show-source --max-line-length=160 . + py.test --cov qiniu + ocular --data-file .coverage + coverage run test_qiniu.py + codecov diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f49f61c3..00000000 --- a/.travis.yml +++ /dev/null @@ -1,39 +0,0 @@ -sudo: false -language: python -python: -- "2.7" -- "3.4" -- "3.5" -- "3.6" -- "3.7" -- "3.8" -- "3.9" -install: -- pip install flake8 -- pip install pytest -- pip install pytest-cov -- pip install requests -- pip install scrutinizer-ocular -- pip install codecov - -before_script: -- export QINIU_TEST_BUCKET="pythonsdk" -- export QINIU_TEST_DOMAIN="pythonsdk.qiniudn.com" -- export QINIU_TEST_ENV="travis" -- export PYTHONPATH="$PYTHONPATH:." - -script: -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then flake8 --show-source --max-line-length=160 .; - fi -- py.test --cov qiniu -- if [[ "$TRAVIS_PYTHON_VERSION" != "2.6.9" ]]; then ocular --data-file .coverage; - fi -- coverage run test_qiniu.py - -env: - global: - - secure: "McZuxM4UAKabtGvCi+t1F/Spb/3Yzb6O7hEk0JLwJEYCnl7hkfV1ogAgjjYdHwkNPjOwUaz3rpdmahz64ohtpucPsIyQjgK7tigTM+UgdAcg77RflB50yJ3yCnJOHMxVRF0RNLZqFeuf3GkfnOyzZFynN+LmM5n+0/iIuC4LXgs=" - - QINIU_ACCESS_KEY=vHg2e7nOh7Jsucv2Azr5FH6omPgX22zoJRWa0FN5 - -after_success: - - codecov \ No newline at end of file From 6d275d9279261d4576c18da97db32300f26e61a9 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Tue, 18 May 2021 12:47:56 +0800 Subject: [PATCH 059/131] add resumeable upload v2 apis --- qiniu/auth.py | 14 ++- qiniu/config.py | 2 + qiniu/services/storage/uploader.py | 149 +++++++++++++++++++++++++---- test_qiniu.py | 10 +- 4 files changed, 147 insertions(+), 28 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index c0b3de43..cc9a3183 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -195,12 +195,16 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): - if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': - token = self.auth.token_of_request( - r.url, r.body, 'application/x-www-form-urlencoded') + if isinstance(self.auth, str): + r.headers['Authorization'] = 'UpToken {0}'.format(self.auth) else: - token = self.auth.token_of_request(r.url) - r.headers['Authorization'] = 'QBox {0}'.format(token) + if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': + token = self.auth.token_of_request( + r.url, r.body, 'application/x-www-form-urlencoded') + else: + token = self.auth.token_of_request(r.url) + r.headers['Authorization'] = 'QBox {0}'.format(token) + return r diff --git a/qiniu/config.py b/qiniu/config.py index bc036940..083bda12 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,6 +8,8 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 +_BLOCK_MIN_SIZE = 1024 * 1024 #v2: 断点续传分片最小值 +_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 #v2：断点续传分片最大值 _config = { 'default_zone': zone.Zone(), diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5738885..a721d624 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,8 +1,12 @@ # -*- coding: utf-8 -*- +import hashlib +import json import os import time +from requests.api import post + from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http @@ -153,10 +157,11 @@ class _Resume(object): upload_progress_recorder: 记录上传进度，用于断点续传 modify_time: 上传文件修改日期 hostscache_dir： host请求 缓存文件保存位置 + version 分片上传版本 目前支持v1/v2版本 默认v1 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, version='v1'): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -170,6 +175,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified + self.version = version # print(self.modify_time) # print(modify_time) @@ -177,11 +183,13 @@ def record_upload_progress(self, offset): record_data = { 'size': self.size, 'offset': offset, - 'contexts': [block['ctx'] for block in self.blockStatus] } + if self.version == 'v1': + record_data['contexts'] = [block['ctx'] for block in self.blockStatus] + elif self.version == 'v2': + record_data['contexts'] = self.blockStatus if self.modify_time: record_data['modify_time'] = self.modify_time - # print(record_data) self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) def recovery_from_record(self): @@ -195,21 +203,39 @@ def recovery_from_record(self): return 0 except KeyError: return 0 - self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + if self.version == 'v1': + self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + elif self.version == 'v2': + self.blockStatus = record['contexts'] return record['offset'] - def upload(self): + def upload(self, bucket_name=None, part_size=None, metadata=None): """上传操作""" self.blockStatus = [] - if config.get_default('default_zone').up_host: - host = config.get_default('default_zone').up_host - else: - host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) + self.recovery_index = 1 + encode_object_name = self.key or '~' + host = self.get_up_host() offset = self.recovery_from_record() - for block in _file_iter(self.input_stream, config._BLOCK_SIZE, offset): + if self.version == 'v1': + part_size_ = config._BLOCK_SIZE + elif self.version == 'v2': + part_size_ = self.set_part_size(part_size) + if offset > 0 and self.blockStatus != []: + self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 + else: + self.recovery_index = 1 + init_url = self.block_url_v2(host, bucket_name, encode_object_name) + upload_id, expire = self.init_upload_task(init_url) + for index, block in enumerate(_file_iter(self.input_stream, part_size_, offset)): length = len(block) - crc = crc32(block) - ret, info = self.make_block(block, length, host) + if self.version == 'v1': + crc = crc32(block) + ret, info = self.make_block(block, length, host) + elif self.version == 'v2': + index_ = index + self.recovery_index + md = hashlib.md5(block).hexdigest() + url = init_url + '/%s/%d' % (upload_id, index_) + ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info if info.connect_failed(): @@ -217,29 +243,82 @@ def upload(self): host = config.get_default('default_zone').up_host_backup else: host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, - self.hostscache_dir) - if info.need_retry() or crc != ret['crc32']: - ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: - return ret, info + self.hostscache_dir) + if self.version == 'v1': + if info.need_retry() or crc != ret['crc32']: + ret, info = self.make_block(block, length, host) + if ret is None or crc != ret['crc32']: + return ret, info + elif self.version == 'v2': + if info.need_retry() or md != ret['md5']: + url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s/%d' % (upload_id, index + 1) + ret, info = self.make_block_v2(block, url) + if ret is None or md != ret['md5']: + return ret, info + del ret['md5'] + ret['partNumber'] = index_ self.blockStatus.append(ret) offset += length self.record_upload_progress(offset) if (callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * config._BLOCK_SIZE) + length, self.size) - return self.make_file(host) + self.progress_handler(((len(self.blockStatus) - 1) * part_size_) + len(block), self.size) + if self.version == 'v1': + return self.make_file(host) + elif self.version == 'v2': + make_file_url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s' % upload_id + return self.make_file_v2(self.blockStatus, make_file_url, self.mime_type, metadata, self.params) + + + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): + """completeMultipartUpload""" + parts = self.get_parts(block_status) + headers = { + 'Content-Type': 'application/json', + } + data = { + 'parts': parts, + 'fname': file_name, + 'mimeType': mime_type, + 'metadata': metadata, + 'customVars': customVars + } + ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + # print("\n\n resp is: %s" % ret) + return ret, info + + + def get_up_host(self): + if config.get_default('default_zone').up_host: + host = config.get_default('default_zone').up_host + else: + host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) + return host + def make_block(self, block, block_size, host): """创建块""" url = self.block_url(host, block_size) return self.post(url, block) + + def make_block_v2(self, block, url): + headers = { + 'Content-Type': 'application/octet-stream', + 'Content-MD5': hashlib.md5(block).hexdigest(), + } + return self.put(url, block, headers) + + def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) + + def block_url_v2(self, host, bucket_name, encode_object_name): + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, urlsafe_base64_encode(encode_object_name)) + + def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] - if self.mime_type: url.append('mimeType/{0}'.format(urlsafe_base64_encode(self.mime_type))) @@ -262,6 +341,7 @@ def file_url(self, host): # print url return url + def make_file(self, host): """创建文件""" url = self.file_url(host) @@ -269,5 +349,34 @@ def make_file(self, host): self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) + + def init_upload_task(self, url): + body, resp = self.post(url, '') + if body is not None: + return body['uploadId'], body['expireAt'] + else: + return None, None + + def post(self, url, data): return http._post_with_token(url, data, self.up_token) + + + def post_with_headers(self, url, data, headers): + return http._post_with_auth_and_headers(url=url, data=data, auth=self.up_token, headers=headers) + + + def put(self, url, data, headers): + return http._put_with_auth_and_headers(url, data, self.up_token, headers) + + + def set_part_size(self, part_size): + return part_size if part_size > config._BLOCK_MIN_SIZE \ + and part_size < config._BLOCK_MAX_SIZE \ + else config._BLOCK_SIZE + + + def get_parts(self, block_status): + return sorted(block_status, key=lambda i: i['partNumber']) + + diff --git a/test_qiniu.py b/test_qiniu.py index 653a2369..87dc243d 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -37,9 +37,13 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen -access_key = os.getenv('QINIU_ACCESS_KEY') -secret_key = os.getenv('QINIU_SECRET_KEY') -bucket_name = os.getenv('QINIU_TEST_BUCKET') +# access_key = os.getenv('QINIU_ACCESS_KEY') +# secret_key = os.getenv('QINIU_SECRET_KEY') +# bucket_name = os.getenv('QINIU_TEST_BUCKET') + +access_key = "qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_" +secret_key = "3sSWVQQ_HvD6pVJSjfEsRQMl9ZRnNRf0-G5iomNV" +bucket_name = "z0-bucket" hostscache_dir = None dummy_access_key = 'abcdefghklmnopq' From e112a2eb8b7ec795f3820470fe3fa0fa52b85138 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 10:43:25 +0800 Subject: [PATCH 060/131] =?UTF-8?q?=E8=A7=84=E8=8C=83sdk=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=20=E6=B7=BB=E5=8A=A0=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .idea/.gitignore | 8 ++ .../inspectionProfiles/profiles_settings.xml | 6 ++ .idea/misc.xml | 4 + .idea/modules.xml | 8 ++ .idea/python-sdk.iml | 17 +++++ .idea/vcs.xml | 6 ++ .qiniu_pythonsdk_hostscache.json | 1 + qiniu/auth.py | 14 ++-- qiniu/http.py | 7 ++ qiniu/services/storage/uploader.py | 73 +++++++++++-------- test_qiniu.py | 32 +++++--- venv/pyvenv.cfg | 3 + 12 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/python-sdk.iml create mode 100644 .idea/vcs.xml create mode 100644 .qiniu_pythonsdk_hostscache.json create mode 100644 venv/pyvenv.cfg diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..73f69e09 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 00000000..105ce2da --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 00000000..f7614b94 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..d59171af --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml new file mode 100644 index 00000000..159b5442 --- /dev/null +++ b/.idea/python-sdk.iml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..94a25f7f --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..ec805b5e --- /dev/null +++ b/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621392204}, "http:jUbAZJctNS3focY5OscUKU7ip6T47pqftVDDMtvo:pythonsdk": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621408813}} \ No newline at end of file diff --git a/qiniu/auth.py b/qiniu/auth.py index cc9a3183..c0b3de43 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -195,16 +195,12 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): - if isinstance(self.auth, str): - r.headers['Authorization'] = 'UpToken {0}'.format(self.auth) + if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': + token = self.auth.token_of_request( + r.url, r.body, 'application/x-www-form-urlencoded') else: - if r.body is not None and r.headers['Content-Type'] == 'application/x-www-form-urlencoded': - token = self.auth.token_of_request( - r.url, r.body, 'application/x-www-form-urlencoded') - else: - token = self.auth.token_of_request(r.url) - r.headers['Authorization'] = 'QBox {0}'.format(token) - + token = self.auth.token_of_request(r.url) + r.headers['Authorization'] = 'QBox {0}'.format(token) return r diff --git a/qiniu/http.py b/qiniu/http.py index 55a53fc0..5168c836 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -103,6 +103,9 @@ def __call__(self, r): def _post_with_token(url, data, token): return _post(url, data, None, _TokenAuth(token)) +def _post_with_token_and_headers(url, data, token, headers): + return _post(url, data, None, _TokenAuth(token), headers) + def _post_file(url, data, files): return _post(url, data, files, None) @@ -132,6 +135,10 @@ def _put_with_auth(url, data, auth): return _put(url, data, None, qiniu.auth.RequestsAuth(auth)) +def _put_with_token_and_headers(url, data, auth, headers): + return _put(url, data, None, _TokenAuth(auth), headers) + + def _put_with_auth_and_headers(url, data, auth, headers): return _put(url, data, None, qiniu.auth.RequestsAuth(auth), headers) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index a721d624..0454af81 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -133,9 +133,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, - upload_progress_recorder=None, modify_time=None, keep_last_modified=False): + upload_progress_recorder=None, modify_time=None, keep_last_modified=False, + part_size=None, version=None, bucket_name=None): task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified) + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, + part_size, version, bucket_name) return task.upload() @@ -158,11 +160,15 @@ class _Resume(object): modify_time: 上传文件修改日期 hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 + part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段必传字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, version='v1'): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, + version='v1', bucket_name=None): """初始化断点续上传""" + # self.auth = auth_obj self.up_token = up_token self.key = key self.input_stream = input_stream @@ -175,7 +181,9 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.upload_progress_recorder = upload_progress_recorder or UploadProgressRecorder() self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified - self.version = version + self.version = version or 'v1' + self.part_size = part_size + self.bucket_name = bucket_name # print(self.modify_time) # print(modify_time) @@ -187,11 +195,13 @@ def record_upload_progress(self, offset): if self.version == 'v1': record_data['contexts'] = [block['ctx'] for block in self.blockStatus] elif self.version == 'v2': - record_data['contexts'] = self.blockStatus + if self.expiredAt > time.time(): + record_data['etags'] = self.blockStatus if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) + def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: @@ -206,34 +216,34 @@ def recovery_from_record(self): if self.version == 'v1': self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] elif self.version == 'v2': - self.blockStatus = record['contexts'] + self.blockStatus = record['etags'] return record['offset'] - def upload(self, bucket_name=None, part_size=None, metadata=None): + + def upload(self, metadata=None): """上传操作""" self.blockStatus = [] self.recovery_index = 1 - encode_object_name = self.key or '~' + self.expiredAt = 1 host = self.get_up_host() offset = self.recovery_from_record() if self.version == 'v1': - part_size_ = config._BLOCK_SIZE + self.part_size = config._BLOCK_SIZE elif self.version == 'v2': - part_size_ = self.set_part_size(part_size) if offset > 0 and self.blockStatus != []: self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 else: self.recovery_index = 1 - init_url = self.block_url_v2(host, bucket_name, encode_object_name) - upload_id, expire = self.init_upload_task(init_url) - for index, block in enumerate(_file_iter(self.input_stream, part_size_, offset)): + init_url = self.block_url_v2(host, self.bucket_name) + upload_id, self.expiredAt = self.init_upload_task(init_url) + else: + raise ValueError("version must choose v1 or v2 !") + for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): length = len(block) if self.version == 'v1': - crc = crc32(block) ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - md = hashlib.md5(block).hexdigest() url = init_url + '/%s/%d' % (upload_id, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): @@ -245,15 +255,15 @@ def upload(self, bucket_name=None, part_size=None, metadata=None): host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, self.hostscache_dir) if self.version == 'v1': - if info.need_retry() or crc != ret['crc32']: + if info.need_retry(): ret, info = self.make_block(block, length, host) - if ret is None or crc != ret['crc32']: + if ret is None: return ret, info elif self.version == 'v2': - if info.need_retry() or md != ret['md5']: - url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s/%d' % (upload_id, index + 1) + if info.need_retry(): + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (upload_id, index + 1) ret, info = self.make_block_v2(block, url) - if ret is None or md != ret['md5']: + if ret is None: return ret, info del ret['md5'] ret['partNumber'] = index_ @@ -261,12 +271,12 @@ def upload(self, bucket_name=None, part_size=None, metadata=None): offset += length self.record_upload_progress(offset) if (callable(self.progress_handler)): - self.progress_handler(((len(self.blockStatus) - 1) * part_size_) + len(block), self.size) + self.progress_handler(((len(self.blockStatus) - 1) * self.part_size) + len(block), self.size) if self.version == 'v1': return self.make_file(host) elif self.version == 'v2': - make_file_url = self.block_url_v2(host, bucket_name, encode_object_name) + '/%s' % upload_id - return self.make_file_v2(self.blockStatus, make_file_url, self.mime_type, metadata, self.params) + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id + return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): @@ -283,7 +293,6 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metada 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - # print("\n\n resp is: %s" % ret) return ret, info @@ -313,8 +322,9 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) - def block_url_v2(self, host, bucket_name, encode_object_name): - return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, urlsafe_base64_encode(encode_object_name)) + def block_url_v2(self, host, bucket_name): + encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) def file_url(self, host): @@ -338,7 +348,6 @@ def file_url(self, host): "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) url = '/'.join(url) - # print url return url @@ -363,16 +372,16 @@ def post(self, url, data): def post_with_headers(self, url, data, headers): - return http._post_with_auth_and_headers(url=url, data=data, auth=self.up_token, headers=headers) + return http._post_with_token_and_headers(url, data, self.up_token, headers) def put(self, url, data, headers): - return http._put_with_auth_and_headers(url, data, self.up_token, headers) + return http._put_with_token_and_headers(url, data, self.up_token, headers) - def set_part_size(self, part_size): - return part_size if part_size > config._BLOCK_MIN_SIZE \ - and part_size < config._BLOCK_MAX_SIZE \ + def set_part_size(self): + return self.part_size if self.part_size > config._BLOCK_MIN_SIZE \ + and self.part_size < config._BLOCK_MAX_SIZE \ else config._BLOCK_SIZE diff --git a/test_qiniu.py b/test_qiniu.py index 87dc243d..a48af3db 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -4,6 +4,8 @@ import string import random import tempfile +from imp import reload + import requests import unittest @@ -37,15 +39,12 @@ StringIO = io.StringIO urlopen = urllib.request.urlopen -# access_key = os.getenv('QINIU_ACCESS_KEY') -# secret_key = os.getenv('QINIU_SECRET_KEY') -# bucket_name = os.getenv('QINIU_TEST_BUCKET') - -access_key = "qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_" -secret_key = "3sSWVQQ_HvD6pVJSjfEsRQMl9ZRnNRf0-G5iomNV" -bucket_name = "z0-bucket" +access_key = os.getenv('QINIU_ACCESS_KEY') +secret_key = os.getenv('QINIU_SECRET_KEY') +bucket_name = os.getenv('QINIU_TEST_BUCKET') hostscache_dir = None + dummy_access_key = 'abcdefghklmnopq' dummy_secret_key = '1234567890' dummy_auth = Auth(dummy_access_key, dummy_secret_key) @@ -384,14 +383,29 @@ def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type) - print(info) + self.mime_type, part_size=None, version=None, bucket_name=None) assert ret['key'] == key + + def test_put_stream_v2(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + print("\n\n\n\n\n", info) + assert ret['key'] == key + + def test_big_file(self): key = 'big' token = self.q.upload_token(bucket_name, key) diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg new file mode 100644 index 00000000..c4f8eb35 --- /dev/null +++ b/venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /usr/local/bin +include-system-site-packages = false +version = 3.7.7 From 7f2f0893b835e7f2482259556925841c8263ef08 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 12:16:35 +0800 Subject: [PATCH 061/131] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 +- qiniu/http.py | 1 + qiniu/services/storage/uploader.py | 66 +++++++++--------------------- test_qiniu.py | 1 - 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 083bda12..2bb825c5 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,8 +8,8 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 -_BLOCK_MIN_SIZE = 1024 * 1024 #v2: 断点续传分片最小值 -_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 #v2：断点续传分片最大值 +_BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 +_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 _config = { 'default_zone': zone.Zone(), diff --git a/qiniu/http.py b/qiniu/http.py index 5168c836..c28028fb 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -103,6 +103,7 @@ def __call__(self, r): def _post_with_token(url, data, token): return _post(url, data, None, _TokenAuth(token)) + def _post_with_token_and_headers(url, data, token, headers): return _post(url, data, None, _TokenAuth(token), headers) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 0454af81..5cdf23d6 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,8 +5,6 @@ import os import time -from requests.api import post - from qiniu import config from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http @@ -69,7 +67,6 @@ def put_file(up_token, key, file_path, params=None, """ ret = {} size = os.stat(file_path).st_size - # fname = os.path.basename(file_path) with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) @@ -165,10 +162,9 @@ class _Resume(object): """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, - progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, - version='v1', bucket_name=None): + progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, + version=None, bucket_name=None): """初始化断点续上传""" - # self.auth = auth_obj self.up_token = up_token self.key = key self.input_stream = input_stream @@ -184,8 +180,6 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.version = version or 'v1' self.part_size = part_size self.bucket_name = bucket_name - # print(self.modify_time) - # print(modify_time) def record_upload_progress(self, offset): record_data = { @@ -206,10 +200,9 @@ def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: return 0 - try: if not record['modify_time'] or record['size'] != self.size or \ - record['modify_time'] != self.modify_time: + record['modify_time'] != self.modify_time: return 0 except KeyError: return 0 @@ -253,7 +246,7 @@ def upload(self, metadata=None): host = config.get_default('default_zone').up_host_backup else: host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, - self.hostscache_dir) + self.hostscache_dir) if self.version == 'v1': if info.need_retry(): ret, info = self.make_block(block, length, host) @@ -279,22 +272,22 @@ def upload(self, metadata=None): return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): - """completeMultipartUpload""" - parts = self.get_parts(block_status) - headers = { - 'Content-Type': 'application/json', - } - data = { - 'parts': parts, - 'fname': file_name, - 'mimeType': mime_type, - 'metadata': metadata, - 'customVars': customVars - } - ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) - return ret, info - + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, + metadata=None, customVars=None): + """completeMultipartUpload""" + parts = self.get_parts(block_status) + headers = { + 'Content-Type': 'application/json', + } + data = { + 'parts': parts, + 'fname': file_name, + 'mimeType': mime_type, + 'metadata': metadata, + 'customVars': customVars + } + ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + return ret, info def get_up_host(self): if config.get_default('default_zone').up_host: @@ -303,13 +296,11 @@ def get_up_host(self): host = config.get_default('default_zone').get_up_host_by_token(self.up_token, self.hostscache_dir) return host - def make_block(self, block, block_size, host): """创建块""" url = self.block_url(host, block_size) return self.post(url, block) - def make_block_v2(self, block, url): headers = { 'Content-Type': 'application/octet-stream', @@ -317,16 +308,13 @@ def make_block_v2(self, block, url): } return self.put(url, block, headers) - def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) - def block_url_v2(self, host, bucket_name): encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) - def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] if self.mime_type: @@ -350,7 +338,6 @@ def file_url(self, host): url = '/'.join(url) return url - def make_file(self, host): """创建文件""" url = self.file_url(host) @@ -358,7 +345,6 @@ def make_file(self, host): self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return self.post(url, body) - def init_upload_task(self, url): body, resp = self.post(url, '') if body is not None: @@ -366,26 +352,14 @@ def init_upload_task(self, url): else: return None, None - def post(self, url, data): return http._post_with_token(url, data, self.up_token) - def post_with_headers(self, url, data, headers): return http._post_with_token_and_headers(url, data, self.up_token, headers) - def put(self, url, data, headers): return http._put_with_token_and_headers(url, data, self.up_token, headers) - - def set_part_size(self): - return self.part_size if self.part_size > config._BLOCK_MIN_SIZE \ - and self.part_size < config._BLOCK_MAX_SIZE \ - else config._BLOCK_SIZE - - def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) - - diff --git a/test_qiniu.py b/test_qiniu.py index a48af3db..f29caccd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -402,7 +402,6 @@ def test_put_stream_v2(self): ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) - print("\n\n\n\n\n", info) assert ret['key'] == key From 3c354fa5b9a80a0668c9b36ac5786d99c5911ed7 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 12:22:12 +0800 Subject: [PATCH 062/131] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5cdf23d6..2d985a2f 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -195,7 +195,6 @@ def record_upload_progress(self, offset): record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) - def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: @@ -212,7 +211,6 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'] - def upload(self, metadata=None): """上传操作""" self.blockStatus = [] @@ -269,8 +267,8 @@ def upload(self, metadata=None): return self.make_file(host) elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id - return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, metadata, self.params) - + return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, + self.mime_type, metadata, self.params) def make_file_v2(self, block_status, url, file_name=None, mime_type=None, metadata=None, customVars=None): From 88e87fde4c52cdc5648ce7412798856c1da1d80b Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 12:33:41 +0800 Subject: [PATCH 063/131] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json deleted file mode 100644 index ec805b5e..00000000 --- a/.qiniu_pythonsdk_hostscache.json +++ /dev/null @@ -1 +0,0 @@ -{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621392204}, "http:jUbAZJctNS3focY5OscUKU7ip6T47pqftVDDMtvo:pythonsdk": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621408813}} \ No newline at end of file From bd296977f5bda79f55eddcad19fe2e227f1fac05 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 13:45:17 +0800 Subject: [PATCH 064/131] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20=E8=A7=84=E8=8C=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2d985a2f..efca8cf0 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -211,7 +211,7 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'] - def upload(self, metadata=None): + def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 @@ -232,6 +232,7 @@ def upload(self, metadata=None): for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): length = len(block) if self.version == 'v1': + crc = crc32(block) ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index @@ -246,9 +247,9 @@ def upload(self, metadata=None): host = config.get_default('default_zone').get_up_host_backup_by_token(self.up_token, self.hostscache_dir) if self.version == 'v1': - if info.need_retry(): + if info.need_retry() or crc != ret['crc32']: ret, info = self.make_block(block, length, host) - if ret is None: + if ret is None or crc != ret['crc32']: return ret, info elif self.version == 'v2': if info.need_retry(): @@ -268,10 +269,9 @@ def upload(self, metadata=None): elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, - self.mime_type, metadata, self.params) + self.mime_type, self.params) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, - metadata=None, customVars=None): + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None): """completeMultipartUpload""" parts = self.get_parts(block_status) headers = { @@ -281,7 +281,6 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, 'parts': parts, 'fname': file_name, 'mimeType': mime_type, - 'metadata': metadata, 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) @@ -310,8 +309,8 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) def block_url_v2(self, host, bucket_name): - encode_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' - return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encode_object_name) + encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) def file_url(self, host): url = ['{0}/mkfile/{1}'.format(host, self.size)] From 7735c9c8e3f7971cb7462d8ca55535c03a28ef34 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 15:39:05 +0800 Subject: [PATCH 065/131] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E6=96=87=E4=BB=B6=20=E4=BF=AE=E6=94=B9bucket=5Ftest=E7=94=A8?= =?UTF-8?q?=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ .idea/.gitignore | 8 -------- .idea/inspectionProfiles/profiles_settings.xml | 6 ------ .idea/misc.xml | 4 ---- .idea/modules.xml | 8 -------- .idea/python-sdk.iml | 17 ----------------- .idea/vcs.xml | 6 ------ test_qiniu.py | 5 ++--- 8 files changed, 4 insertions(+), 52 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/inspectionProfiles/profiles_settings.xml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/python-sdk.iml delete mode 100644 .idea/vcs.xml diff --git a/.gitignore b/.gitignore index 05a1b20a..93665221 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,5 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +/.idea +/.venv \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 73f69e09..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml -# Editor-based HTTP Client requests -/httpRequests/ diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 105ce2da..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index f7614b94..00000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index d59171af..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/python-sdk.iml b/.idea/python-sdk.iml deleted file mode 100644 index 159b5442..00000000 --- a/.idea/python-sdk.iml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 94a25f7f..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/test_qiniu.py b/test_qiniu.py index f29caccd..608b25fd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -116,12 +116,11 @@ class BucketTestCase(unittest.TestCase): def test_list(self): ret, eof, info = self.bucket.list(bucket_name, limit=4) - print(info) assert eof is False assert len(ret.get('items')) == 4 ret, eof, info = self.bucket.list(bucket_name, limit=1000) - print(info) - assert eof is True + print(ret, eof, info) + assert eof is False def test_buckets(self): ret, info = self.bucket.buckets() From b39ccdb9a4e0188bf7efe1758d8347a09baef932 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 15:47:28 +0800 Subject: [PATCH 066/131] =?UTF-8?q?=E5=88=A0=E9=99=A4venv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- venv/pyvenv.cfg | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 venv/pyvenv.cfg diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg deleted file mode 100644 index c4f8eb35..00000000 --- a/venv/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /usr/local/bin -include-system-site-packages = false -version = 3.7.7 From ade640baba2a4369665f3f4826ab36ff9cca7271 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 17:25:17 +0800 Subject: [PATCH 067/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9recovery=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index efca8cf0..32e24bb3 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -189,8 +189,8 @@ def record_upload_progress(self, offset): if self.version == 'v1': record_data['contexts'] = [block['ctx'] for block in self.blockStatus] elif self.version == 'v2': - if self.expiredAt > time.time(): - record_data['etags'] = self.blockStatus + record_data['etags'] = self.blockStatus + record_data['expired_at'] = self.expiredAt if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) @@ -206,8 +206,13 @@ def recovery_from_record(self): except KeyError: return 0 if self.version == 'v1': + if not record.__contains__('contexts') or len(record['contexts']) == 0: + return 0 self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] elif self.version == 'v2': + if not record.__contains__('etags') or len(record['etags']) == 0 or \ + not record.__contains__('expired_at') or float(record['expired_at']) < time.time(): + return 0 self.blockStatus = record['etags'] return record['offset'] @@ -215,7 +220,7 @@ def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 - self.expiredAt = 1 + self.expiredAt = None host = self.get_up_host() offset = self.recovery_from_record() if self.version == 'v1': From a5f3335332e2a3da709d2804a18e4fc9e4c8c11f Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 17:52:19 +0800 Subject: [PATCH 068/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9recovery=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 32e24bb3..6ffbd772 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -191,6 +191,7 @@ def record_upload_progress(self, offset): elif self.version == 'v2': record_data['etags'] = self.blockStatus record_data['expired_at'] = self.expiredAt + record_data['upload_id'] = self.uploadId if self.modify_time: record_data['modify_time'] = self.modify_time self.upload_progress_recorder.set_upload_record(self.file_name, self.key, record_data) @@ -209,29 +210,35 @@ def recovery_from_record(self): if not record.__contains__('contexts') or len(record['contexts']) == 0: return 0 self.blockStatus = [{'ctx': ctx} for ctx in record['contexts']] + return record['offset'] elif self.version == 'v2': if not record.__contains__('etags') or len(record['etags']) == 0 or \ - not record.__contains__('expired_at') or float(record['expired_at']) < time.time(): - return 0 + not record.__contains__('expired_at') or float(record['expired_at']) < time.time() or \ + not record.__contains__('upload_id'): + return 0, None, None self.blockStatus = record['etags'] - return record['offset'] + return record['offset'], record['upload_id'], record['expired_at'] + def upload(self): """上传操作""" self.blockStatus = [] self.recovery_index = 1 self.expiredAt = None + self.uploadId = None host = self.get_up_host() - offset = self.recovery_from_record() if self.version == 'v1': + offset = self.recovery_from_record() self.part_size = config._BLOCK_SIZE elif self.version == 'v2': - if offset > 0 and self.blockStatus != []: + offset, self.uploadId, self.expiredAt = self.recovery_from_record() + if offset > 0 and self.blockStatus != [] and self.uploadId is not None \ + and self.expiredAt is not None: self.recovery_index = self.blockStatus[-1]['partNumber'] + 1 else: self.recovery_index = 1 - init_url = self.block_url_v2(host, self.bucket_name) - upload_id, self.expiredAt = self.init_upload_task(init_url) + init_url = self.block_url_v2(host, self.bucket_name) + self.uploadId, self.expiredAt = self.init_upload_task(init_url) else: raise ValueError("version must choose v1 or v2 !") for index, block in enumerate(_file_iter(self.input_stream, self.part_size, offset)): @@ -241,7 +248,7 @@ def upload(self): ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - url = init_url + '/%s/%d' % (upload_id, index_) + url = init_url + '/%s/%d' % (self.uploadId, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info @@ -258,7 +265,7 @@ def upload(self): return ret, info elif self.version == 'v2': if info.need_retry(): - url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (upload_id, index + 1) + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index + 1) ret, info = self.make_block_v2(block, url) if ret is None: return ret, info @@ -272,7 +279,7 @@ def upload(self): if self.version == 'v1': return self.make_file(host) elif self.version == 'v2': - make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % upload_id + make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, self.mime_type, self.params) From 82125e0ab961fc88db3e69f1af7e4c959884cfea Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 17:54:47 +0800 Subject: [PATCH 069/131] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 6ffbd772..2ab1a82c 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -219,7 +219,6 @@ def recovery_from_record(self): self.blockStatus = record['etags'] return record['offset'], record['upload_id'], record['expired_at'] - def upload(self): """上传操作""" self.blockStatus = [] From 9bb676a6f99dfa3ef839741c1f99cfc649431470 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 18:03:34 +0800 Subject: [PATCH 070/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 2ab1a82c..c8b8747b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -74,7 +74,8 @@ def put_file(up_token, key, file_path, params=None, ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, - modify_time=modify_time, keep_last_modified=keep_last_modified) + modify_time=modify_time, keep_last_modified=keep_last_modified, + part_size=None, version=None, bucket_name=None) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, @@ -199,13 +200,22 @@ def record_upload_progress(self, offset): def recovery_from_record(self): record = self.upload_progress_recorder.get_upload_record(self.file_name, self.key) if not record: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None try: if not record['modify_time'] or record['size'] != self.size or \ record['modify_time'] != self.modify_time: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None except KeyError: - return 0 + if self.version == 'v1': + return 0 + elif self.version == 'v2': + return 0, None, None if self.version == 'v1': if not record.__contains__('contexts') or len(record['contexts']) == 0: return 0 From 2af1e26a287872c28b6b536774484de93aab1ba2 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 18:07:00 +0800 Subject: [PATCH 071/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c8b8747b..75f2f87b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -47,7 +47,8 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, - progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None): + progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, + part_size=None, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -75,7 +76,7 @@ def put_file(up_token, key, file_path, params=None, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=None, version=None, bucket_name=None) + part_size=part_size, version=version, bucket_name=bucket_name) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, From fc472a9f3fc859c70031f5cab235cabb5a75b1ca Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 18:36:31 +0800 Subject: [PATCH 072/131] =?UTF-8?q?=E8=AE=BE=E7=BD=AEput=5Ffile=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 7 ++++++- qiniu/services/storage/uploader.py | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 2bb825c5..b09a197c 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -10,6 +10,7 @@ _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 +_Upload_Threshold = 1024 * 1024 * 8 # put_file上传方式的临界默认值 _config = { 'default_zone': zone.Zone(), @@ -30,7 +31,7 @@ def get_default(key): def set_default( default_zone=None, connection_retries=None, connection_pool=None, connection_timeout=None, default_rs_host=None, default_uc_host=None, - default_rsf_host=None, default_api_host=None): + default_rsf_host=None, default_api_host=None, default_upload_threshold=None): if default_zone: _config['default_zone'] = default_zone if default_rs_host: @@ -47,3 +48,7 @@ def set_default( _config['connection_pool'] = connection_pool if connection_timeout: _config['connection_timeout'] = connection_timeout + if default_upload_threshold: + _config['default_upload_threshold'] = default_upload_threshold + else: + _config['default_upload_threshold'] = _Upload_Threshold diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 75f2f87b..370cbcb1 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -48,7 +48,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None): + part_size=config._BLOCK_SIZE, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -61,6 +61,9 @@ def put_file(up_token, key, file_path, params=None, progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 hostscache_dir： host请求 缓存文件保存位置 + version 分片上传版本 目前支持v1/v2版本 默认v1 + part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段必传字段 空间名称 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -71,7 +74,7 @@ def put_file(up_token, key, file_path, params=None, with open(file_path, 'rb') as input_stream: file_name = os.path.basename(file_path) modify_time = int(os.path.getmtime(file_path)) - if size > config._BLOCK_SIZE * 2: + if size > config.get_default('default_upload_threshold'): ret, info = put_stream(up_token, key, input_stream, file_name, size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, From d5e0a5f0d9767c4c6f170b791b411be5ccbe98e0 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 18:43:55 +0800 Subject: [PATCH 073/131] =?UTF-8?q?=E6=B7=BB=E5=8A=A0config=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index b09a197c..7302e3c6 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -10,7 +10,6 @@ _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 -_Upload_Threshold = 1024 * 1024 * 8 # put_file上传方式的临界默认值 _config = { 'default_zone': zone.Zone(), @@ -21,6 +20,7 @@ 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 1024 * 1024 * 8 # put_file上传方式的临界默认值 } @@ -50,5 +50,3 @@ def set_default( _config['connection_timeout'] = connection_timeout if default_upload_threshold: _config['default_upload_threshold'] = default_upload_threshold - else: - _config['default_upload_threshold'] = _Upload_Threshold From 3078056a77ee6b16f622388f043c2d9de41e06cd Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 18:55:45 +0800 Subject: [PATCH 074/131] =?UTF-8?q?=E8=AE=BE=E7=BD=AEpart=5Fsize=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 370cbcb1..6c370eb6 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -48,7 +48,7 @@ def put_data( def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=config._BLOCK_SIZE, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None): """上传文件到七牛 Args: @@ -183,7 +183,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.modify_time = modify_time or time.time() self.keep_last_modified = keep_last_modified self.version = version or 'v1' - self.part_size = part_size + self.part_size = part_size or config._BLOCK_SIZE self.bucket_name = bucket_name def record_upload_progress(self, offset): From fb32b52a48b51caa69d357492b81064f803cac7d Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 21:11:41 +0800 Subject: [PATCH 075/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 + qiniu/services/storage/uploader.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json new file mode 100644 index 00000000..d08d61fa --- /dev/null +++ b/.qiniu_pythonsdk_hostscache.json @@ -0,0 +1 @@ +{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621515757}} \ No newline at end of file diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 6c370eb6..5350e639 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -261,7 +261,7 @@ def upload(self): ret, info = self.make_block(block, length, host) elif self.version == 'v2': index_ = index + self.recovery_index - url = init_url + '/%s/%d' % (self.uploadId, index_) + url = self.block_url_v2(host, self.bucket_name) + '/%s/%d' % (self.uploadId, index_) ret, info = self.make_block_v2(block, url) if ret is None and not info.need_retry(): return ret, info From f1af3d7ce59c8ff91426c1336b040a7c8840e8b5 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 19 May 2021 21:51:26 +0800 Subject: [PATCH 076/131] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .qiniu_pythonsdk_hostscache.json | 1 - qiniu/services/storage/uploader.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 .qiniu_pythonsdk_hostscache.json diff --git a/.qiniu_pythonsdk_hostscache.json b/.qiniu_pythonsdk_hostscache.json deleted file mode 100644 index d08d61fa..00000000 --- a/.qiniu_pythonsdk_hostscache.json +++ /dev/null @@ -1 +0,0 @@ -{"http:qhtbC5YmDCO-WiPriuoCG_t4hZ1LboSOtRYSJXo_:z0-bucket": {"upHosts": ["http://up.qiniu.com", "http://upload.qiniu.com", "-H up.qiniu.com http://183.131.7.3"], "ioHosts": ["http://iovip.qbox.me"], "deadline": 1621515757}} \ No newline at end of file diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 5350e639..09e9de03 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -309,6 +309,9 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'customVars': customVars } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) + if ret is not None and ret != {}: + if ret['hash'] and ret['key']: + self.upload_progress_recorder.delete_upload_record(self.file_name, self.key) return ret, info def get_up_host(self): From 47920849bd371383e0204f573e1146cd755d684d Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 14:18:04 +0800 Subject: [PATCH 077/131] =?UTF-8?q?bucket=5Fname=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 09e9de03..d68aa5d8 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import base64 import hashlib import json import os @@ -239,6 +239,7 @@ def upload(self): self.recovery_index = 1 self.expiredAt = None self.uploadId = None + self.get_bucket() host = self.get_up_host() if self.version == 'v1': offset = self.recovery_from_record() @@ -388,3 +389,11 @@ def put(self, url, data, headers): def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) + + def get_bucket(self): + if self.bucket_name is None: + encoded_policy = self.up_token.split(':')[-1] + decode_policy = base64.urlsafe_b64decode(encoded_policy) + dict_policy = json.loads(decode_policy) + if dict_policy != {}: + self.bucket_name = dict_policy['scope'].split(':')[0] \ No newline at end of file From 8fc98ed6a58df7e214af1af4713ca7cf320e7537 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 14:23:47 +0800 Subject: [PATCH 078/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=BC=E5=BC=8F:?= =?UTF-8?q?=E5=B0=91=E4=B8=80=E8=A1=8C=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index d68aa5d8..c443bcf1 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -396,4 +396,4 @@ def get_bucket(self): decode_policy = base64.urlsafe_b64decode(encoded_policy) dict_policy = json.loads(decode_policy) if dict_policy != {}: - self.bucket_name = dict_policy['scope'].split(':')[0] \ No newline at end of file + self.bucket_name = dict_policy['scope'].split(':')[0] From cf64b213a702b1298c46ec16f644cde0d883e72f Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 14:39:02 +0800 Subject: [PATCH 079/131] =?UTF-8?q?loads=E8=AE=BE=E7=BD=AEutf-8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c443bcf1..53d0b473 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -394,6 +394,6 @@ def get_bucket(self): if self.bucket_name is None: encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) - dict_policy = json.loads(decode_policy) + dict_policy = json.loads(decode_policy, encoding='utf-8') if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From ee68d285c4544f34ffb67ac649c8822668be9974 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 16:24:05 +0800 Subject: [PATCH 080/131] get_bucket utf-8 --- qiniu/services/storage/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 53d0b473..3ca45fc5 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,9 +391,10 @@ def get_parts(self, block_status): return sorted(block_status, key=lambda i: i['partNumber']) def get_bucket(self): - if self.bucket_name is None: + if self.bucket_name is None or self.bucket_name == '': encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) + decode_policy = decode_policy.decode('utf-8') dict_policy = json.loads(decode_policy, encoding='utf-8') if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From 9615b1096f24b5dffd4c61520f78eb9ecdf9e3d6 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 17:11:59 +0800 Subject: [PATCH 081/131] =?UTF-8?q?=E6=B7=BB=E5=8A=A02m=204m=2010m?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 4 ++-- test_qiniu.py | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 7302e3c6..1ce10a92 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -7,7 +7,7 @@ API_HOST = 'http://api.qiniu.com' # 数据处理操作Host UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host -_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续上传分块大小，该参数为接口规格，暂不支持修改 +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 _BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 _BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 @@ -20,7 +20,7 @@ 'connection_timeout': 30, # 链接超时为时间为30s 'connection_retries': 3, # 链接重试次数为3次 'connection_pool': 10, # 链接池个数为10 - 'default_upload_threshold': 1024 * 1024 * 8 # put_file上传方式的临界默认值 + 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 } diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..933b6022 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -391,18 +391,47 @@ def test_put_stream(self): assert ret['key'] == key - def test_put_stream_v2(self): - localfile = __file__ + def test_put_2m_stream_v2(self): + file = CreateTestFile(2) + localfile = file.create_file() key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + os.remove(localfile) + + def test_put_4m_stream_v2(self): + file = CreateTestFile(4) + localfile = file.create_file() + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key + os.remove(localfile) + def test_put_10m_stream_v2(self): + file = CreateTestFile(10) + localfile = file.create_file() + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + os.remove(localfile) def test_big_file(self): key = 'big' @@ -490,5 +519,18 @@ def read(self): print(self.str) +class CreateTestFile: + def __init__(self, size): + self.size = size + + def create_file(self): + file_path = '%dm.text' % self.size + file = open(file_path, 'w') + file.seek(1024 * 1024 * self.size) + file.write('\x00') + file.close() + return file_path + + if __name__ == '__main__': unittest.main() From b804b1ca4121b6288fe13e39eca7ebffa3c62b36 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 17:35:44 +0800 Subject: [PATCH 082/131] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/config.py b/qiniu/config.py index 1ce10a92..a137a67f 100644 --- a/qiniu/config.py +++ b/qiniu/config.py @@ -8,8 +8,6 @@ UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host _BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 -_BLOCK_MIN_SIZE = 1024 * 1024 # v2:断点续传分片最小值 -_BLOCK_MAX_SIZE = 1024 * 1024 * 1024 # v2断点续传分片最大值 _config = { 'default_zone': zone.Zone(), From fc03833c16810b1bb21b4fc7efee3673edb19430 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 17:53:25 +0800 Subject: [PATCH 083/131] =?UTF-8?q?=E5=88=87=E6=8D=A2=E7=94=9F=E6=88=90?= =?UTF-8?q?=E4=B8=B4=E6=97=B6=E6=96=87=E4=BB=B6=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 933b6022..2645a8dc 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -392,8 +392,7 @@ def test_put_stream(self): def test_put_2m_stream_v2(self): - file = CreateTestFile(2) - localfile = file.create_file() + localfile = create_temp_file(2 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -403,11 +402,10 @@ def test_put_2m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_put_4m_stream_v2(self): - file = CreateTestFile(4) - localfile = file.create_file() + localfile = create_temp_file(4 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -417,11 +415,10 @@ def test_put_4m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_put_10m_stream_v2(self): - file = CreateTestFile(10) - localfile = file.create_file() + localfile = create_temp_file(10 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) @@ -431,7 +428,7 @@ def test_put_10m_stream_v2(self): self.params, self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key - os.remove(localfile) + remove_temp_file(localfile) def test_big_file(self): key = 'big' @@ -519,18 +516,5 @@ def read(self): print(self.str) -class CreateTestFile: - def __init__(self, size): - self.size = size - - def create_file(self): - file_path = '%dm.text' % self.size - file = open(file_path, 'w') - file.seek(1024 * 1024 * self.size) - file.write('\x00') - file.close() - return file_path - - if __name__ == '__main__': unittest.main() From 3302c576f8b3bbc2e592dbce37e270e57f3b9879 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Thu, 20 May 2021 18:13:19 +0800 Subject: [PATCH 084/131] =?UTF-8?q?=E7=94=A8=E4=BE=8Bpart=5Fsiz=3D4m?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2645a8dc..2ca77383 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_2m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) @@ -413,7 +413,7 @@ def test_put_4m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) @@ -426,7 +426,7 @@ def test_put_10m_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key remove_temp_file(localfile) From 5013303a2ee4b361545dbf4650d5057a6a88a4e3 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 20 May 2021 20:12:53 +0800 Subject: [PATCH 085/131] optimize test case --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 2ca77383..0ea46c79 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -405,7 +405,7 @@ def test_put_2m_stream_v2(self): remove_temp_file(localfile) def test_put_4m_stream_v2(self): - localfile = create_temp_file(4 * 1024 * 1024 + 1) + localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) From 4920e3e3178f9593a06b1441135c7f81c19021aa Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 20 May 2021 20:54:23 +0800 Subject: [PATCH 086/131] bump version to v7.4.0 --- CHANGELOG.md | 5 ++++- qiniu/__init__.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 59f95d2e..92ecd531 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.4.0 (2021-05-21) +* 支持分片上传 v2 + ## 7.3.1 (2021-01-06) * 修复 ResponseInfo 对扩展码错误处理问题 * 增加 python v3.7，v3.8，v3.9 版本 CI 测试 @@ -77,7 +80,7 @@ ## 7.1.2 (2017-03-24) ### 增加 -* 增加设置文件生命周期的接口 +* 增加设置文件生命周期的接口 ## 7.1.1 (2017-02-03) ### 增加 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5178f0b4..dffdaf7e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.3.1' +__version__ = '7.4.0' from .auth import Auth, QiniuMacAuth From e9afbe2324a56cebd1abaf4f613756f5733b08a1 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 10:40:41 +0800 Subject: [PATCH 087/131] =?UTF-8?q?python3.9=20json.loads=E4=B8=8D?= =?UTF-8?q?=E6=94=AF=E6=8C=81encoding=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 3ca45fc5..68dfe279 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -395,6 +395,6 @@ def get_bucket(self): encoded_policy = self.up_token.split(':')[-1] decode_policy = base64.urlsafe_b64decode(encoded_policy) decode_policy = decode_policy.decode('utf-8') - dict_policy = json.loads(decode_policy, encoding='utf-8') + dict_policy = json.loads(decode_policy) if dict_policy != {}: self.bucket_name = dict_policy['scope'].split(':')[0] From 4e61f5159f005743864f47a970036946accce460 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 10:43:06 +0800 Subject: [PATCH 088/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..9d95c7d7 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) assert ret['key'] == key From 153046225a26759a05df16181020df16deac1b15 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 11:08:41 +0800 Subject: [PATCH 089/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test_qiniu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_qiniu.py b/test_qiniu.py index 9d95c7d7..608b25fd 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -400,7 +400,7 @@ def test_put_stream_v2(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key From 8fd99e2366ff80a3821d70f2c0a5b575bd08c682 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 15:06:43 +0800 Subject: [PATCH 090/131] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E7=AD=96=E7=95=A5=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 16 +++++++++++++++- qiniu/services/storage/uploader.py | 9 ++------- test_qiniu.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index c0b3de43..d260df8a 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- - +import base64 import hmac import time from hashlib import sha1 @@ -158,6 +158,20 @@ def upload_token( return self.__upload_token(args) + @staticmethod + def up_token_decode(up_token): + up_token_list = up_token.split(':') + ak = up_token_list[0] + sign = base64.urlsafe_b64decode(up_token_list[1]) + decode_policy = base64.urlsafe_b64decode(up_token_list[2]) + decode_policy = decode_policy.decode('utf-8') + dict_policy = json.loads(decode_policy) + if dict_policy != {}: + bucket_name = dict_policy['scope'].split(':')[0] + else: + bucket_name = None + return ak, sign, bucket_name + def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) return self.token_with_data(data) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 68dfe279..0cb03fac 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -5,7 +5,7 @@ import os import time -from qiniu import config +from qiniu import config, Auth from qiniu.utils import urlsafe_base64_encode, crc32, file_crc32, _file_iter, rfc_from_timestamp from qiniu import http from .upload_progress_recorder import UploadProgressRecorder @@ -392,9 +392,4 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - encoded_policy = self.up_token.split(':')[-1] - decode_policy = base64.urlsafe_b64decode(encoded_policy) - decode_policy = decode_policy.decode('utf-8') - dict_policy = json.loads(decode_policy) - if dict_policy != {}: - self.bucket_name = dict_policy['scope'].split(':')[0] + self.bucket_name = Auth(None, None).up_token_decode(self.up_token)[-1] diff --git a/test_qiniu.py b/test_qiniu.py index 608b25fd..eb499c45 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -403,6 +403,17 @@ def test_put_stream_v2(self): self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) assert ret['key'] == key + def test_put_stream_v2_without_bucket_name(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + assert ret['key'] == key def test_big_file(self): key = 'big' From 5a4176320f2b02e1fbf9fb58b40e8fa074a358f5 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 15:16:14 +0800 Subject: [PATCH 091/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/auth.py | 6 +----- qiniu/services/storage/uploader.py | 5 +++-- test_qiniu.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index d260df8a..b386115b 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -166,11 +166,7 @@ def up_token_decode(up_token): decode_policy = base64.urlsafe_b64decode(up_token_list[2]) decode_policy = decode_policy.decode('utf-8') dict_policy = json.loads(decode_policy) - if dict_policy != {}: - bucket_name = dict_policy['scope'].split(':')[0] - else: - bucket_name = None - return ak, sign, bucket_name + return ak, sign, dict_policy def __upload_token(self, policy): data = json.dumps(policy, separators=(',', ':')) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 0cb03fac..f5c6e76d 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import base64 import hashlib import json import os @@ -392,4 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - self.bucket_name = Auth(None, None).up_token_decode(self.up_token)[-1] + pulicy = Auth(None, None).up_token_decode(self.up_token) + if pulicy != {}: + self.bucket_name = pulicy['scope'].split(':')[0] diff --git a/test_qiniu.py b/test_qiniu.py index eb499c45..8696e712 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -412,7 +412,7 @@ def test_put_stream_v2_without_bucket_name(self): token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=None) + self.mime_type, part_size=1024 * 1024 * 10, version='v2') assert ret['key'] == key def test_big_file(self): From dddfcefac91137bd943eeacc98bd2bd1b16bd4d3 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 15:19:15 +0800 Subject: [PATCH 092/131] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=BC=A9=E8=BF=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index f5c6e76d..644b7b04 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -393,4 +393,4 @@ def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': pulicy = Auth(None, None).up_token_decode(self.up_token) if pulicy != {}: - self.bucket_name = pulicy['scope'].split(':')[0] + self.bucket_name = pulicy['scope'].split(':')[0] From 369618ca7086183b34ea8d59761d8d9ae08e6610 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 15:23:35 +0800 Subject: [PATCH 093/131] ak,sk --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 644b7b04..74db6f49 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(None, None).up_token_decode(self.up_token) + pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token) if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 13e1d7c7cf848beac25253bc13ad9abbb5ce5cce Mon Sep 17 00:00:00 2001 From: okbang9 Date: Fri, 21 May 2021 15:27:41 +0800 Subject: [PATCH 094/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 74db6f49..a8bf47c0 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,7 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token) + pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), + os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 119d80b63f81100f7eca4c1606206d1ad416e1ea Mon Sep 17 00:00:00 2001 From: okbang9 Date: Mon, 24 May 2021 10:24:30 +0800 Subject: [PATCH 095/131] =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index a8bf47c0..1bc5ae6a 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,7 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth(os.getenv('QINIU_ACCESS_KEY'), - os.getenv('QINIU_SECRET_KEY')).up_token_decode(self.up_token)[-1] + pulicy = Auth.up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 2841baa62c4a5119647becdb97c7b10a500cad58 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Mon, 24 May 2021 16:23:39 +0800 Subject: [PATCH 096/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9issue=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20iunput=5Fstream=E8=AF=BB=E5=AE=8C=E5=90=8Eseek(0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- qiniu/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 1bc5ae6a..36e69662 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - pulicy = Auth.up_token_decode(self.up_token)[-1] + _, _, pulicy = Auth.up_token_decode(self.up_token)[-1] if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] diff --git a/qiniu/utils.py b/qiniu/utils.py index 8b7eaa66..07bcdeb6 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -90,6 +90,7 @@ def _file_iter(input_stream, size, offset=0): while d: yield d d = input_stream.read(size) + input_stream.seek(0) def _sha1(data): From 5897beb2e21f702b6ce8c2d61f831e9184692912 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Mon, 24 May 2021 16:25:17 +0800 Subject: [PATCH 097/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9issue=E9=97=AE?= =?UTF-8?q?=E9=A2=98=20=E8=A7=84=E8=8C=83get=5Fbucket=E5=8F=96=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 36e69662..d9e8af44 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - _, _, pulicy = Auth.up_token_decode(self.up_token)[-1] + _, _, pulicy = Auth.up_token_decode(self.up_token) if pulicy != {}: self.bucket_name = pulicy['scope'].split(':')[0] From 332168b212772c24cf8eafa6371e972a3db1d0bb Mon Sep 17 00:00:00 2001 From: okbang9 Date: Tue, 25 May 2021 15:23:31 +0800 Subject: [PATCH 098/131] pulicy->policy --- qiniu/services/storage/uploader.py | 6 +++--- test_qiniu.py | 34 ++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index d9e8af44..56626718 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -391,6 +391,6 @@ def get_parts(self, block_status): def get_bucket(self): if self.bucket_name is None or self.bucket_name == '': - _, _, pulicy = Auth.up_token_decode(self.up_token) - if pulicy != {}: - self.bucket_name = pulicy['scope'].split(':')[0] + _, _, policy = Auth.up_token_decode(self.up_token) + if policy != {}: + self.bucket_name = policy['scope'].split(':')[0] diff --git a/test_qiniu.py b/test_qiniu.py index 8696e712..8db8c5ea 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -390,18 +390,44 @@ def test_put_stream(self): self.mime_type, part_size=None, version=None, bucket_name=None) assert ret['key'] == key + def test_put_2m_stream_v2(self): + localfile = create_temp_file(2 * 1024 * 1024 + 1) + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + remove_temp_file(localfile) - def test_put_stream_v2(self): - localfile = __file__ + def test_put_4m_stream_v2(self): + localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size set_default(default_zone=Zone('http://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) - ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) + assert ret['key'] == key + remove_temp_file(localfile) + + def test_put_10m_stream_v2(self): + localfile = create_temp_file(10 * 1024 * 1024 + 1) + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, self.params, - self.mime_type, part_size=1024 * 1024 * 10, version='v2', bucket_name=bucket_name) + self.mime_type, part_size=1024 * 1024 * 4, version='v2', bucket_name=bucket_name) assert ret['key'] == key + remove_temp_file(localfile) def test_put_stream_v2_without_bucket_name(self): localfile = __file__ From 627188ec61c5f40b5bdb57569d3aa163918eae97 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Tue, 25 May 2021 16:34:04 +0800 Subject: [PATCH 099/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 175 ++++++++++++++++++++++++++++++++++++++++++++++ qiniu/__init__.py | 30 ++++++++ qiniu/config.py | 50 +++++++++++++ 3 files changed, 255 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 qiniu/__init__.py create mode 100644 qiniu/config.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..92ecd531 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,175 @@ +# Changelog + +## 7.4.0 (2021-05-21) +* 支持分片上传 v2 + +## 7.3.1 (2021-01-06) +* 修复 ResponseInfo 对扩展码错误处理问题 +* 增加 python v3.7，v3.8，v3.9 版本 CI 测试 + +## 7.3.0 (2020-09-23) +新增 +* sms[云短信]：新增查询短信发送记录方法：get_messages_info +* cdn: 新增上线域名 domain_online 方法、下线域名 domain_offline 方法和删除域名 delete_domain 方法 +* 对象存储：新增批量解冻build_batch_restoreAr方法、获取空间列表bucket_domain方法和修改空间访问权限change_bucket_permission方法 + +## 7.2.10 （2020-08-21） +* 修复上传策略中forceSaveKey参数没有签算进上传token，导致上传失败的问题 +## 7.2.9 (2020-08-07) +* 支持指定本地ctx缓存文件.qiniu_pythonsdk_hostscache.json 文件路径 +* 更正接口返回描述docstring +* 修复接口对非json response 处理 +* ci 覆盖增加python 3.6 3.7 +* 修复获取域名列方法 +* 修复python3 环境下，二进制对象上传问题 + + +## 7.2.8(2020-03-27) +* add restoreAr + +## 7.2.7(2020-03-10) +* fix bucket_info + +## 7.2.6(2019-06-26) +* 添加sms + +## 7.2.5 (2019-06-06) +* 添加sms + +## 7.2.4 (2019-04-01) +* 默认导入region类 + +## 7.2.3 (2019-02-25) +* 新增region类，zone继承 +* 上传可以指定上传域名 +* 新增上传指定上传空间和qvm指定上传内网的例子 +* 新增列举账号空间，创建空间，查询空间信息，改变文件状态接口，并提供例子 + +## 7.2.2 (2018-05-10) +* 增加连麦rtc服务端API功能 + +## 7.2.0（2017-11-23） +* 修复put_data不支持file like object的问题 +* 增加空间写错时，抛出异常提示客户的功能 +* 增加创建空间的接口功能 + +## 7.1.9（2017-11-01） +* 修复python2情况下，中文文件名上传失败的问题 +* 修复python2环境下，中文文件使用分片上传时失败的问题 + +## 7.1.8 （2017-10-18） +* 恢复kirk的API为原来的状态 + +## 7.1.7 (2017-09-27) + +* 修复从时间戳获取rfc http格式的时间字符串问题 + +## 7.1.6 (2017-09-26) + +* 给 `put_file` 功能增加保持本地文件Last Modified功能，以支持切换源站的客户CDN不回源 + +## 7.1.5 (2017-08-26) + +* 设置表单上传默认校验crc32 +* 增加PutPolicy新参数isPrefixalScope +* 修复手动指定的zone无效的问题 + +## 7.1.4 (2017-06-05) +### 修正 +* cdn功能中获取域名日志列表的参数错误 + +## 7.1.2 (2017-03-24) +### 增加 +* 增加设置文件生命周期的接口 + +## 7.1.1 (2017-02-03) +### 增加 +* 增加cdn刷新，预取，日志获取，时间戳防盗链生成功能 + +### 修正 +* 修复分片上传的upload record path遇到中文时的问题，现在使用md5来计算文件名 + +## 7.1.0 (2016-12-08) +### 增加 +* 通用计算支持 + +## 7.0.10 (2016-11-29) +### 修正 +* 去掉homedir + +## 7.0.9 (2016-10-09) +### 增加 +* 多机房接口调用支持 + +## 7.0.8 (2016-07-05) +### 修正 +* 修复表单上传大于20M文件的400错误 + +### 增加 +* copy 和 move 操作增加 force 字段，允许强制覆盖 copy 和 move +* 增加上传策略 deleteAfterDays 字段 +* 一些 demo + +## 7.0.7 (2016-05-05) +### 修正 +* 修复大于4M的文件hash计算错误的问题 +* add fname + +### 增加 +* 一些demo +* travis 直接发布 + +## 7.0.6 (2015-12-05) +### 修正 +* 2.x unicode 问题 by @hunter007 +* 上传重试判断 +* 上传时 dns劫持处理 + +### 增加 +* fsizeMin 上传策略 +* 断点上传记录 by @hokein +* 计算stream etag +* 3.5 ci 支持 + +## 7.0.5 (2015-06-25) +### 变更 +* 配置up_host 改为配置zone + +### 增加 +* fectch 支持不指定key + +## 7.0.4 (2015-05-04) +### 修正 +* 上传重试为空文件 +* 回调应该只对form data 签名。 + +## 7.0.3 (2015-03-11) +### 增加 +* 可以配置 io/rs/api/rsf host + +## 7.0.2 (2014-12-24) +### 修正 +* 内部http get当没有auth会出错 +* python3下的qiniupy 没有参数时 arg parse会抛异常 +* 增加callback policy + +## 7.0.1 (2014-11-26) +### 增加 +* setup.py从文件中读取版本号，而不是用导入方式 +* 补充及修正了一些单元测试 + +## 7.0.0 (2014-11-13) + +### 增加 +* 简化上传接口 +* 自动选择断点续上传还是直传 +* 重构代码，接口和内部结构更清晰 +* 同时支持python 2.x 和 3.x +* 支持pfop +* 支持verify callback +* 改变mime +* 代码覆盖度报告 +* policy改为dict, 便于灵活增加，并加入过期字段检查 +* 文件列表支持目录形式 + + diff --git a/qiniu/__init__.py b/qiniu/__init__.py new file mode 100644 index 00000000..dffdaf7e --- /dev/null +++ b/qiniu/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +''' +Qiniu Resource Storage SDK for Python +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For detailed document, please see: + +''' + +# flake8: noqa + +__version__ = '7.4.0' + +from .auth import Auth, QiniuMacAuth + +from .config import set_default +from .zone import Zone +from .region import Region + +from .services.storage.bucket import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, \ + build_batch_stat, build_batch_delete, build_batch_restoreAr +from .services.storage.uploader import put_data, put_file, put_stream +from .services.cdn.manager import CdnManager, create_timestamp_anti_leech_url, DomainManager +from .services.processing.pfop import PersistentFop +from .services.processing.cmd import build_op, pipe_cmd, op_save +from .services.compute.app import AccountClient +from .services.compute.qcos_api import QcosClient +from .services.sms.sms import Sms +from .services.pili.rtc_server_manager import RtcServer, get_room_token +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry diff --git a/qiniu/config.py b/qiniu/config.py new file mode 100644 index 00000000..a137a67f --- /dev/null +++ b/qiniu/config.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +from qiniu import zone + +RS_HOST = 'http://rs.qiniu.com' # 管理操作Host +RSF_HOST = 'http://rsf.qbox.me' # 列举操作Host +API_HOST = 'http://api.qiniu.com' # 数据处理操作Host +UC_HOST = 'https://uc.qbox.me' # 获取空间信息Host + +_BLOCK_SIZE = 1024 * 1024 * 4 # 断点续传分块大小，该参数为接口规格，暂不支持修改 + +_config = { + 'default_zone': zone.Zone(), + 'default_rs_host': RS_HOST, + 'default_rsf_host': RSF_HOST, + 'default_api_host': API_HOST, + 'default_uc_host': UC_HOST, + 'connection_timeout': 30, # 链接超时为时间为30s + 'connection_retries': 3, # 链接重试次数为3次 + 'connection_pool': 10, # 链接池个数为10 + 'default_upload_threshold': 2 * _BLOCK_SIZE # put_file上传方式的临界默认值 +} + + +def get_default(key): + return _config[key] + + +def set_default( + default_zone=None, connection_retries=None, connection_pool=None, + connection_timeout=None, default_rs_host=None, default_uc_host=None, + default_rsf_host=None, default_api_host=None, default_upload_threshold=None): + if default_zone: + _config['default_zone'] = default_zone + if default_rs_host: + _config['default_rs_host'] = default_rs_host + if default_rsf_host: + _config['default_rsf_host'] = default_rsf_host + if default_api_host: + _config['default_api_host'] = default_api_host + if default_uc_host: + _config['default_uc_host'] = default_uc_host + if connection_retries: + _config['connection_retries'] = connection_retries + if connection_pool: + _config['connection_pool'] = connection_pool + if connection_timeout: + _config['connection_timeout'] = connection_timeout + if default_upload_threshold: + _config['default_upload_threshold'] = default_upload_threshold From 8f3193a2cccbe892064f61ca0b6357f6c3117573 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Tue, 25 May 2021 18:17:50 +0800 Subject: [PATCH 100/131] =?UTF-8?q?=E5=8D=87=E7=BA=A7=E5=88=B0=20v7.4.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 +++ qiniu/__init__.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92ecd531..98764db4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.4.1 (2021-05-25) +* 分片上传 v2 方法不再强制要求 bucket_name 参数 + ## 7.4.0 (2021-05-21) * 支持分片上传 v2 diff --git a/qiniu/__init__.py b/qiniu/__init__.py index dffdaf7e..16c52eb8 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.4.0' +__version__ = '7.4.1' from .auth import Auth, QiniuMacAuth From b24e85b9158ba17abe490a23f061abe872f92c1e Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 26 May 2021 14:41:23 +0800 Subject: [PATCH 101/131] =?UTF-8?q?=E5=8E=BB=E6=8E=89bucket=5Fname?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 56626718..3a96fbdf 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -62,7 +62,6 @@ def put_file(up_token, key, file_path, params=None, hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段必传字段 空间名称 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -162,7 +161,6 @@ class _Resume(object): hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段必传字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, From d385c08d9852e3d0fc05ef36e12759821c9bce34 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 26 May 2021 15:18:10 +0800 Subject: [PATCH 102/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/services/storage/uploader.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 3a96fbdf..03729790 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -62,6 +62,7 @@ def put_file(up_token, key, file_path, params=None, hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段 空间名称 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -161,6 +162,7 @@ class _Resume(object): hostscache_dir： host请求 缓存文件保存位置 version 分片上传版本 目前支持v1/v2版本 默认v1 part_size 分片上传v2必传字段 分片大小范围为1 MB - 1 GB + bucket_name 分片上传v2字段 空间名称 """ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, From 21c83ed441c3ebd310a3d948f3c058fba2ca598a Mon Sep 17 00:00:00 2001 From: okbang9 Date: Tue, 15 Jun 2021 12:01:08 +0800 Subject: [PATCH 103/131] =?UTF-8?q?=E4=BF=AE=E6=94=B9hosts=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=A7=A3=E6=9E=90=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/region.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index f143795b..d253e0d5 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- - +import logging import os import time +import trace + import requests from qiniu import compat from qiniu import utils @@ -127,6 +129,9 @@ def get_bucket_hosts_to_cache(self, key, home_dir): if (len(self.host_cache) == 0): self.host_cache_from_file(home_dir) + if self.host_cache == {}: + return ret + if (not (key in self.host_cache)): return ret @@ -147,8 +152,11 @@ def host_cache_from_file(self, home_dir): if not os.path.isfile(path): return None with open(path, 'r') as f: - bucket_hosts = compat.json.load(f) - self.host_cache = bucket_hosts + try: + bucket_hosts = compat.json.load(f) + self.host_cache = bucket_hosts + except Exception as e: + logging.error(e) f.close() return From 0461a4301f51f996979bb9cd525506d97f7a8135 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Tue, 15 Jun 2021 12:04:55 +0800 Subject: [PATCH 104/131] =?UTF-8?q?=E8=A7=84=E8=8C=83=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiniu/region.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qiniu/region.py b/qiniu/region.py index d253e0d5..c1f906dc 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -2,8 +2,6 @@ import logging import os import time -import trace - import requests from qiniu import compat from qiniu import utils From 2f7f74292401a3994973de7d8af98b462476a0fb Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 1 Sep 2021 12:21:22 +0800 Subject: [PATCH 105/131] add decode except --- qiniu/http.py | 8 ++++++-- qiniu/region.py | 4 +--- qiniu/services/storage/upload_progress_recorder.py | 5 ++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index c28028fb..db70bc6a 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +import json +import logging import platform import requests @@ -23,8 +25,10 @@ def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) resp.encoding = 'utf-8' - ret = resp.json() if resp.text != '' else {} - if ret is None: # json null + try: + ret = json.loads(resp.text) + except ValueError: + logging.debug("response body decode error: %s" % resp.text) ret = {} return ret, ResponseInfo(resp) diff --git a/qiniu/region.py b/qiniu/region.py index c1f906dc..3fdda809 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -11,9 +11,7 @@ class Region(object): """七牛上传区域类 - 该类主要内容上传区域地址。 - """ def __init__( @@ -171,4 +169,4 @@ def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data + return data \ No newline at end of file diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 4b4af7ac..8cf70cdb 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -37,7 +37,10 @@ def get_upload_record(self, file_name, key): if not os.path.isfile(upload_record_file_path): return None with open(upload_record_file_path, 'r') as f: - json_data = json.load(f) + try: + json_data = json.load(f) + except ValueError: + json_data = None return json_data def set_upload_record(self, file_name, key, data): From 6a91117f6c5b0dc4d1c6f1aaa35007a65334f417 Mon Sep 17 00:00:00 2001 From: okbang9 Date: Wed, 1 Sep 2021 12:31:41 +0800 Subject: [PATCH 106/131] add new line --- qiniu/region.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/region.py b/qiniu/region.py index 3fdda809..9f8db051 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -169,4 +169,4 @@ def bucket_hosts(self, ak, bucket): url = "{0}/v1/query?ak={1}&bucket={2}".format(UC_HOST, ak, bucket) ret = requests.get(url) data = compat.json.dumps(ret.json(), separators=(',', ':')) - return data \ No newline at end of file + return data From 9bf65007db97904308ffeefdc27b9b097049c417 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Wed, 1 Sep 2021 14:44:13 +0800 Subject: [PATCH 107/131] try to handle hosts file read error --- qiniu/services/storage/upload_progress_recorder.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/qiniu/services/storage/upload_progress_recorder.py b/qiniu/services/storage/upload_progress_recorder.py index 8cf70cdb..83a0dd65 100644 --- a/qiniu/services/storage/upload_progress_recorder.py +++ b/qiniu/services/storage/upload_progress_recorder.py @@ -36,11 +36,15 @@ def get_upload_record(self, file_name, key): upload_record_file_path = os.path.join(self.record_folder, record_file_name) if not os.path.isfile(upload_record_file_path): return None - with open(upload_record_file_path, 'r') as f: - try: - json_data = json.load(f) - except ValueError: - json_data = None + try: + with open(upload_record_file_path, 'r') as f: + try: + json_data = json.load(f) + except ValueError: + json_data = None + except IOError: + json_data = None + return json_data def set_upload_record(self, file_name, key, data): From dfe78d0a4d79b8565456c5962f235e637a9250eb Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Wed, 22 Sep 2021 18:07:42 +0800 Subject: [PATCH 108/131] add keylimit to upload policy --- qiniu/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qiniu/auth.py b/qiniu/auth.py index b386115b..f74e40bf 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,6 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 + 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，api必须同时提供keyname,目前只在表单和新/旧分片上传中生效(mkfile, initial-parts,upload-parts,complete-parts) 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From 731d645b7beea2915ebd79bfa5cd9f78fe06db1f Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 23 Sep 2021 09:45:39 +0800 Subject: [PATCH 109/131] add test case for keylimit upload policy --- test_qiniu.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/test_qiniu.py b/test_qiniu.py index c6b1c661..9309597f 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -462,6 +462,23 @@ def test_retry(self): assert ret['key'] == key assert ret['hash'] == etag(localfile) + def test_put_stream_with_key_limits(self): + localfile = __file__ + key = 'test_file_r' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type) + assert info.status_code == 403 + token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d', 'test_file_r']}) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, + self.mime_type) + assert info.status_code == 200 + class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) From 7a6c7d3f2305022951cd4bb638f58254382a8e5c Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 23 Sep 2021 12:02:54 +0800 Subject: [PATCH 110/131] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98764db4..9f413e2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.5.0 (2021-09-23) +* 上传策略新增对部分字段支持 + ## 7.4.1 (2021-05-25) * 分片上传 v2 方法不再强制要求 bucket_name 参数 From 5bd1ce168067c603dab3a994b6143b9a36040920 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 23 Sep 2021 12:04:45 +0800 Subject: [PATCH 111/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 16c52eb8..76db3375 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.4.1' +__version__ = '7.5.0' from .auth import Auth, QiniuMacAuth From 2466f5ea210db1ec88a021705d83ee655ea9684f Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 23 Sep 2021 12:05:25 +0800 Subject: [PATCH 112/131] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index b2eb13c4..1b962d06 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,6 @@ import qiniu ``` 更多参见SDK使用指南： http://developer.qiniu.com/code/v7/sdk/python.html - -### 命令行工具 -安装完后附带有命令行工具，可以计算etag -```bash -$ qiniupy etag yourfile ``` ## 测试 From 526a10cd23e0cb593d105b0b2403ae73b3597d7c Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Thu, 23 Sep 2021 12:32:15 +0800 Subject: [PATCH 113/131] use badge from github action instead of travis --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1b962d06..7846505d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE) -[![Build Status](https://travis-ci.org/qiniu/python-sdk.svg)](https://travis-ci.org/qiniu/python-sdk) +[![Build Status](https://github.com/qiniu/python-sdk/actions/workflows/ci-test.yml/badge.svg)](https://travis-ci.org/qiniu/python-sdk) [![GitHub release](https://img.shields.io/github/v/tag/qiniu/python-sdk.svg?label=release)](https://github.com/qiniu/python-sdk/releases) [![Latest Stable Version](https://img.shields.io/pypi/v/qiniu.svg)](https://pypi.python.org/pypi/qiniu) [![Download Times](https://img.shields.io/pypi/dm/qiniu.svg)](https://pypi.python.org/pypi/qiniu) @@ -19,10 +19,10 @@ $ pip install qiniu ## 运行环境 -| Qiniu SDK版本 | Python 版本 | -|:--------------------:|:---------------------------:| -| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9| -| 6.x | 2.7 | +| Qiniu SDK版本 | Python 版本 | +| :-----------: | :------------------------------------: | +| 7.x | 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9 | +| 6.x | 2.7 | ## 使用方法 From 5bf64deeaaf7f028f2822609b2788f06b18ac199 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 23 Sep 2021 15:02:59 +0800 Subject: [PATCH 114/131] Update auth.py --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index f74e40bf..3f826c77 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,7 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 - 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，api必须同时提供keyname,目前只在表单和新/旧分片上传中生效(mkfile, initial-parts,upload-parts,complete-parts) + 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，上传时必须提供key 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From c160e086d4cb5591f389f2891f6da09b3451951c Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 23 Sep 2021 19:40:20 +0800 Subject: [PATCH 115/131] Update auth.py --- qiniu/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index 3f826c77..a1680301 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -28,7 +28,7 @@ 'mimeLimit', # MimeType限制 'fsizeLimit', # 上传文件大小限制 'fsizeMin', # 上传文件最少字节数 - 'keylimit', # 设置允许的key列表，不超过20个，如果设置了这个字段，上传时必须提供key + 'keylimit', # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key 'persistentOps', # 持久化处理操作 'persistentNotifyUrl', # 持久化处理结果通知URL From 3de48b399e91d8833e9889a0f6b045afc54f15de Mon Sep 17 00:00:00 2001 From: LiHS Date: Mon, 13 Dec 2021 11:23:52 +0800 Subject: [PATCH 116/131] add json decode error test --- qiniu/http.py | 3 +-- test_qiniu.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/qiniu/http.py b/qiniu/http.py index db70bc6a..93600f56 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import json import logging import platform @@ -26,7 +25,7 @@ def __return_wrapper(resp): return None, ResponseInfo(resp) resp.encoding = 'utf-8' try: - ret = json.loads(resp.text) + ret = resp.json() except ValueError: logging.debug("response body decode error: %s" % resp.text) ret = {} diff --git a/test_qiniu.py b/test_qiniu.py index c6b1c661..15a27f11 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -21,6 +21,8 @@ from qiniu.services.storage.uploader import _form_put +from qiniu.http import __return_wrapper as return_wrapper + import qiniu.config if is_py2: @@ -75,6 +77,23 @@ def is_travis(): return os.environ['QINIU_TEST_ENV'] == 'travis' +class HttpTest(unittest.TestCase): + def test_json_decode_error(self): + def mock_res(): + r = requests.Response() + r.status_code = 200 + r.headers.__setitem__('X-Reqid', 'mockedReqid') + + def json_func(): + raise ValueError('%s: line %d column %d (char %d)' % ('Expecting value', 0, 0, 0)) + r.json = json_func + + return r + mocked_res = mock_res() + ret, _ = return_wrapper(mocked_res) + assert ret == {} + + class UtilsTest(unittest.TestCase): def test_urlsafe(self): a = 'hello\x96' From f2043430782e5c691e122728b72399d491ba7104 Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 8 Mar 2022 14:23:09 +0800 Subject: [PATCH 117/131] fix auth Qiniu sign generate wrong raw text --- qiniu/__init__.py | 2 +- qiniu/auth.py | 34 +++++---- qiniu/utils.py | 32 +++++++++ test_qiniu.py | 173 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 226 insertions(+), 15 deletions(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 76db3375..7b647f36 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -27,4 +27,4 @@ from .services.compute.qcos_api import QcosClient from .services.sms.sms import Sms from .services.pili.rtc_server_manager import RtcServer, get_room_token -from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry +from .utils import urlsafe_base64_encode, urlsafe_base64_decode, etag, entry, canonical_mime_header_key diff --git a/qiniu/auth.py b/qiniu/auth.py index a1680301..a1065d06 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -5,7 +5,7 @@ from hashlib import sha1 from requests.auth import AuthBase from .compat import urlparse, json, b -from .utils import urlsafe_base64_encode +from .utils import urlsafe_base64_encode, canonical_mime_header_key # 上传策略，参数规格详见 # https://developer.qiniu.com/kodo/manual/1206/put-policy @@ -265,15 +265,21 @@ def token_of_request( path_with_query = path if query != '': path_with_query = ''.join([path_with_query, '?', query]) - data = ''.join(["%s %s" % - (method, path_with_query), "\n", "Host: %s" % - host, "\n"]) + data = ''.join([ + "%s %s" % (method, path_with_query), + "\n", + "Host: %s" % host + ]) if content_type: - data += "Content-Type: %s" % (content_type) + "\n" + data += "\n" + data += "Content-Type: %s" % content_type - data += qheaders - data += "\n" + if qheaders: + data += "\n" + data += qheaders + + data += "\n\n" if content_type and content_type != "application/octet-stream" and body: if isinstance(body, bytes): @@ -283,11 +289,13 @@ def token_of_request( return '{0}:{1}'.format(self.__access_key, self.__token(data)) def qiniu_headers(self, headers): - res = "" - for key in headers: - if key.startswith(self.qiniu_header_prefix): - res += key + ": %s\n" % (headers.get(key)) - return res + qiniu_fields = list(filter( + lambda k: k.startswith(self.qiniu_header_prefix) and len(k) > len(self.qiniu_header_prefix), + headers, + )) + return "\n".join([ + "%s: %s" % (canonical_mime_header_key(key), headers.get(key)) for key in sorted(qiniu_fields) + ]) @staticmethod def __checkKey(access_key, secret_key): @@ -300,6 +308,8 @@ def __init__(self, auth): self.auth = auth def __call__(self, r): + if r.headers.get('Content-Type', None) is None: + r.headers['Content-Type'] = 'application/x-www-form-urlencoded' token = self.auth.token_of_request( r.method, r.headers.get('Host', None), r.url, self.auth.qiniu_headers(r.headers), diff --git a/qiniu/utils.py b/qiniu/utils.py index 07bcdeb6..92a81449 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -172,3 +172,35 @@ def rfc_from_timestamp(timestamp): last_modified_str = last_modified_date.strftime( '%a, %d %b %Y %H:%M:%S GMT') return last_modified_str + + +def _valid_header_key_char(ch): + is_token_table = [ + "!", "#", "$", "%", "&", "\\", "*", "+", "-", ".", + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", + "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", + "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", + "U", "W", "V", "X", "Y", "Z", + "^", "_", "`", + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", + "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", + "u", "v", "w", "x", "y", "z", + "|", "~"] + return 0 <= ord(ch) < 128 and ch in is_token_table + + +def canonical_mime_header_key(field_name): + for ch in field_name: + if not _valid_header_key_char(ch): + return field_name + result = "" + upper = True + for ch in field_name: + if upper and "a" <= ch <= "z": + result += ch.upper() + elif not upper and "A" <= ch <= "Z": + result += ch.lower() + else: + result += ch + upper = ch == "-" + return result diff --git a/test_qiniu.py b/test_qiniu.py index 9309597f..4964dd3c 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -11,11 +11,11 @@ import unittest import pytest -from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone +from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone, QiniuMacAuth from qiniu import put_data, put_file, put_stream from qiniu import BucketManager, build_batch_copy, build_batch_rename, build_batch_move, build_batch_stat, \ build_batch_delete, DomainManager -from qiniu import urlsafe_base64_encode, urlsafe_base64_decode +from qiniu import urlsafe_base64_encode, urlsafe_base64_decode, canonical_mime_header_key from qiniu.compat import is_py2, is_py3, b @@ -81,6 +81,37 @@ def test_urlsafe(self): u = urlsafe_base64_encode(a) assert b(a) == urlsafe_base64_decode(u) + def test_canonical_mime_header_key(self): + field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "content-type", + "CONTENT-LENGTH", + "oRiGin", + "ReFer", + "Last-Modified", + "acCePt-ChArsEt", + "x-test-3", + "cache-control", + ] + expect_canonical_field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "Content-Type", + "Content-Length", + "Origin", + "Refer", + "Last-Modified", + "Accept-Charset", + "X-Test-3", + "Cache-Control", + ] + assert len(field_names) == len(expect_canonical_field_names) + for i in range(len(field_names)): + assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] + class AuthTestCase(unittest.TestCase): def test_token(self): @@ -103,6 +134,144 @@ def test_token_of_request(self): token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' + def test_QiniuMacRequestsAuth(self): + auth = QiniuMacAuth("ak", "sk") + test_cases = [ + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:0i1vKClRDWFyNkcTFzwcE7PzX74=", + }, + { + "method": "GET", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:K1DI0goT05yhGizDFE5FiPJxAj4=", + }, + { + "method": "POST", + "host": None, + "url": "", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:0ujEjW_vLRZxebsveBgqa3JyQ-w=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "X-Qiniu-": "a", + "X-Qiniu": "b", + "Content-Type": "application/x-www-form-urlencoded", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:GShw5NitGmd5TLoo38nDkGUofRw=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/json", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/json", + "body": "{\"name\": \"test\"}", + "except_sign_token": "ak:DhNA1UCaBqSHCsQjMOLRfVn63GQ=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:KUAhrYh32P9bv0COD8ugZjDCmII=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:fkRck5_LeyfwdkyyLk-hyNwGKac=", + }, + { + "method": "GET", + "host": "upload.qiniup.com", + "url": "http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df", + "qheaders": { + "Content-Type": "application/x-www-form-urlencoded", + "X-Qiniu-Bbb": "BBB", + "X-Qiniu-Aaa": "DDD", + "X-Qiniu-": "a", + "X-Qiniu": "b", + }, + "content_type": "application/x-www-form-urlencoded", + "body": "name=test&language=go", + "except_sign_token": "ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=", + }, + ] + + for test_case in test_cases: + sign_token = auth.token_of_request( + method=test_case["method"], + host=test_case["host"], + url=test_case["url"], + qheaders=auth.qiniu_headers(test_case["qheaders"]), + content_type=test_case["content_type"], + body=test_case["body"], + ) + assert sign_token == test_case["except_sign_token"] + def test_verify_callback(self): body = 'name=sunflower.jpg&hash=Fn6qeQi4VDLQ347NiRm-RlQx_4O2&location=Shanghai&price=1500.00&uid=123' url = 'test.qiniu.com/callback' From d9d890e94a53b8d282d71ce9c9d19547232e40ec Mon Sep 17 00:00:00 2001 From: LiHS Date: Thu, 10 Mar 2022 15:09:48 +0800 Subject: [PATCH 118/131] promote Qiniu sign in bucket --- qiniu/auth.py | 3 +++ qiniu/http.py | 35 +++++++++----------------------- qiniu/services/storage/bucket.py | 7 ++++--- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/qiniu/auth.py b/qiniu/auth.py index a1065d06..5c2819de 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -58,6 +58,9 @@ def __init__(self, access_key, secret_key): def get_access_key(self): return self.__access_key + def get_secret_key(self): + return self.__secret_key + def __token(self, data): data = b(data) hashed = hmac.new(self.__secret_key, data, sha1) diff --git a/qiniu/http.py b/qiniu/http.py index c28028fb..c3dc69a1 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -76,16 +76,16 @@ def _get(url, params, auth, headers=None): if _session is None: _init() try: - post_headers = _headers.copy() + get_headers = _headers.copy() if headers is not None: for k, v in headers.items(): - post_headers.update({k: v}) + get_headers.update({k: v}) r = _session.get( url, params=params, auth=auth, timeout=config.get_default('connection_timeout'), - headers=post_headers) + headers=get_headers) except Exception as e: return None, ResponseInfo(None, e) return __return_wrapper(r) @@ -150,31 +150,16 @@ def _put_with_qiniu_mac_and_headers(url, data, auth, headers): def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( - auth) if auth is not None else None - timeout = config.get_default('connection_timeout') - try: - r = requests.post( - url, - json=data, - auth=qn_auth, - timeout=timeout, - headers=_headers) - except Exception as e: - return None, ResponseInfo(None, e) - return __return_wrapper(r) + auth + ) if auth is not None else None + return _post(url, data, None, qn_auth) def _get_with_qiniu_mac(url, params, auth): - try: - r = requests.get( - url, - params=params, - auth=qiniu.auth.QiniuMacRequestsAuth(auth) if auth is not None else None, - timeout=config.get_default('connection_timeout'), - headers=_headers) - except Exception as e: - return None, ResponseInfo(None, e) - return __return_wrapper(r) + qn_auth = qiniu.auth.QiniuMacRequestsAuth( + auth + ) if auth is not None else None + return _get(url, params, qn_auth) def _get_with_qiniu_mac_and_headers(url, params, auth, headers): diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 2f54260a..187a5433 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from qiniu import config +from qiniu import config, QiniuMacAuth from qiniu import http from qiniu.utils import urlsafe_base64_encode, entry @@ -17,6 +17,7 @@ class BucketManager(object): def __init__(self, auth, zone=None): self.auth = auth + self.mac_auth = QiniuMacAuth(auth.get_access_key(), auth.get_secret_key()) if (zone is None): self.zone = config.get_default('default_zone') else: @@ -387,10 +388,10 @@ def __server_do(self, host, operation, *args): return self.__post(url) def __post(self, url, data=None): - return http._post_with_auth(url, data, self.auth) + return http._post_with_qiniu_mac(url, data, self.mac_auth) def __get(self, url, params=None): - return http._get_with_auth(url, params, self.auth) + return http._get_with_qiniu_mac(url, params, self.mac_auth) def _build_op(*args): From 40f0b8a8cbee8146c2584731d25d2d49863d06bf Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Mon, 28 Mar 2022 10:48:19 +0800 Subject: [PATCH 119/131] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f413e2b..7b85e8db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 7.6.0 (2022-03-28) +* 优化了错误处理机制 +* 支持 [Qiniu](https://developer.qiniu.com/kodo/1201/access-token) 签名算法 + ## 7.5.0 (2021-09-23) * 上传策略新增对部分字段支持 From e71237c570daaa2d7bad98698bfb34e1d86988dd Mon Sep 17 00:00:00 2001 From: Bai Long Date: Mon, 28 Mar 2022 11:58:34 +0800 Subject: [PATCH 120/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 7b647f36..5da35d0e 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.5.0' +__version__ = '7.6.0' from .auth import Auth, QiniuMacAuth From bb6f167aecf36e8f8f3a4349da997f02ef57213e Mon Sep 17 00:00:00 2001 From: LiHS Date: Wed, 20 Apr 2022 11:58:08 +0800 Subject: [PATCH 121/131] support set object lifecycle --- examples/set_object_lifecycle.py | 34 +++++++++++++++++++++++ qiniu/auth.py | 2 +- qiniu/services/storage/bucket.py | 47 +++++++++++++++++++++++++++++--- test_qiniu.py | 34 +++++++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 examples/set_object_lifecycle.py diff --git a/examples/set_object_lifecycle.py b/examples/set_object_lifecycle.py new file mode 100644 index 00000000..45005458 --- /dev/null +++ b/examples/set_object_lifecycle.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +from qiniu import Auth +from qiniu import BucketManager + +access_key = 'your_ak' +secret_key = 'your_sk' + +# 初始化 Auth +q = Auth(access_key, secret_key) + +# 初始化 BucketManager +bucket = BucketManager(q) + +# 目标空间 +bucket_name = 'your_bucket_name' +# 目标 key +key = 'path/to/key' + +# bucket_name 更新 rule +ret, info = bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40, + cond={ + 'hash': 'object_hash' + } +) +print(ret) +print(info) diff --git a/qiniu/auth.py b/qiniu/auth.py index 5c2819de..100500a9 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -34,7 +34,7 @@ 'persistentNotifyUrl', # 持久化处理结果通知URL 'persistentPipeline', # 持久化处理独享队列 'deleteAfterDays', # 文件多少天后自动删除 - 'fileType', # 文件的存储类型，0为普通存储，1为低频存储 + 'fileType', # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储 'isPrefixalScope' # 指定上传文件必须使用的前缀 ]) diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 187a5433..46ec2045 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -219,21 +219,21 @@ def change_mime(self, bucket, key, mime): def change_type(self, bucket, key, storage_type): """修改文件的存储类型 - 修改文件的存储类型为普通存储或者是低频存储，参考文档： + 修改文件的存储类型，参考文档： https://developer.qiniu.com/kodo/api/3710/modify-the-file-type Args: bucket: 待操作资源所在空间 key: 待操作资源文件名 - storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储 + storage_type: 待操作资源存储类型，0为普通存储，1为低频存储，2 为归档存储，3 为深度归档 """ resource = entry(bucket, key) return self.__rs_do('chtype', resource, 'type/{0}'.format(storage_type)) def restoreAr(self, bucket, key, freezeAfter_days): - """解冻归档存储文件 + """解冻归档存储、深度归档存储文件 - 修改文件的存储类型为普通存储或者是低频存储，参考文档： + 对归档存储、深度归档存储文件，进行解冻操作参考文档： https://developer.qiniu.com/kodo/api/6380/restore-archive Args: @@ -263,6 +263,45 @@ def change_status(self, bucket, key, status, cond): return self.__rs_do('chstatus', resource, 'status/{0}'.format(status), 'cond', condstr) return self.__rs_do('chstatus', resource, 'status/{0}'.format(status)) + def set_object_lifecycle( + self, + bucket, + key, + to_line_after_days=0, + to_archive_after_days=0, + to_deep_archive_after_days=0, + delete_after_days=0, + cond=None + ): + """ + + 设置对象的生命周期 + + Args: + bucket: 目标空间 + key: 目标资源 + to_line_after_days: 多少天后将文件转为低频存储，设置为 -1 表示取消已设置的转低频存储的生命周期规则， 0 表示不修改转低频生命周期规则。 + to_archive_after_days: 多少天后将文件转为归档存储，设置为 -1 表示取消已设置的转归档存储的生命周期规则， 0 表示不修改转归档生命周期规则。 + to_deep_archive_after_days: 多少天后将文件转为深度归档存储，设置为 -1 表示取消已设置的转深度归档存储的生命周期规则， 0 表示不修改转深度归档生命周期规则 + delete_after_days: 多少天后将文件删除，设置为 -1 表示取消已设置的删除存储的生命周期规则， 0 表示不修改删除存储的生命周期规则。 + cond: 匹配条件，只有条件匹配才会设置成功，当前支持设置 hash、mime、fsize、putTime。 + + Returns: + resBody, respInfo + + """ + options = [ + 'toIAAfterDays', str(to_line_after_days), + 'toArchiveAfterDays', str(to_archive_after_days), + 'toDeepArchiveAfterDays', str(to_deep_archive_after_days), + 'deleteAfterDays', str(delete_after_days) + ] + if cond and isinstance(cond, dict): + cond_str = '&'.join(["{0}={1}".format(k, v) for k, v in cond.items()]) + options += ['cond', urlsafe_base64_encode(cond_str)] + resource = entry(bucket, key) + return self.__rs_do('lifecycle', resource, *options) + def batch(self, operations): """批量操作: diff --git a/test_qiniu.py b/test_qiniu.py index 30687f04..8e1552e0 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -460,6 +460,40 @@ def test_delete_after_days(self): ret, info = self.bucket.delete_after_days(bucket_name, key, days) assert info.status_code == 200 + def test_set_object_lifecycle(self): + key = 'test_set_object_lifecycle' + rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + assert info.status_code == 200 + ret, info = self.bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40 + ) + assert info.status_code == 200 + + def test_set_object_lifecycle_with_cond(self): + key = 'test_set_object_lifecycle_cond' + rand_string(8) + ret, info = self.bucket.copy(bucket_name, 'copyfrom', bucket_name, key) + assert info.status_code == 200 + ret, info = self.bucket.stat(bucket_name, key) + assert info.status_code == 200 + key_hash = ret['hash'] + ret, info = self.bucket.set_object_lifecycle( + bucket=bucket_name, + key=key, + to_line_after_days=10, + to_archive_after_days=20, + to_deep_archive_after_days=30, + delete_after_days=40, + cond={ + 'hash': key_hash + } + ) + assert info.status_code == 200 + class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" From 072b350d5a4163d567f7cf8be4cdda5074bb9ed0 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Fri, 29 Apr 2022 15:40:02 +0800 Subject: [PATCH 122/131] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b85e8db..349d87cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.7.0 (2022-04-29) +* 对象存储，新增 set_object_lifecycle （设置 Object 生命周期） API + ## 7.6.0 (2022-03-28) * 优化了错误处理机制 * 支持 [Qiniu](https://developer.qiniu.com/kodo/1201/access-token) 签名算法 From 1a81a25cd4d33261fd7f1e4484a3f12e4afaedea Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Fri, 29 Apr 2022 15:41:02 +0800 Subject: [PATCH 123/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 5da35d0e..bb1513e6 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.6.0' +__version__ = '7.7.0' from .auth import Auth, QiniuMacAuth From 6034ce5c03aa0e25e80eb1fec962e62e29f85f0d Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 10 May 2022 13:31:47 +0800 Subject: [PATCH 124/131] fix upload v2 with key=None --- qiniu/services/storage/uploader.py | 2 +- test_qiniu.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index 03729790..c596393d 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -337,7 +337,7 @@ def block_url(self, host, size): return '{0}/mkblk/{1}'.format(host, size) def block_url_v2(self, host, bucket_name): - encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '～' + encoded_object_name = urlsafe_base64_encode(self.key) if self.key is not None else '~' return '{0}/buckets/{1}/objects/{2}/uploads'.format(host, bucket_name, encoded_object_name) def file_url(self, host): diff --git a/test_qiniu.py b/test_qiniu.py index 8e1552e0..f0d64eac 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -663,6 +663,20 @@ def test_put_10m_stream_v2(self): assert ret['key'] == key remove_temp_file(localfile) + def test_put_stream_v2_without_key(self): + part_size = 1024 * 1024 * 4 + localfile = create_temp_file(part_size + 1) + key = None + size = os.stat(localfile).st_size + set_default(default_zone=Zone('http://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, + self.mime_type, part_size=part_size, version='v2', bucket_name=bucket_name) + assert ret['key'] == ret['hash'] + remove_temp_file(localfile) + def test_big_file(self): key = 'big' token = self.q.upload_token(bucket_name, key) From 3fbe52afb8e48d545cd106040da493d14d95306f Mon Sep 17 00:00:00 2001 From: LiHS Date: Tue, 10 May 2022 14:17:52 +0800 Subject: [PATCH 125/131] change tests to use https --- test_qiniu.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index f0d64eac..2db63081 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -148,9 +148,9 @@ def test_noKey(self): Auth('', '').token('nokey') def test_token_of_request(self): - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', '') + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', '') assert token == 'abcdefghklmnopq:cFyRVoWrE3IugPIMP5YJFTO-O-Y=' - token = dummy_auth.token_of_request('http://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') + token = dummy_auth.token_of_request('https://www.qiniu.com?go=1', 'test', 'application/x-www-form-urlencoded') assert token == 'abcdefghklmnopq:svWRNcacOE-YMsc70nuIYdaa1e4=' def test_QiniuMacRequestsAuth(self): @@ -554,7 +554,7 @@ def test_putWithoutKey(self): def test_withoutRead_withoutSeek_retry(self): key = 'retry' data = 'hello retry!' - set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) token = self.q.upload_token(bucket_name) ret, info = put_data(token, key, data) print(info) @@ -604,7 +604,7 @@ def test_put_stream(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -616,7 +616,7 @@ def test_put_stream_v2_without_bucket_name(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, @@ -628,7 +628,7 @@ def test_put_2m_stream_v2(self): localfile = create_temp_file(2 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -641,7 +641,7 @@ def test_put_4m_stream_v2(self): localfile = create_temp_file(4 * 1024 * 1024) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -654,7 +654,7 @@ def test_put_10m_stream_v2(self): localfile = create_temp_file(10 * 1024 * 1024 + 1) key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -668,7 +668,7 @@ def test_put_stream_v2_without_key(self): localfile = create_temp_file(part_size + 1) key = None size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key) ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, @@ -682,7 +682,7 @@ def test_big_file(self): token = self.q.upload_token(bucket_name, key) localfile = create_temp_file(4 * 1024 * 1024 + 1) progress_handler = lambda progress, total: progress - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) ret, info = put_file(token, key, localfile, self.params, self.mime_type, progress_handler=progress_handler) print(info) assert ret['key'] == key @@ -691,7 +691,7 @@ def test_big_file(self): def test_retry(self): localfile = __file__ key = 'test_file_r_retry' - qiniu.set_default(default_zone=Zone('http://a', 'http://upload.qiniup.com')) + qiniu.set_default(default_zone=Zone('http://a', 'https://upload.qiniup.com')) token = self.q.upload_token(bucket_name, key) ret, info = put_file(token, key, localfile, self.params, self.mime_type) print(info) @@ -702,7 +702,7 @@ def test_put_stream_with_key_limits(self): localfile = __file__ key = 'test_file_r' size = os.stat(localfile).st_size - set_default(default_zone=Zone('http://upload.qiniup.com')) + set_default(default_zone=Zone('https://upload.qiniup.com')) with open(localfile, 'rb') as input_stream: token = self.q.upload_token(bucket_name, key, policy={'keylimit': ['test_file_d']}) ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, From bfcfc308cd12c24e7c549e40a517b4355ce245e6 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 11 May 2022 10:53:25 +0800 Subject: [PATCH 126/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index bb1513e6..058ba0d1 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.7.0' +__version__ = '7.7.1' from .auth import Auth, QiniuMacAuth From 852a32e2fce16781973ce293bb3f978a130c90b6 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Wed, 11 May 2022 10:55:37 +0800 Subject: [PATCH 127/131] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 349d87cc..676be882 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.7.1 (2022-05-11) +* 对象存储，修复上传不制定 key 部分情况下会上传失败问题。 + ## 7.7.0 (2022-04-29) * 对象存储，新增 set_object_lifecycle （设置 Object 生命周期） API From af1485cc22f493ae70b7d11ec7725d9b7481b9ce Mon Sep 17 00:00:00 2001 From: LiHS Date: Thu, 26 May 2022 15:21:15 +0800 Subject: [PATCH 128/131] upload can add metadata --- qiniu/services/storage/uploader.py | 52 ++++++++++++++--------- test_qiniu.py | 68 ++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/qiniu/services/storage/uploader.py b/qiniu/services/storage/uploader.py index c596393d..f453160b 100644 --- a/qiniu/services/storage/uploader.py +++ b/qiniu/services/storage/uploader.py @@ -12,7 +12,7 @@ def put_data( up_token, key, data, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, - fname=None, hostscache_dir=None): + fname=None, hostscache_dir=None, metadata=None): """上传二进制流到七牛 Args: @@ -23,7 +23,8 @@ def put_data( mime_type: 上传数据的mimeType check_crc: 是否校验crc32 progress_handler: 上传进度 - hostscache_dir： host请求 缓存文件保存位置 + hostscache_dir: host请求 缓存文件保存位置 + metadata: 元数据 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -41,13 +42,14 @@ def put_data( final_data = data crc = crc32(final_data) - return _form_put(up_token, key, final_data, params, mime_type, crc, hostscache_dir, progress_handler, fname) + return _form_put(up_token, key, final_data, params, mime_type, + crc, hostscache_dir, progress_handler, fname, metadata=metadata) def put_file(up_token, key, file_path, params=None, mime_type='application/octet-stream', check_crc=False, progress_handler=None, upload_progress_recorder=None, keep_last_modified=False, hostscache_dir=None, - part_size=None, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None, metadata=None): """上传文件到七牛 Args: @@ -59,10 +61,11 @@ def put_file(up_token, key, file_path, params=None, check_crc: 是否校验crc32 progress_handler: 上传进度 upload_progress_recorder: 记录上传进度，用于断点续传 - hostscache_dir： host请求 缓存文件保存位置 - version 分片上传版本 目前支持v1/v2版本 默认v1 - part_size 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB - bucket_name 分片上传v2字段 空间名称 + hostscache_dir: host请求 缓存文件保存位置 + version: 分片上传版本 目前支持v1/v2版本 默认v1 + part_size: 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + bucket_name: 分片上传v2字段 空间名称 + metadata: 元数据信息 Returns: 一个dict变量，类似 {"hash": "", "key": ""} @@ -78,18 +81,17 @@ def put_file(up_token, key, file_path, params=None, mime_type, progress_handler, upload_progress_recorder=upload_progress_recorder, modify_time=modify_time, keep_last_modified=keep_last_modified, - part_size=part_size, version=version, bucket_name=bucket_name) + part_size=part_size, version=version, bucket_name=bucket_name, metadata=metadata) else: crc = file_crc32(file_path) ret, info = _form_put(up_token, key, input_stream, params, mime_type, crc, hostscache_dir, progress_handler, file_name, - modify_time=modify_time, keep_last_modified=keep_last_modified) + modify_time=modify_time, keep_last_modified=keep_last_modified, metadata=metadata) return ret, info def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, progress_handler=None, file_name=None, - modify_time=None, - keep_last_modified=False): + modify_time=None, keep_last_modified=False, metadata=None): fields = {} if params: for k, v in params.items(): @@ -114,6 +116,11 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, if modify_time and keep_last_modified: fields['x-qn-meta-!Last-Modified'] = rfc_from_timestamp(modify_time) + if metadata: + for k, v in metadata.items(): + if k.startswith('x-qn-meta-'): + fields[k] = str(v) + r, info = http._post_file(url, data=fields, files={'file': (fname, data, mime_type)}) if r is None and info.need_retry(): if info.connect_failed: @@ -135,10 +142,10 @@ def _form_put(up_token, key, data, params, mime_type, crc, hostscache_dir=None, def put_stream(up_token, key, input_stream, file_name, data_size, hostscache_dir=None, params=None, mime_type=None, progress_handler=None, upload_progress_recorder=None, modify_time=None, keep_last_modified=False, - part_size=None, version=None, bucket_name=None): + part_size=None, version=None, bucket_name=None, metadata=None): task = _Resume(up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder, modify_time, keep_last_modified, - part_size, version, bucket_name) + part_size, version, bucket_name, metadata) return task.upload() @@ -167,7 +174,7 @@ class _Resume(object): def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache_dir, params, mime_type, progress_handler, upload_progress_recorder, modify_time, keep_last_modified, part_size=None, - version=None, bucket_name=None): + version=None, bucket_name=None, metadata=None): """初始化断点续上传""" self.up_token = up_token self.key = key @@ -184,6 +191,7 @@ def __init__(self, up_token, key, input_stream, file_name, data_size, hostscache self.version = version or 'v1' self.part_size = part_size or config._BLOCK_SIZE self.bucket_name = bucket_name + self.metadata = metadata def record_upload_progress(self, offset): record_data = { @@ -294,9 +302,9 @@ def upload(self): elif self.version == 'v2': make_file_url = self.block_url_v2(host, self.bucket_name) + '/%s' % self.uploadId return self.make_file_v2(self.blockStatus, make_file_url, self.file_name, - self.mime_type, self.params) + self.mime_type, self.params, self.metadata) - def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None): + def make_file_v2(self, block_status, url, file_name=None, mime_type=None, customVars=None, metadata=None): """completeMultipartUpload""" parts = self.get_parts(block_status) headers = { @@ -306,7 +314,8 @@ def make_file_v2(self, block_status, url, file_name=None, mime_type=None, custom 'parts': parts, 'fname': file_name, 'mimeType': mime_type, - 'customVars': customVars + 'customVars': customVars, + 'metadata': metadata } ret, info = self.post_with_headers(url, json.dumps(data), headers=headers) if ret is not None and ret != {}: @@ -354,12 +363,17 @@ def file_url(self, host): if self.params: for k, v in self.params.items(): url.append('{0}/{1}'.format(k, urlsafe_base64_encode(v))) - pass if self.modify_time and self.keep_last_modified: url.append( "x-qn-meta-!Last-Modified/{0}".format(urlsafe_base64_encode(rfc_from_timestamp(self.modify_time)))) + if self.metadata: + for k, v in self.metadata.items(): + if k.startswith('x-qn-meta-'): + url.append( + "{0}/{1}".format(k, urlsafe_base64_encode(v))) + url = '/'.join(url) return url diff --git a/test_qiniu.py b/test_qiniu.py index 2db63081..d8375c15 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -498,7 +498,12 @@ def test_set_object_lifecycle_with_cond(self): class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" params = {'x:a': 'a'} + metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + } q = Auth(access_key, secret_key) + bucket = BucketManager(q) def test_put(self): key = 'a\\b\\c"hello' @@ -594,11 +599,40 @@ def test_putData_without_fname2(self): print(info) assert ret is not None + def test_put_file_with_metadata(self): + localfile = __file__ + key = 'test_file_with_metadata' + + token = self.q.upload_token(bucket_name, key) + ret, info = put_file(token, key, localfile, metadata=self.metadata) + assert ret['key'] == key + assert ret['hash'] == etag(localfile) + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_data_with_metadata(self): + key = 'put_data_with_metadata' + data = 'hello metadata!' + token = self.q.upload_token(bucket_name, key) + ret, info = put_data(token, key, data, metadata=self.metadata) + assert ret['key'] == key + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + class ResumableUploaderTestCase(unittest.TestCase): mime_type = "text/plain" params = {'x:a': 'a'} + metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + } q = Auth(access_key, secret_key) + bucket = BucketManager(q) def test_put_stream(self): localfile = __file__ @@ -715,6 +749,40 @@ def test_put_stream_with_key_limits(self): self.mime_type) assert info.status_code == 200 + def test_put_stream_with_metadata(self): + localfile = __file__ + key = 'test_put_stream_with_metadata' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('https://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(__file__), size, hostscache_dir, + self.params, self.mime_type, + part_size=None, version=None, bucket_name=None, metadata=self.metadata) + assert ret['key'] == key + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + + def test_put_stream_v2_with_metadata(self): + part_size = 1024 * 1024 * 4 + localfile = create_temp_file(part_size + 1) + key = 'test_put_stream_v2_with_metadata' + size = os.stat(localfile).st_size + set_default(default_zone=Zone('https://upload.qiniup.com')) + with open(localfile, 'rb') as input_stream: + token = self.q.upload_token(bucket_name, key) + ret, info = put_stream(token, key, input_stream, os.path.basename(localfile), size, hostscache_dir, + self.params, self.mime_type, + part_size=part_size, version='v2', bucket_name=bucket_name, metadata=self.metadata) + assert ret['key'] == key + remove_temp_file(localfile) + ret, info = self.bucket.stat(bucket_name, key) + assert 'x-qn-meta' in ret + assert ret['x-qn-meta']['name'] == 'qiniu' + assert ret['x-qn-meta']['age'] == '18' + class DownloadTestCase(unittest.TestCase): q = Auth(access_key, secret_key) From c4cdd53758f0c0c9759e4686bfbe7663cc02577e Mon Sep 17 00:00:00 2001 From: LiHS Date: Wed, 18 May 2022 14:45:20 +0800 Subject: [PATCH 129/131] add X-Qiniu-Date --- .github/workflows/ci-test.yml | 2 +- qiniu/auth.py | 6 ++++-- qiniu/http.py | 23 +++++++++++++++++++++-- qiniu/services/storage/bucket.py | 5 ++++- test_qiniu.py | 31 +++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index f331727d..2614bb3e 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov requests scrutinizer-ocular codecov + pip install flake8 pytest pytest-cov freezegun requests scrutinizer-ocular codecov - name: Run cases env: QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} diff --git a/qiniu/auth.py b/qiniu/auth.py index 100500a9..a1466564 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -49,11 +49,12 @@ class Auth(object): __secret_key: 账号密钥对重的secretKey，详见 https://portal.qiniu.com/user/key """ - def __init__(self, access_key, secret_key): + def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None): """初始化Auth类""" self.__checkKey(access_key, secret_key) self.__access_key = access_key self.__secret_key = b(secret_key) + self.disable_qiniu_timestamp_signature = disable_qiniu_timestamp_signature def get_access_key(self): return self.__access_key @@ -229,11 +230,12 @@ class QiniuMacAuth(object): http://kirk-docs.qiniu.com/apidocs/#TOC_325b437b89e8465e62e958cccc25c63f """ - def __init__(self, access_key, secret_key): + def __init__(self, access_key, secret_key, disable_qiniu_timestamp_signature=None): self.qiniu_header_prefix = "X-Qiniu-" self.__checkKey(access_key, secret_key) self.__access_key = access_key self.__secret_key = b(secret_key) + self.disable_qiniu_timestamp_signature = disable_qiniu_timestamp_signature def __token(self, data): data = b(data) diff --git a/qiniu/http.py b/qiniu/http.py index b12ac8ad..6d239808 100644 --- a/qiniu/http.py +++ b/qiniu/http.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import logging +import os import platform +from datetime import datetime import requests from requests.auth import AuthBase @@ -20,6 +22,19 @@ _headers = {'User-Agent': USER_AGENT} +def __add_auth_headers(headers, auth): + x_qiniu_date = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ') + if auth.disable_qiniu_timestamp_signature is not None: + if not auth.disable_qiniu_timestamp_signature: + headers['X-Qiniu-Date'] = x_qiniu_date + elif os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE'): + if os.getenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE').lower() != 'true': + headers['X-Qiniu-Date'] = x_qiniu_date + else: + headers['X-Qiniu-Date'] = x_qiniu_date + return headers + + def __return_wrapper(resp): if resp.status_code != 200 or resp.headers.get('X-Reqid') is None: return None, ResponseInfo(resp) @@ -155,14 +170,18 @@ def _post_with_qiniu_mac(url, data, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - return _post(url, data, None, qn_auth) + headers = __add_auth_headers({}, auth) + + return _post(url, data, None, qn_auth, headers=headers) def _get_with_qiniu_mac(url, params, auth): qn_auth = qiniu.auth.QiniuMacRequestsAuth( auth ) if auth is not None else None - return _get(url, params, qn_auth) + headers = __add_auth_headers({}, auth) + + return _get(url, params, qn_auth, headers=headers) def _get_with_qiniu_mac_and_headers(url, params, auth, headers): diff --git a/qiniu/services/storage/bucket.py b/qiniu/services/storage/bucket.py index 46ec2045..507e5977 100644 --- a/qiniu/services/storage/bucket.py +++ b/qiniu/services/storage/bucket.py @@ -17,7 +17,10 @@ class BucketManager(object): def __init__(self, auth, zone=None): self.auth = auth - self.mac_auth = QiniuMacAuth(auth.get_access_key(), auth.get_secret_key()) + self.mac_auth = QiniuMacAuth( + auth.get_access_key(), + auth.get_secret_key(), + auth.disable_qiniu_timestamp_signature) if (zone is None): self.zone = config.get_default('default_zone') else: diff --git a/test_qiniu.py b/test_qiniu.py index d8375c15..513723be 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -10,6 +10,7 @@ import unittest import pytest +from freezegun import freeze_time from qiniu import Auth, set_default, etag, PersistentFop, build_op, op_save, Zone, QiniuMacAuth from qiniu import put_data, put_file, put_stream @@ -494,6 +495,36 @@ def test_set_object_lifecycle_with_cond(self): ) assert info.status_code == 200 + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date(self): + ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') + assert ret is None + assert info.status_code == 403 + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_with_disable_date_sign(self): + q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=True) + bucket = BucketManager(q) + ret, info = bucket.stat(bucket_name, 'python-sdk.html') + assert 'hash' in ret + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_env(self): + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' + ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + assert 'hash' in ret + + @freeze_time("1970-01-01") + def test_invalid_x_qiniu_date_env_be_ignored(self): + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' + q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False) + bucket = BucketManager(q) + ret, info = bucket.stat(bucket_name, 'python-sdk.html') + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + assert ret is None + assert info.status_code == 403 + class UploaderTestCase(unittest.TestCase): mime_type = "text/plain" From 901016a684a89edd0e10fb2032faa0d7f485bb1d Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 9 Jun 2022 10:44:11 +0800 Subject: [PATCH 130/131] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 676be882..d094496e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog + +## 7.8.0（2022-06-08） +* 对象存储，管理类 API 发送请求时增加 [X-Qiniu-Date](https://developer.qiniu.com/kodo/3924/common-request-headers) （生成请求的时间） header + ## 7.7.1 (2022-05-11) * 对象存储，修复上传不制定 key 部分情况下会上传失败问题。 From 92f44e80fbfdca7e14984c3faaf1178902f2619f Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Thu, 9 Jun 2022 10:44:50 +0800 Subject: [PATCH 131/131] Update __init__.py --- qiniu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 058ba0d1..d785510b 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.7.1' +__version__ = '7.8.0' from .auth import Auth, QiniuMacAuth