From b4c7d4131e3a06359e0a97c5d0df4da8f93646f9 Mon Sep 17 00:00:00 2001 From: lihsai0 Date: Wed, 9 Oct 2024 15:46:20 +0800 Subject: [PATCH 1/5] fix: windows compatibility - fix file write error - fix datetime to timestamp error - fix read file line by line then json.parse error - test, add windows test actions - build, remove useless test packages - test, fix windows platform not work with unicode characters in codecov config --- .github/workflows/ci-test.yml | 100 ++++++++++++++++++++++- codecov.yml | 16 ++-- qiniu/http/regions_provider.py | 11 ++- qiniu/region.py | 4 +- qiniu/utils.py | 30 ++++++- setup.py | 4 - test_qiniu.py | 123 ---------------------------- tests/cases/test_utils.py | 145 +++++++++++++++++++++++++++++++++ 8 files changed, 287 insertions(+), 146 deletions(-) create mode 100644 tests/cases/test_utils.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index c705019b..db8752d4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -60,14 +60,106 @@ jobs: QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} QINIU_TEST_ENV: "travis" MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" - PYTHONPATH: "$PYTHONPATH:." run: | flake8 --show-source --max-line-length=160 ./qiniu - coverage run -m pytest ./test_qiniu.py ./tests/cases - ocular --data-file .coverage - codecov + python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml + - name: Post Setup mock server + if: ${{ always() }} + shell: bash + run: | + set +e cat mock-server.pid | xargs kill + rm mock-server.pid - name: Print mock server log if: ${{ failure() }} run: | cat py-mock-server.log + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + test-win: + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python_version: ['2.7', '3.5', '3.9'] + runs-on: windows-2019 + # make sure only one test running, + # remove this when cases could run in parallel. + needs: test + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Setup miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + channels: conda-forge + python-version: ${{ matrix.python_version }} + activate-environment: qiniu-sdk + auto-activate-base: false + - name: Setup pip + env: + PYTHON_VERSION: ${{ matrix.python_version }} + PIP_BOOTSTRAP_SCRIPT_PREFIX: https://bootstrap.pypa.io/pip + run: | + # reinstall pip by some python(<3.7) not compatible + $pyversion = [Version]"$ENV:PYTHON_VERSION" + if ($pyversion -lt [Version]"3.7") { + Invoke-WebRequest "$ENV:PIP_BOOTSTRAP_SCRIPT_PREFIX/$($pyversion.Major).$($pyversion.Minor)/get-pip.py" -OutFile "$ENV:TEMP\get-pip.py" + python $ENV:TEMP\get-pip.py --user + Remove-Item -Path "$ENV:TEMP\get-pip.py" + } + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -I -e ".[dev]" + - name: Run cases + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_NO_ACC_BUCKET: ${{ secrets.QINIU_TEST_NO_ACC_BUCKET }} + QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} + QINIU_UPLOAD_CALLBACK_URL: ${{secrets.QINIU_UPLOAD_CALLBACK_URL}} + QINIU_TEST_ENV: "github" + MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000" + PYTHONPATH: "$PYTHONPATH:." + run: | + Write-Host "======== Setup Mock Server =========" + conda create -y -n mock-server python=3.10 + conda activate mock-server + python --version + $processOptions = @{ + FilePath="python" + ArgumentList="tests/mock_server/main.py", "--port", "9000" + PassThru=$true + RedirectStandardOutput="py-mock-server.log" + } + $mocksrvp = Start-Process @processOptions + $mocksrvp.Id | Out-File -FilePath "mock-server.pid" + conda deactivate + Write-Host "======== Running Test =========" + python --version + python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml + - name: Post Setup mock server + if: ${{ always() }} + run: | + Try { + $mocksrvpid = Get-Content -Path "mock-server.pid" + Stop-Process -Id $mocksrvpid + Remove-Item -Path "mock-server.pid" + } Catch { + Write-Host -Object $_ + } + - name: Print mock server log + if: ${{ failure() }} + run: | + Get-Content -Path "py-mock-server.log" + - name: Upload results to Codecov + uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/codecov.yml b/codecov.yml index 3f36c50a..0aab28d3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,14 +1,14 @@ codecov: ci: - - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要 - require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。 + - prow.qiniu.io # prow need this. seems useless + require_ci_to_pass: no # `no` means the bot will comment on the PR even before all ci passed -github_checks: #关闭github checks +github_checks: # close github checks annotations: false comment: layout: "reach, diff, flags, files" - behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。 + behavior: new # `new` means the bot will comment a new message instead of edit the old one require_changes: false # if true: only post the comment if coverage changes require_base: no # [yes :: must have a base report to post] require_head: yes # [yes :: must have a head report to post] @@ -16,13 +16,13 @@ comment: - "master" coverage: - status: # 评判 pr 通过的标准 + status: # check coverage status to pass or fail patch: off - project: # project 统计所有代码x + project: # project analyze all code in the project default: # basic - target: 73.5% # 总体通过标准 - threshold: 3% # 允许单次下降的幅度 + target: 73.5% # the minimum coverage ratio that the commit must meet + threshold: 3% # allow the coverage to drop base: auto if_not_found: success if_ci_failed: error diff --git a/qiniu/http/regions_provider.py b/qiniu/http/regions_provider.py index 8b52822c..13d1800a 100644 --- a/qiniu/http/regions_provider.py +++ b/qiniu/http/regions_provider.py @@ -5,9 +5,10 @@ import logging import tempfile import os +import shutil from qiniu.compat import json, b as to_bytes -from qiniu.utils import io_md5 +from qiniu.utils import io_md5, dt2ts from .endpoint import Endpoint from .region import Region, ServiceName @@ -264,7 +265,7 @@ def _persist_region(region): }, ttl=region.ttl, # use datetime.datetime.timestamp() when min version of python >= 3 - createTime=int(float(region.create_time.strftime('%s.%f')) * 1000) + createTime=dt2ts(region.create_time) )._asdict() @@ -338,8 +339,10 @@ def _walk_persist_cache_file(persist_path, ignore_parse_error=False): with open(persist_path, 'r') as f: for line in f: + if not line.strip(): + continue try: - cache_key, regions = _parse_persisted_regions(line) + cache_key, regions = _parse_persisted_regions(line.strip()) yield cache_key, regions except Exception as err: if not ignore_parse_error: @@ -655,7 +658,7 @@ def __shrink_cache(self): ) # rename file - os.rename(shrink_file_path, self._cache_scope.persist_path) + shutil.move(shrink_file_path, self._cache_scope.persist_path) except FileAlreadyLocked: pass finally: diff --git a/qiniu/region.py b/qiniu/region.py index 09ac791d..a59d488e 100644 --- a/qiniu/region.py +++ b/qiniu/region.py @@ -6,7 +6,7 @@ from .compat import json, s as str_from_bytes -from .utils import urlsafe_base64_decode +from .utils import urlsafe_base64_decode, dt2ts from .config import UC_HOST, is_customized_default, get_default from .http.endpoint import Endpoint as _HTTPEndpoint from .http.regions_provider import Region as _HTTPRegion, ServiceName, get_default_regions_provider @@ -190,7 +190,7 @@ def get_bucket_hosts(self, ak, bucket, home_dir=None, force=False): ttl = region.ttl if region.ttl > 0 else 24 * 3600 # 1 day # use datetime.datetime.timestamp() when min version of python >= 3 - create_time = int(float(region.create_time.strftime('%s.%f')) * 1000) + create_time = dt2ts(region.create_time) bucket_hosts['deadline'] = create_time + ttl return bucket_hosts diff --git a/qiniu/utils.py b/qiniu/utils.py index f8517e35..197b8813 100644 --- a/qiniu/utils.py +++ b/qiniu/utils.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- from hashlib import sha1, new as hashlib_new from base64 import urlsafe_b64encode, urlsafe_b64decode -from datetime import datetime +from datetime import datetime, tzinfo, timedelta + from .compat import b, s try: @@ -236,3 +237,30 @@ def canonical_mime_header_key(field_name): result += ch upper = ch == "-" return result + + +class _UTC_TZINFO(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=0) + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return timedelta(0) + + +def dt2ts(dt): + """ + converte datetime to timestamp + + Parameters + ---------- + dt: datetime.datetime + """ + if not dt.tzinfo: + st = (dt - datetime(1970, 1, 1)).total_seconds() + else: + st = (dt - datetime(1970, 1, 1, tzinfo=_UTC_TZINFO())).total_seconds() + + return int(st) diff --git a/setup.py b/setup.py index cf97eae2..fa920d45 100644 --- a/setup.py +++ b/setup.py @@ -42,10 +42,8 @@ def find_version(*file_paths): 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', @@ -66,8 +64,6 @@ def find_version(*file_paths): 'pytest', 'pytest-cov', 'freezegun', - 'scrutinizer-ocular', - 'codecov' ] }, diff --git a/test_qiniu.py b/test_qiniu.py index 2b71aa22..16f59844 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -65,129 +65,6 @@ def remove_temp_file(file): except OSError: pass - -class UtilsTest(unittest.TestCase): - def test_urlsafe(self): - a = 'hello\x96' - u = urlsafe_base64_encode(a) - assert b(a) == urlsafe_base64_decode(u) - - def test_canonical_mime_header_key(self): - field_names = [ - ":status", - ":x-test-1", - ":x-Test-2", - "content-type", - "CONTENT-LENGTH", - "oRiGin", - "ReFer", - "Last-Modified", - "acCePt-ChArsEt", - "x-test-3", - "cache-control", - ] - expect_canonical_field_names = [ - ":status", - ":x-test-1", - ":x-Test-2", - "Content-Type", - "Content-Length", - "Origin", - "Refer", - "Last-Modified", - "Accept-Charset", - "X-Test-3", - "Cache-Control", - ] - assert len(field_names) == len(expect_canonical_field_names) - for i in range(len(field_names)): - assert canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] - - def test_entry(self): - case_list = [ - { - 'msg': 'normal', - 'bucket': 'qiniuphotos', - 'key': 'gogopher.jpg', - 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' - }, - { - 'msg': 'key empty', - 'bucket': 'qiniuphotos', - 'key': '', - 'expect': 'cWluaXVwaG90b3M6' - }, - { - 'msg': 'key undefined', - 'bucket': 'qiniuphotos', - 'key': None, - 'expect': 'cWluaXVwaG90b3M=' - }, - { - 'msg': 'key need replace plus symbol', - 'bucket': 'qiniuphotos', - 'key': '012ts>a', - 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' - }, - { - 'msg': 'key need replace slash symbol', - 'bucket': 'qiniuphotos', - 'key': '012ts?a', - 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' - } - ] - for c in case_list: - assert c.get('expect') == entry(c.get('bucket'), c.get('key')), c.get('msg') - - def test_decode_entry(self): - case_list = [ - { - 'msg': 'normal', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': 'gogopher.jpg' - }, - 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' - }, - { - 'msg': 'key empty', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '' - }, - 'entry': 'cWluaXVwaG90b3M6' - }, - { - 'msg': 'key undefined', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': None - }, - 'entry': 'cWluaXVwaG90b3M=' - }, - { - 'msg': 'key need replace plus symbol', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '012ts>a' - }, - 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' - }, - { - 'msg': 'key need replace slash symbol', - 'expect': { - 'bucket': 'qiniuphotos', - 'key': '012ts?a' - }, - 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' - } - ] - for c in case_list: - bucket, key = decode_entry(c.get('entry')) - assert bucket == c.get('expect').get('bucket'), c.get('msg') - assert key == c.get('expect').get('key'), c.get('msg') - - class BucketTestCase(unittest.TestCase): q = Auth(access_key, secret_key) bucket = BucketManager(q) diff --git a/tests/cases/test_utils.py b/tests/cases/test_utils.py new file mode 100644 index 00000000..11d9db77 --- /dev/null +++ b/tests/cases/test_utils.py @@ -0,0 +1,145 @@ +from datetime import datetime, timedelta, tzinfo + +from qiniu import utils, compat + + +class _CN_TZINFO(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=8) + + def tzname(self, dt): + return "CST" + + def dst(self, dt): + return timedelta(0) + + +class TestUtils: + def test_urlsafe(self): + a = 'hello\x96' + u = utils.urlsafe_base64_encode(a) + assert compat.b(a) == utils.urlsafe_base64_decode(u) + + def test_canonical_mime_header_key(self): + field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "content-type", + "CONTENT-LENGTH", + "oRiGin", + "ReFer", + "Last-Modified", + "acCePt-ChArsEt", + "x-test-3", + "cache-control", + ] + expect_canonical_field_names = [ + ":status", + ":x-test-1", + ":x-Test-2", + "Content-Type", + "Content-Length", + "Origin", + "Refer", + "Last-Modified", + "Accept-Charset", + "X-Test-3", + "Cache-Control", + ] + assert len(field_names) == len(expect_canonical_field_names) + for i in range(len(field_names)): + assert utils.canonical_mime_header_key(field_names[i]) == expect_canonical_field_names[i] + + def test_entry(self): + case_list = [ + { + 'msg': 'normal', + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg', + 'expect': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'bucket': 'qiniuphotos', + 'key': '', + 'expect': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'bucket': 'qiniuphotos', + 'key': None, + 'expect': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts>a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'bucket': 'qiniuphotos', + 'key': '012ts?a', + 'expect': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + assert c.get('expect') == utils.entry(c.get('bucket'), c.get('key')), c.get('msg') + + def test_decode_entry(self): + case_list = [ + { + 'msg': 'normal', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': 'gogopher.jpg' + }, + 'entry': 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + 'msg': 'key empty', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '' + }, + 'entry': 'cWluaXVwaG90b3M6' + }, + { + 'msg': 'key undefined', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': None + }, + 'entry': 'cWluaXVwaG90b3M=' + }, + { + 'msg': 'key need replace plus symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts>a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + 'msg': 'key need replace slash symbol', + 'expect': { + 'bucket': 'qiniuphotos', + 'key': '012ts?a' + }, + 'entry': 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ] + for c in case_list: + bucket, key = utils.decode_entry(c.get('entry')) + assert bucket == c.get('expect', {}).get('bucket'), c.get('msg') + assert key == c.get('expect', {}).get('key'), c.get('msg') + + def test_dt2ts(self): + dt = datetime(year=2011, month=8, day=3, tzinfo=_CN_TZINFO()) + expect = 1312300800 + assert utils.dt2ts(dt) == expect + + base_dt = datetime(year=2011, month=8, day=3) + now_dt = datetime.now() + assert int((now_dt - base_dt).total_seconds()) == utils.dt2ts(now_dt) - utils.dt2ts(base_dt) From da2a7bd5ae9745ae2114f30595057bc8a57017e9 Mon Sep 17 00:00:00 2001 From: lihsai0 Date: Sat, 12 Oct 2024 11:42:18 +0800 Subject: [PATCH 2/5] test: fix test cases compatibility on Windows --- .github/workflows/ci-test.yml | 5 +++-- tests/cases/test_http/test_region.py | 2 +- tests/cases/test_http/test_regions_provider.py | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index db8752d4..52195876 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -135,13 +135,14 @@ jobs: python --version $processOptions = @{ FilePath="python" - ArgumentList="tests/mock_server/main.py", "--port", "9000" + ArgumentList="tests\mock_server\main.py", "--port", "9000" PassThru=$true RedirectStandardOutput="py-mock-server.log" } $mocksrvp = Start-Process @processOptions $mocksrvp.Id | Out-File -FilePath "mock-server.pid" conda deactivate + Sleep 3 Write-Host "======== Running Test =========" python --version python -m pytest ./test_qiniu.py tests --cov qiniu --cov-report=xml @@ -158,7 +159,7 @@ jobs: - name: Print mock server log if: ${{ failure() }} run: | - Get-Content -Path "py-mock-server.log" + Get-Content -Path "py-mock-server.log" | Write-Host - name: Upload results to Codecov uses: codecov/codecov-action@v4 with: diff --git a/tests/cases/test_http/test_region.py b/tests/cases/test_http/test_region.py index a66b16c9..976d2619 100644 --- a/tests/cases/test_http/test_region.py +++ b/tests/cases/test_http/test_region.py @@ -36,7 +36,7 @@ def test_custom_options(self): k in region.services for k in chain(ServiceName, ['custom-service']) ) - assert datetime.now() - region.create_time > timedelta(days=1) + assert datetime.now() - region.create_time >= timedelta(days=1) assert region.ttl == 3600 assert not region.is_live diff --git a/tests/cases/test_http/test_regions_provider.py b/tests/cases/test_http/test_regions_provider.py index 163f19d2..7289f5ca 100644 --- a/tests/cases/test_http/test_regions_provider.py +++ b/tests/cases/test_http/test_regions_provider.py @@ -186,8 +186,10 @@ def test_getter_with_base_regions_provider(self, cached_regions_provider): assert list(cached_regions_provider) == regions line_num = 0 with open(cached_regions_provider.persist_path, 'r') as f: - for _ in f: - line_num += 1 + for l in f: + # ignore empty line + if l.strip(): + line_num += 1 assert line_num == 1 @pytest.mark.parametrize( From 2c9c165d19f5bca6e7bda72b50f48f0c6d057d1a Mon Sep 17 00:00:00 2001 From: lihsai0 Date: Sat, 12 Oct 2024 14:09:39 +0800 Subject: [PATCH 3/5] test: fix unsetenv not exists in earlier python on windows --- test_qiniu.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test_qiniu.py b/test_qiniu.py index 16f59844..c8dce456 100644 --- a/test_qiniu.py +++ b/test_qiniu.py @@ -285,7 +285,11 @@ def test_invalid_x_qiniu_date_with_disable_date_sign(self): def test_invalid_x_qiniu_date_env(self): os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = 'True' ret, info = self.bucket.stat(bucket_name, 'python-sdk.html') - os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + if hasattr(os, 'unsetenv'): + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + else: + # fix unsetenv not exists in earlier python on windows + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = '' assert 'hash' in ret @freeze_time("1970-01-01") @@ -294,7 +298,11 @@ def test_invalid_x_qiniu_date_env_be_ignored(self): q = Auth(access_key, secret_key, disable_qiniu_timestamp_signature=False) bucket = BucketManager(q) ret, info = bucket.stat(bucket_name, 'python-sdk.html') - os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + if hasattr(os, 'unsetenv'): + os.unsetenv('DISABLE_QINIU_TIMESTAMP_SIGNATURE') + else: + # fix unsetenv not exists in earlier python on windows + os.environ['DISABLE_QINIU_TIMESTAMP_SIGNATURE'] = '' assert ret is None assert info.status_code == 403 From a26c833514345d571b0bddf4efb6c91675441bac Mon Sep 17 00:00:00 2001 From: lihsai0 Date: Mon, 14 Oct 2024 20:07:02 +0800 Subject: [PATCH 4/5] feat: add workflow template id for pfop --- .gitignore | 3 +- qiniu/auth.py | 3 +- qiniu/services/processing/pfop.py | 19 ++++- .../test_processing/test_pfop.py | 76 +++++++++++++++---- .../test_storage/test_upload_pfop.py | 66 +++++++++++----- 5 files changed, 128 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 93665221..261e665c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ pip-log.txt .coverage .tox nosetests.xml +coverage.xml # Translations *.mo @@ -45,4 +46,4 @@ nosetests.xml .project .pydevproject /.idea -/.venv \ No newline at end of file +/.venv* diff --git a/qiniu/auth.py b/qiniu/auth.py index d3e0a055..1647199e 100644 --- a/qiniu/auth.py +++ b/qiniu/auth.py @@ -34,10 +34,11 @@ str('fsizeMin'), # 上传文件最少字节数 str('keylimit'), # 设置允许上传的key列表，字符串数组类型，数组长度不可超过20个，如果设置了这个字段，上传时必须提供key - str('persistentOps'), # 持久化处理操作 + str('persistentOps'), # 持久化处理操作，与 persistentWorkflowTemplateID 二选一 str('persistentNotifyUrl'), # 持久化处理结果通知URL str('persistentPipeline'), # 持久化处理独享队列 str('persistentType'), # 为 `1` 时，开启闲时任务，必须是 int 类型 + str('persistentWorkflowTemplateID'), # 工作流模板 ID，与 persistentOps 二选一 str('deleteAfterDays'), # 文件多少天后自动删除 str('fileType'), # 文件的存储类型，0为标准存储，1为低频存储，2为归档存储，3为深度归档存储，4为归档直读存储 diff --git a/qiniu/services/processing/pfop.py b/qiniu/services/processing/pfop.py index 346e6277..4b2641e2 100644 --- a/qiniu/services/processing/pfop.py +++ b/qiniu/services/processing/pfop.py @@ -24,7 +24,7 @@ def __init__(self, auth, bucket, pipeline=None, notify_url=None): self.pipeline = pipeline self.notify_url = notify_url - def execute(self, key, fops, force=None, persistent_type=None): + def execute(self, key, fops=None, force=None, persistent_type=None, workflow_template_id=None): """ 执行持久化处理 @@ -32,28 +32,39 @@ def execute(self, key, fops, force=None, persistent_type=None): ---------- key: str 待处理的源文件 - fops: list[str] + fops: list[str], optional 处理详细操作，规格详见 https://developer.qiniu.com/dora/manual/1291/persistent-data-processing-pfop + 与 template_id 二选一 force: int or str, optional 强制执行持久化处理开关 persistent_type: int or str, optional 持久化处理类型，为 '1' 时开启闲时任务 + template_id: str, optional + 与 fops 二选一 Returns ------- ret: dict 持久化处理的 persistentId，类似 {"persistentId": 5476bedf7823de4068253bae}; resp: ResponseInfo """ - ops = ';'.join(fops) - data = {'bucket': self.bucket, 'key': key, 'fops': ops} + if not fops and not workflow_template_id: + raise ValueError('Must provide one of fops or template_id') + data = { + 'bucket': self.bucket, + 'key': key, + } if self.pipeline: data['pipeline'] = self.pipeline if self.notify_url: data['notifyURL'] = self.notify_url + if fops: + data['fops'] = ';'.join(fops) if force == 1 or force == '1': data['force'] = str(force) if persistent_type and type(int(persistent_type)) is int: data['type'] = str(persistent_type) + if workflow_template_id: + data['workflowTemplateID'] = workflow_template_id url = '{0}/pfop'.format(config.get_default('default_api_host')) return http._post_with_auth(url, data, self.auth) diff --git a/tests/cases/test_services/test_processing/test_pfop.py b/tests/cases/test_services/test_processing/test_pfop.py index 003be43f..ebaf18f4 100644 --- a/tests/cases/test_services/test_processing/test_pfop.py +++ b/tests/cases/test_services/test_processing/test_pfop.py @@ -1,6 +1,5 @@ import pytest - from qiniu import PersistentFop, op_save @@ -16,6 +15,7 @@ def test_pfop_execute(self, qn_auth): ] ret, resp = pfop.execute('sintel_trailer.mp4', ops, 1) assert resp.status_code == 200, resp + assert ret is not None, resp assert ret['persistentId'] is not None, resp global persistent_id persistent_id = ret['persistentId'] @@ -27,23 +27,71 @@ def test_pfop_get_status(self, qn_auth): assert resp.status_code == 200, resp assert ret is not None, resp - def test_pfop_idle_time_task(self, set_conf_default, qn_auth): - persistence_key = 'python-sdk-pfop-test/test-pfop-by-api' + @pytest.mark.parametrize( + 'persistent_options', + ( + # included by above test_pfop_execute + # { + # 'persistent_type': None, + # }, + { + 'persistent_type': 0, + }, + { + 'persistent_type': 1, + }, + { + 'workflow_template_id': 'test-workflow', + }, + ) + ) + def test_pfop_idle_time_task( + self, + set_conf_default, + qn_auth, + bucket_name, + persistent_options, + ): + persistent_type = persistent_options.get('persistent_type') + workflow_template_id = persistent_options.get('workflow_template_id', None) + + execute_opts = {} + if workflow_template_id: + execute_opts['workflow_template_id'] = workflow_template_id + else: + persistent_key = '_'.join([ + 'test-pfop/test-pfop-by-api', + 'type', + str(persistent_type) + ]) + execute_opts['fops'] = [ + op_save( + op='avinfo', + bucket=bucket_name, + key=persistent_key + ) + ] + + if persistent_type is not None: + execute_opts['persistent_type'] = persistent_type + + pfop = PersistentFop(qn_auth, bucket_name) + key = 'qiniu.png' + ret, resp = pfop.execute( + key, + **execute_opts + ) - key = 'sintel_trailer.mp4' - pfop = PersistentFop(qn_auth, 'testres') - ops = [ - op_save( - op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', - bucket='pythonsdk', - key=persistence_key - ) - ] - ret, resp = pfop.execute(key, ops, force=1, persistent_type=1) assert resp.status_code == 200, resp + assert ret is not None assert 'persistentId' in ret, resp ret, resp = pfop.get_status(ret['persistentId']) assert resp.status_code == 200, resp - assert ret['type'] == 1, resp + assert ret is not None assert ret['creationDate'] is not None, resp + + if persistent_id == 1: + assert ret['type'] == 1, resp + elif workflow_template_id: + assert workflow_template_id in ret['taskFrom'], resp diff --git a/tests/cases/test_services/test_storage/test_upload_pfop.py b/tests/cases/test_services/test_storage/test_upload_pfop.py index 78818ba4..3effa9c7 100644 --- a/tests/cases/test_services/test_storage/test_upload_pfop.py +++ b/tests/cases/test_services/test_storage/test_upload_pfop.py @@ -12,36 +12,60 @@ # or this test will continue to occupy bucket space. class TestPersistentFopByUpload: @pytest.mark.parametrize('temp_file', [10 * MB], indirect=True) - @pytest.mark.parametrize('persistent_type', [None, 0, 1]) + @pytest.mark.parametrize( + 'persistent_options', + ( + { + 'persistent_type': None, + }, + { + 'persistent_type': 0, + }, + { + 'persistent_type': 1, + }, + { + 'persistent_workflow_template_id': 'test-workflow', + }, + ) + ) def test_pfop_with_upload( self, set_conf_default, qn_auth, bucket_name, temp_file, - persistent_type + persistent_options, ): - key = 'test-pfop-upload-file' - persistent_key = '_'.join([ - 'test-pfop-by-upload', - 'type', - str(persistent_type) - ]) - persistent_ops = ';'.join([ - qiniu.op_save( - op='avthumb/m3u8/segtime/10/vcodec/libx264/s/320x240', - bucket=bucket_name, - key=persistent_key - ) - ]) + key = 'test-pfop/upload-file' + persistent_type = persistent_options.get('persistent_type') + persistent_workflow_template_id = persistent_options.get('persistent_workflow_template_id') + + upload_policy = {} - upload_policy = { - 'persistentOps': persistent_ops - } + # set pfops or tmplate id + if persistent_workflow_template_id: + upload_policy['persistentWorkflowTemplateID'] = persistent_workflow_template_id + else: + persistent_key = '_'.join([ + 'test-pfop/test-pfop-by-upload', + 'type', + str(persistent_type) + ]) + persistent_ops = ';'.join([ + qiniu.op_save( + op='avinfo', + bucket=bucket_name, + key=persistent_key + ) + ]) + upload_policy['persistentOps'] = persistent_ops + # set persistent type if persistent_type is not None: upload_policy['persistentType'] = persistent_type + # upload token = qn_auth.upload_token( bucket_name, key, @@ -61,6 +85,10 @@ def test_pfop_with_upload( pfop = qiniu.PersistentFop(qn_auth, bucket_name) ret, resp = pfop.get_status(ret['persistentId']) assert resp.status_code == 200, resp + assert ret is not None, resp + assert ret['creationDate'] is not None, resp + if persistent_type == 1: assert ret['type'] == 1, resp - assert ret['creationDate'] is not None, resp + elif persistent_workflow_template_id: + assert persistent_workflow_template_id in ret['taskFrom'], resp From 945075b190dfd0a5da03d40cb0345d721b5825c7 Mon Sep 17 00:00:00 2001 From: lihsai0 Date: Wed, 16 Oct 2024 09:39:48 +0800 Subject: [PATCH 5/5] update version to 7.15.0 and changelog --- CHANGELOG.md | 8 +++++--- qiniu/__init__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bf26f46..5722b56b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 7.15.0 +* 对象存储，持久化处理支持工作流模版 +* 对象存储，修复 Windows 平台兼容性问题 + ## 7.14.0 * 对象存储，空间管理、上传文件新增备用域名重试逻辑 * 对象存储，调整查询区域主备域名 @@ -44,7 +48,7 @@ ## 7.9.0（2022-07-20） * 对象存储，支持使用时不配置区域信息，SDK 自动获取； * 对象存储，新增 list_domains API 用于查询空间绑定的域名 -* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API +* 对象存储，上传 API 新增支持设置自定义元数据，详情见 put_data, put_file, put_stream API * 解决部分已知问题 ## 7.8.0（2022-06-08） @@ -237,5 +241,3 @@ * 代码覆盖度报告 * policy改为dict, 便于灵活增加，并加入过期字段检查 * 文件列表支持目录形式 - - diff --git a/qiniu/__init__.py b/qiniu/__init__.py index 55acfb25..1ae68c00 100644 --- a/qiniu/__init__.py +++ b/qiniu/__init__.py @@ -9,7 +9,7 @@ # flake8: noqa -__version__ = '7.14.0' +__version__ = '7.15.0' from .auth import Auth, QiniuMacAuth