From d3b9c3b9f824e3e175784aa34d8f94c0eb418f94 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 13 Feb 2024 22:07:07 +0000 Subject: [PATCH 1/7] ci: add support for pre-release sessions --- .github/workflows/tests.yml | 35 +++++++++++++++-- noxfile.py | 75 ++++++++++++++++++++++++++++-------- setup.py | 2 +- testing/constraints-3.13.txt | 0 4 files changed, 92 insertions(+), 20 deletions(-) create mode 100644 testing/constraints-3.13.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4fad78dc..648ec8fd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -42,19 +42,22 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] - variant: ['', 'cpp', 'upb'] + python: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] + variant: ['cpp', 'python', 'upb'] exclude: - variant: "cpp" python: 3.11 - variant: "cpp" python: 3.12 + - variant: "cpp" + python: 3.13 steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Install nox run: | pip install nox @@ -68,12 +71,38 @@ jobs: env: COVERAGE_FILE: .coverage-${{ matrix.variant }}-${{ env.PYTHON_VERSION_TRIMMED }} run: | - nox -s unit${{ matrix.variant }}-${{ env.PYTHON_VERSION_TRIMMED }} + nox -s "unit-${{ env.PYTHON_VERSION_TRIMMED }}(implementation='${{ matrix.variant }}')" - name: Upload coverage results uses: actions/upload-artifact@v4 with: name: coverage-artifact-${{ matrix.variant }}-${{ env.PYTHON_VERSION_TRIMMED }} path: .coverage-${{ matrix.variant }}-${{ env.PYTHON_VERSION_TRIMMED }} + prerelease: + runs-on: ubuntu-20.04 + strategy: + matrix: + python: ['3.12'] + variant: ['python', 'upb'] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + allow-prereleases: true + - name: Install nox + run: | + pip install nox + - name: Run unit tests + env: + COVERAGE_FILE: .coverage-prerelease-${{ matrix.variant }} + run: | + nox -s "prerelease_deps(implementation='${{ matrix.variant }}')" + - name: Upload coverage results + uses: actions/upload-artifact@v4 + with: + name: coverage-artifact-prerelease-${{ matrix.variant }} + path: .coverage-prerelease-${{ matrix.variant }} cover: runs-on: ubuntu-latest needs: diff --git a/noxfile.py b/noxfile.py index d4da4b9b..f398c5f8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -13,10 +13,9 @@ # limitations under the License. from __future__ import absolute_import -import os -import pathlib import nox +import pathlib CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() @@ -30,6 +29,7 @@ "3.10", "3.11", "3.12", + "3.13", ] # Error if a python version is missing @@ -37,26 +37,27 @@ @nox.session(python=PYTHON_VERSIONS) -def unit(session, proto="python"): +@nox.parametrize("implementation", ["cpp", "upb", "python"]) +def unit(session, implementation): """Run the unit test suite.""" constraints_path = str( CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" ) - session.env["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = proto + session.env["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = implementation session.install("coverage", "pytest", "pytest-cov", "pytz") session.install("-e", ".[testing]", "-c", constraints_path) - if proto == "cpp": # 4.20 does not have cpp. + if implementation == "cpp": # 4.20 does not have cpp. session.install("protobuf==3.19.0") # TODO(https://github.com/googleapis/proto-plus-python/issues/403): re-enable `-W=error` # The warnings-as-errors flag `-W=error` was removed in # https://github.com/googleapis/proto-plus-python/pull/400. # It should be re-added once issue - # https://github.com/protocolbuffers/protobuf/issues/12186 is fixed. + # https://github.com/protocolbuffers/protobuf/issues/15077 is fixed. session.run( - "py.test", + "pytest", "--quiet", *( session.posargs # Coverage info when running individual tests is annoying. @@ -71,17 +72,59 @@ def unit(session, proto="python"): ) -# Check if protobuf has released wheels for new python versions -# https://pypi.org/project/protobuf/#files -# This list will generally be shorter than 'unit' -@nox.session(python=["3.6", "3.7", "3.8", "3.9", "3.10"]) -def unitcpp(session): - return unit(session, proto="cpp") +# Only test upb and python implementation backends. +# As of protobuf 4.x, the "ccp" implementation is not available in the PyPI pacakge as per +# https://github.com/protocolbuffers/protobuf/tree/main/python#implementation-backends +@nox.session(python=PYTHON_VERSIONS[-2]) +@nox.parametrize("implementation", ["python", "upb"]) +def prerelease_deps(session, implementation): + """Run the unit test suite against pre-release versions of dependencies.""" + session.env["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = implementation -@nox.session(python=PYTHON_VERSIONS) -def unitupb(session): - return unit(session, proto="upb") + # Install test environment dependencies + session.install("coverage", "pytest", "pytest-cov", "pytz") + + # Install the package without dependencies + session.install("-e", ".", "--no-deps") + + prerel_deps = [ + "google-api-core", + # dependency of google-api-core + "googleapis-common-protos", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + session.install("--pre", "--upgrade", "protobuf") + # Print out prerelease package versions + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run( + "python", "-c", "import google.api_core; print(google.api_core.__version__)" + ) + + # TODO(https://github.com/googleapis/proto-plus-python/issues/403): re-enable `-W=error` + # The warnings-as-errors flag `-W=error` was removed in + # https://github.com/googleapis/proto-plus-python/pull/400. + # It should be re-added once issue + # https://github.com/protocolbuffers/protobuf/issues/15077 is fixed. + session.run( + "pytest", + "--quiet", + *( + session.posargs # Coverage info when running individual tests is annoying. + or [ + "--cov=proto", + "--cov-config=.coveragerc", + "--cov-report=term", + "--cov-report=html", + "tests", + ] + ), + ) @nox.session(python="3.9") diff --git a/setup.py b/setup.py index 796f0a91..1de20941 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ install_requires=("protobuf >= 3.19.0, <5.0.0dev",), extras_require={ "testing": [ - "google-api-core[grpc] >= 1.31.5", + "google-api-core >= 1.31.5", ], }, python_requires=">=3.6", diff --git a/testing/constraints-3.13.txt b/testing/constraints-3.13.txt new file mode 100644 index 00000000..e69de29b From 448770fe27ef69a0ecde937148a72db5963b5eca Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 13 Feb 2024 23:14:51 +0000 Subject: [PATCH 2/7] add compatibility with protobuf 5.x --- proto/message.py | 88 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/proto/message.py b/proto/message.py index 7232d42f..048ca665 100644 --- a/proto/message.py +++ b/proto/message.py @@ -18,6 +18,7 @@ import re from typing import List, Type +import google.protobuf from google.protobuf import descriptor_pb2 from google.protobuf import message from google.protobuf.json_format import MessageToDict, MessageToJson, Parse @@ -32,6 +33,8 @@ from proto.utils import has_upb +PROTOBUF_VERSION = google.protobuf.__version__ + _upb = has_upb() # Important to cache result here. @@ -379,6 +382,7 @@ def to_json( sort_keys=False, indent=2, float_precision=None, + always_print_fields_with_no_presence=True, ) -> str: """Given a message instance, serialize it to json @@ -398,18 +402,43 @@ def to_json( Pass None for the most compact representation without newlines. float_precision (Optional(int)): If set, use this to specify float field valid digits. Default is None. + always_print_fields_with_no_presence (Optional(bool)): If True, fields without + presence (implicit presence scalars, repeated fields, and map fields) will + always be serialized. Any field that supports presence is not affected by + this option (including singular message fields and oneof fields). If + `including_default_value_fields` is set to False, this option has no effect. Returns: str: The json string representation of the protocol buffer. """ - return MessageToJson( - cls.pb(instance), - use_integers_for_enums=use_integers_for_enums, - including_default_value_fields=including_default_value_fields, - preserving_proto_field_name=preserving_proto_field_name, - sort_keys=sort_keys, - indent=indent, - float_precision=float_precision, - ) + # For backwards compatibility of this breaking change in protobuf 5.x which is specific to proto2 + # https://github.com/protocolbuffers/protobuf/commit/26995798757fbfef5cf6648610848e389db1fecf + if PROTOBUF_VERSION[0] in ("3", "4"): + return MessageToJson( + cls.pb(instance), + use_integers_for_enums=use_integers_for_enums, + including_default_value_fields=including_default_value_fields, + preserving_proto_field_name=preserving_proto_field_name, + sort_keys=sort_keys, + indent=indent, + float_precision=float_precision, + ) + else: + # The `including_default_value_fields` argument was removed from protobuf 5.x + # and replaced with `always_print_fields_with_no_presence` which very similar but has + # consistent handling of optional fields by not affecting them. + # The old flag accidentally had inconsistent behavior between proto2 optional and proto3 optional fields. + return MessageToJson( + cls.pb(instance), + use_integers_for_enums=use_integers_for_enums, + always_print_fields_with_no_presence=( + including_default_value_fields + and always_print_fields_with_no_presence + ), + preserving_proto_field_name=preserving_proto_field_name, + sort_keys=sort_keys, + indent=indent, + float_precision=float_precision, + ) def from_json(cls, payload, *, ignore_unknown_fields=False) -> "Message": """Given a json string representing an instance, @@ -436,6 +465,7 @@ def to_dict( preserving_proto_field_name=True, including_default_value_fields=True, float_precision=None, + always_print_fields_with_no_presence=True, ) -> "Message": """Given a message instance, return its representation as a python dict. @@ -448,24 +478,48 @@ def to_dict( preserving_proto_field_name (Optional(bool)): An option that determines whether field name representations preserve proto case (snake_case) or use lowerCamelCase. Default is True. - including_default_value_fields (Optional(bool)): An option that + including_default_value_fields (Optional(bool)): Deprecated. Use argument + `always_print_fields_with_no_presence` instead. An option that determines whether the default field values should be included in the results. Default is True. float_precision (Optional(int)): If set, use this to specify float field valid digits. Default is None. + always_print_fields_with_no_presence (Optional(bool)): If True, fields without + presence (implicit presence scalars, repeated fields, and map fields) will + always be serialized. Any field that supports presence is not affected by + this option (including singular message fields and oneof fields). If + `including_default_value_fields` is set to False, this option has no effect. Returns: dict: A representation of the protocol buffer using pythonic data structures. Messages and map fields are represented as dicts, repeated fields are represented as lists. """ - return MessageToDict( - cls.pb(instance), - including_default_value_fields=including_default_value_fields, - preserving_proto_field_name=preserving_proto_field_name, - use_integers_for_enums=use_integers_for_enums, - float_precision=float_precision, - ) + # For backwards compatibility of this breaking change in protobuf 5.x which is specific to proto2 + # https://github.com/protocolbuffers/protobuf/commit/26995798757fbfef5cf6648610848e389db1fecf + if PROTOBUF_VERSION[0] in ("3", "4"): + return MessageToDict( + cls.pb(instance), + including_default_value_fields=including_default_value_fields, + preserving_proto_field_name=preserving_proto_field_name, + use_integers_for_enums=use_integers_for_enums, + float_precision=float_precision, + ) + else: + # The `including_default_value_fields` argument was removed from protobuf 5.x + # and replaced with `always_print_fields_with_no_presence` which very similar but has + # consistent handling of optional fields by not affecting them. + # The old flag accidentally had inconsistent behavior between proto2 optional and proto3 optional fields + return MessageToDict( + cls.pb(instance), + always_print_fields_with_no_presence=( + including_default_value_fields + and always_print_fields_with_no_presence + ), + preserving_proto_field_name=preserving_proto_field_name, + use_integers_for_enums=use_integers_for_enums, + float_precision=float_precision, + ) def copy_from(cls, instance, other): """Equivalent for protobuf.Message.CopyFrom From 19c487f2a9a52d9c0f698937ca2f7eda0d1c6c4f Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 14 Feb 2024 15:07:01 +0000 Subject: [PATCH 3/7] fix RecursionError: maximum recursion depth exceeded while calling a Python object in tests --- tests/conftest.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 765326c6..252ac30c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -67,6 +67,14 @@ def pytest_runtest_setup(item): for name in dir(item.module): if name.endswith("_pb2") and not name.startswith("test_"): module = getattr(item.module, name) + + # Exclude `google.protobuf.descriptor_pb2` which causes error + # `RecursionError: maximum recursion depth exceeded while calling a Python object` + # when running the test suite and is not required for tests. + # See https://github.com/googleapis/proto-plus-python/issues/425 + if module.__package__ == "google.protobuf" and name == "descriptor_pb2": + continue + pool.AddSerializedFile(module.DESCRIPTOR.serialized_pb) fd = pool.FindFileByName(module.DESCRIPTOR.name) From 69796dd34867961721e371cb9bbcc7ebdd398af3 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 14 Feb 2024 15:19:40 +0000 Subject: [PATCH 4/7] update required checks --- .github/sync-repo-settings.yaml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index f1068589..169e5af7 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -7,24 +7,27 @@ branchProtectionRules: requiredStatusCheckContexts: - 'style-check' - 'docs' - - 'unit (3.6)' - 'unit (3.6, cpp)' - - 'unit (3.7)' + - 'unit (3.6, python)' + - 'unit (3.6, upb)' - 'unit (3.7, cpp)' + - 'unit (3.7, python)' - 'unit (3.7, upb)' - - 'unit (3.8)' - 'unit (3.8, cpp)' + - 'unit (3.8, python)' - 'unit (3.8, upb)' - - 'unit (3.9)' - 'unit (3.9, cpp)' + - 'unit (3.9, python)' - 'unit (3.9, upb)' - - 'unit (3.10)' - 'unit (3.10, cpp)' + - 'unit (3.10, python)' - 'unit (3.10, upb)' - 'unit (3.11)' - 'unit (3.11, upb)' - 'unit (3.12)' - 'unit (3.12, upb)' + - 'prerelease (3.12, python)' + - 'prerelease (3.12, upb)' - cover - OwlBot Post Processor - 'cla/google' From ef74377c8e54c259a64be0e87d64f07c0f0f66c7 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 14 Feb 2024 15:23:57 +0000 Subject: [PATCH 5/7] update comments --- proto/message.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/proto/message.py b/proto/message.py index 048ca665..768f7020 100644 --- a/proto/message.py +++ b/proto/message.py @@ -392,6 +392,11 @@ def to_json( use_integers_for_enums (Optional(bool)): An option that determines whether enum values should be represented by strings (False) or integers (True). Default is True. + including_default_value_fields (Optional(bool)): Deprecated. Use argument + `always_print_fields_with_no_presence` instead. An option that + determines whether the default field values should be included in the results. + Default is True. If `always_print_fields_with_no_presence` is set to False, this + option has no effect. preserving_proto_field_name (Optional(bool)): An option that determines whether field name representations preserve proto case (snake_case) or use lowerCamelCase. Default is False. @@ -481,7 +486,8 @@ def to_dict( including_default_value_fields (Optional(bool)): Deprecated. Use argument `always_print_fields_with_no_presence` instead. An option that determines whether the default field values should be included in the results. - Default is True. + Default is True. If `always_print_fields_with_no_presence` is set to False, this + option has no effect. float_precision (Optional(int)): If set, use this to specify float field valid digits. Default is None. always_print_fields_with_no_presence (Optional(bool)): If True, fields without From 947db0819f7ee2e89de4e3444f2c245beaf5a30c Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 14 Feb 2024 15:24:54 +0000 Subject: [PATCH 6/7] update required checks --- .github/sync-repo-settings.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 169e5af7..1bb27933 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -22,9 +22,9 @@ branchProtectionRules: - 'unit (3.10, cpp)' - 'unit (3.10, python)' - 'unit (3.10, upb)' - - 'unit (3.11)' + - 'unit (3.11, python)' - 'unit (3.11, upb)' - - 'unit (3.12)' + - 'unit (3.12, python)' - 'unit (3.12, upb)' - 'prerelease (3.12, python)' - 'prerelease (3.12, upb)' From 4fae4f9e7aedd0e165281614864dfa72684ccb43 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 14 Feb 2024 15:47:05 +0000 Subject: [PATCH 7/7] chore(main): release 1.24.0.dev0 --- CHANGELOG.md | 18 ++++++++++++++++++ proto/version.py | 2 +- setup.py | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a22bc063..de815ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [1.24.0.dev0](https://github.com/googleapis/proto-plus-python/compare/v1.22.3...v1.24.0dev0) (2024-02-14) + + +### Features + +* Add `always_print_fields_with_no_presence` fields to `to_json` and `to_dict` ([#433])(https://github.com/googleapis/proto-plus-python/pull/433) + + +### Bug Fixes + +* Fix compatibility with `protobuf==5.26.0rc2` ([#433])(https://github.com/googleapis/proto-plus-python/pull/433) + + +### Documentation + +* Deprecate field `including_default_value_fields` in `to_json` and `to_dict` ([#433])(https://github.com/googleapis/proto-plus-python/pull/433) + + ## [1.23.0](https://github.com/googleapis/proto-plus-python/compare/v1.22.3...v1.23.0) (2023-12-01) diff --git a/proto/version.py b/proto/version.py index f11d15fb..a4245c2d 100644 --- a/proto/version.py +++ b/proto/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "1.23.0" +__version__ = "1.24.0.dev0" diff --git a/setup.py b/setup.py index 1de20941..3df74401 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ version = None with open(os.path.join(PACKAGE_ROOT, "proto/version.py")) as fp: - version_candidates = re.findall(r"(?<=\")\d+.\d+.\d+(?=\")", fp.read()) + version_candidates = re.findall(r"(?<=\")\d+.\d+.\d+.dev\d+", fp.read()) assert len(version_candidates) == 1 version = version_candidates[0]