From bede3e70a713a7abc0a59e52a2d5c6b86b027f79 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:33:27 -0500 Subject: [PATCH 01/59] bump to the now-current nixpkgs release --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 85817dde..4e8ad455 100644 --- a/default.nix +++ b/default.nix @@ -1,7 +1,7 @@ let sources = import nix/sources.nix; in -{ pkgs ? import sources.release2105 {} +{ pkgs ? import sources.release2111 {} , pypiData ? sources.pypi-deps-db , mach-nix ? import sources.mach-nix { inherit pkgs pypiData; } , tahoe-lafs-source ? "tahoe-lafs" From 96e4c469183463e426a13d442fd50c62cb7d9315 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:33:55 -0500 Subject: [PATCH 02/59] parameterize python version in the nix package --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 4e8ad455..e68d6e04 100644 --- a/default.nix +++ b/default.nix @@ -6,10 +6,10 @@ in , mach-nix ? import sources.mach-nix { inherit pkgs pypiData; } , tahoe-lafs-source ? "tahoe-lafs" , tahoe-lafs-repo ? sources.${tahoe-lafs-source} +, python ? "python27" }: let lib = pkgs.lib; - python = "python27"; providers = { _default = "sdist,nixpkgs,wheel"; # mach-nix doesn't provide a good way to depend on mach-nix packages, From 22b0a17c8b71a9330ef9e8f014f642995c47d264 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:54:03 -0500 Subject: [PATCH 03/59] fix problem with the new collections-extended dependency --- default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/default.nix b/default.nix index e68d6e04..29fc4f48 100644 --- a/default.nix +++ b/default.nix @@ -34,6 +34,11 @@ in # The version of Klein we get doesn't need / can't have the patch that # comes from the nixpkgs derivation mach-nix picks up from 21.05. klein = "wheel"; + + # - has an undetected poetry dependency and when trying to work around + # this another way, dependencies have undetected dependencies, easier + # to just use the wheel. + collections-extended = "wheel"; }; in rec { From d844dedab4dcc1b906662788ab68ff860a2c9a64 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:54:25 -0500 Subject: [PATCH 04/59] fix problem with the new cryptography build system --- default.nix | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/default.nix b/default.nix index 29fc4f48..a0b529ef 100644 --- a/default.nix +++ b/default.nix @@ -39,6 +39,11 @@ in # this another way, dependencies have undetected dependencies, easier # to just use the wheel. collections-extended = "wheel"; + + # From nixpkgs or sdist, fails with + # cp: cannot stat 'benchmark/': No such file or directory + # cp: cannot stat 'tests/': No such file or directory + tomli = "wheel"; }; in rec { @@ -62,6 +67,12 @@ in foolscap == 0.13.1 configparser eliot + # undetected cryptography build dependency + # https://github.com/DavHau/mach-nix/issues/305 + setuptools_rust + # undetected tomli build dependency + # probably same underlying cause as cryptography issue + flit_core ''; postPatch = '' cat > src/allmydata/_version.py < Date: Mon, 27 Dec 2021 10:55:23 -0500 Subject: [PATCH 05/59] fix the new conditional foolscap dependency --- default.nix | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index a0b529ef..cee9ff5d 100644 --- a/default.nix +++ b/default.nix @@ -62,11 +62,20 @@ in # going on and discover the real version specified by `src` below. version = "1.17.0.post999"; # See https://github.com/DavHau/mach-nix/issues/190 - requirementsExtra = '' + requirementsExtra = + let + foolscap-version = + if python <= "python30" + then "foolscap == 0.13.1" + else "foolscap >= 21.7.0"; + in + '' + # See https://github.com/DavHau/mach-nix/issues/190 pyrsistent < 0.17 - foolscap == 0.13.1 configparser eliot + ${foolscap-version} + # undetected cryptography build dependency # https://github.com/DavHau/mach-nix/issues/305 setuptools_rust From b3bacf40142448862204c74286e87b319f2a17c2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:55:37 -0500 Subject: [PATCH 06/59] update the pypi database to get fresh-enough versions --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index fdde626e..467ec325 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -29,10 +29,10 @@ "homepage": "", "owner": "DavHau", "repo": "pypi-deps-db", - "rev": "96d01556b4597c022647acbf8c3b58d2a99bc963", - "sha256": "0s6ll2hi40gj6mp2zdg7w3dq17g381gnfkm390mqgp574lmbq6yw", + "rev": "856d67ab093a68425c3896ec2961cea3b95ae93f", + "sha256": "0a7avm4xvm454gxy4dq0fc19j3f9ik8gf3kjpsqhszfkamrn9y0p", "type": "tarball", - "url": "https://github.com/DavHau/pypi-deps-db/archive/96d01556b4597c022647acbf8c3b58d2a99bc963.tar.gz", + "url": "https://github.com/DavHau/pypi-deps-db/archive/856d67ab093a68425c3896ec2961cea3b95ae93f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "release2105": { From b7f22fc40071e8b31ee5ae260be01965b44d5ac4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:55:54 -0500 Subject: [PATCH 07/59] add source for the now-current nixos release --- nix/sources.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nix/sources.json b/nix/sources.json index 467ec325..965c4b8e 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -41,6 +41,18 @@ "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz", "url_template": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz" }, + "release2111": { + "branch": "21.11", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31", + "sha256": "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "tahoe-lafs": { "branch": "master", "description": "The Tahoe-LAFS decentralized secure filesystem.", From 013d87316fde96e3a4cd5cc304d782c89faa887f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:56:08 -0500 Subject: [PATCH 08/59] bump the shell environment nixos release as well --- shell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.nix b/shell.nix index 023fc1bd..5f5d6489 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,7 @@ let inherit (tests) pkgs; in pkgs.mkShell { - buildInputs = [ + packages = [ tests.python tests.lint-python pkgs.niv From cd120f39f068be468a62896040f4ea9e5f3f9c57 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 10:58:17 -0500 Subject: [PATCH 09/59] respect the chosen python version in the tests and lint tools --- tests.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.nix b/tests.nix index 40412a75..bf355d84 100644 --- a/tests.nix +++ b/tests.nix @@ -35,7 +35,7 @@ let }; lint-python = mach-nix.mkPython { - python = "python39"; + inherit (zkapauthorizer.meta.mach-nix) python; requirements = '' isort black From 0a0fe9dd5051a11b84bcd45657bf156872958744 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:09:38 -0500 Subject: [PATCH 10/59] nope, use python39 for lint because tools are missing from py27 ecosystem --- tests.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests.nix b/tests.nix index bf355d84..40412a75 100644 --- a/tests.nix +++ b/tests.nix @@ -35,7 +35,7 @@ let }; lint-python = mach-nix.mkPython { - inherit (zkapauthorizer.meta.mach-nix) python; + python = "python39"; requirements = '' isort black From 96137fe04396d3c82de48bd7840fbcbd7b1e6f65 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:10:16 -0500 Subject: [PATCH 11/59] switch to flake8 to be able to suppress warnings there will be tons of pyflakes warnings from the python3 porting strategy --- setup.cfg | 7 +++++++ tests.nix | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6d39a54c..65fd066c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,3 +63,10 @@ versionfile_source = src/_zkapauthorizer/_version.py versionfile_build = _zkapauthorizer/_version.py tag_prefix = release- parentdir_prefix = ZKAPAuthorizer + +[flake8] +# Enforce all pyflakes constraints, and also prohibit tabs for indentation. +# Reference: +# https://flake8.pycqa.org/en/latest/user/error-codes.html +# https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +select = F, W191 diff --git a/tests.nix b/tests.nix index 40412a75..f51d5727 100644 --- a/tests.nix +++ b/tests.nix @@ -51,9 +51,9 @@ let mkdir -p $out pushd ${zkapauthorizer.src} - ${python}/bin/pyflakes src ${lint-python}/bin/black --check src ${lint-python}/bin/isort --check src + ${lint-python}/bin/flake8 src popd ZKAPAUTHORIZER_HYPOTHESIS_PROFILE=${hypothesisProfile'} ${python}/bin/python -m ${if collectCoverage From 244cc324c6e54c06194e44cdc65a134d25276539 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:10:41 -0500 Subject: [PATCH 12/59] switch from the nixos release tag to branch the branch has some python packaging fixes in nixpkgs that we need --- nix/sources.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 965c4b8e..3107940a 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -42,15 +42,15 @@ "url_template": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz" }, "release2111": { - "branch": "21.11", + "branch": "release-21.11", "description": "Nix Packages collection", "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31", - "sha256": "162dywda2dvfj1248afxc45kcrg83appjd0nmdb541hl7rnncf02", + "rev": "d887ac7aee92e8fc54dde9060d60d927afae9d69", + "sha256": "1bpgfv45b1yvrgpwdgc4fm4a6sav198yd41bsrvlmm3jn2wi6qx5", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/a7ecde854aee5c4c7cd6177f54a99d2c1ff28a31.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/d887ac7aee92e8fc54dde9060d60d927afae9d69.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "tahoe-lafs": { From 86a8aa1a88d2c77932949a82a58e63523f936171 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:11:30 -0500 Subject: [PATCH 13/59] fix nixpkgs pkgconfig so mach-nix works at all also refactor tests.nix and default.nix so they share more of the environment setup which is now slightly more complex --- default.nix | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/default.nix b/default.nix index cee9ff5d..5166f554 100644 --- a/default.nix +++ b/default.nix @@ -1,17 +1,31 @@ let sources = import nix/sources.nix; + + # nixpkgs 21.11 packages a version of pkgconfig that is broken on Python 2.7 + # so supply our own. mach-nix depends on this to set up its own build + # environment and cannot discover a working version from pypi the way it + # works for other dependencies. + fixPkgconfig = self: super: { + python27 = super.python27.override { + packageOverrides = python-self: python-super: { + pkgconfig = python-super.pythonPackages.callPackage ./pkgconfig.nix {}; + }; + }; + }; in -{ pkgs ? import sources.release2111 {} +{ pkgs ? import sources.release2111 { overlays = [ fixPkgconfig ]; } , pypiData ? sources.pypi-deps-db -, mach-nix ? import sources.mach-nix { inherit pkgs pypiData; } +, python ? "python27" +, mach-nix ? import sources.mach-nix { inherit pkgs pypiData python; } , tahoe-lafs-source ? "tahoe-lafs" , tahoe-lafs-repo ? sources.${tahoe-lafs-source} -, python ? "python27" +, ... }: let lib = pkgs.lib; providers = { _default = "sdist,nixpkgs,wheel"; + # mach-nix doesn't provide a good way to depend on mach-nix packages, # so we get it as a nixpkgs dependency from an overlay. See below for # details. From 2bd1f45bef9728197962a7881789d27022e7cb1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:18:37 -0500 Subject: [PATCH 14/59] initial porting changes --- src/_zkapauthorizer/_storage_client.py | 2 +- src/_zkapauthorizer/_storage_server.py | 19 ++- src/_zkapauthorizer/controller.py | 83 ++++++----- src/_zkapauthorizer/eliot.py | 67 +++++---- src/_zkapauthorizer/model.py | 130 ++++++++++-------- src/_zkapauthorizer/resource.py | 72 +++++----- src/_zkapauthorizer/storage_common.py | 40 ++++-- src/_zkapauthorizer/tests/strategies.py | 65 +++++---- .../tests/test_client_resource.py | 121 ++++++++-------- src/_zkapauthorizer/tests/test_controller.py | 53 +++++++ src/_zkapauthorizer/tests/test_model.py | 2 +- src/_zkapauthorizer/tests/test_plugin.py | 35 +++-- .../tests/test_pricecalculator.py | 10 ++ 13 files changed, 418 insertions(+), 281 deletions(-) diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index 5f4e5685..28670659 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -373,7 +373,7 @@ def add_lease( None, ) ).values() - num_passes = required_passes(self._pass_value, share_sizes) + num_passes = required_passes(self._pass_value, list(share_sizes)) result = yield call_with_passes( lambda passes: rref.callRemote( diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 809b3851..2d774aca 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -22,6 +22,13 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import timedelta from errno import ENOENT @@ -122,13 +129,13 @@ def _is_invalid_pass(cls, message, pass_, signing_key): """ Cryptographically check the validity of a single pass. - :param unicode message: The shared message for pass validation. + :param str message: The shared message for pass validation. :param Pass pass_: The pass to validate. :return bool: ``False`` (invalid) if the pass includes a valid signature, ``True`` (valid) otherwise. """ - assert isinstance(message, unicode), "message %r not unicode" % (message,) + assert isinstance(message, str), "message %r not str" % (message,) assert isinstance(pass_, Pass), "pass %r not a Pass" % (pass_,) try: preimage = TokenPreimage.decode_base64(pass_.preimage) @@ -148,7 +155,7 @@ def validate_passes(cls, message, passes, signing_key): """ Check all of the given passes for validity. - :param unicode message: The shared message for pass validation. + :param str message: The shared message for pass validation. :param list[bytes] passes: The encoded passes to validate. :param SigningKey signing_key: The signing key to use to check the passes. @@ -398,7 +405,7 @@ def remote_advise_corrupt_share(self, *a, **kw): def remote_share_sizes(self, storage_index_or_slot, sharenums): with start_action( - action_type=u"zkapauthorizer:storage-server:remote:share-sizes", + action_type="zkapauthorizer:storage-server:remote:share-sizes", storage_index_or_slot=storage_index_or_slot, ): return dict( @@ -443,7 +450,7 @@ def remote_slot_testv_and_readv_and_writev( Note that the lease is *not* renewed in this case (see #254). """ with start_action( - action_type=u"zkapauthorizer:storage-server:remote:slot-testv-and-readv-and-writev", + action_type="zkapauthorizer:storage-server:remote:slot-testv-and-readv-and-writev", storage_index=b2a(storage_index), path=storage_index_to_dir(storage_index), ): @@ -877,7 +884,7 @@ def get_share_path(storage_server, storage_index, sharenum): return ( FilePath(storage_server.sharedir) .preauthChild(storage_index_to_dir(storage_index)) - .child(u"{}".format(sharenum)) + .child("{}".format(sharenum)) ) diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index ab52c5bc..c4f202ee 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -18,6 +18,13 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from base64 import b64decode, b64encode from datetime import timedelta @@ -97,7 +104,7 @@ class RedemptionResult(object): :ivar list[UnblindedToken] unblinded_tokens: The tokens which resulted from the redemption. - :ivar unicode public_key: The public key which the server proved was + :ivar str public_key: The public key which the server proved was involved in the redemption process. """ @@ -238,13 +245,13 @@ class ErrorRedeemer(object): configured error. """ - details = attr.ib(validator=attr.validators.instance_of(unicode)) + details = attr.ib(validator=attr.validators.instance_of(str)) @classmethod def make(cls, section_name, node_config, announcement, reactor): details = node_config.get_config( section=section_name, - option=u"details", + option="details", ).decode("ascii") return cls(details) @@ -325,7 +332,7 @@ def dummy_random_token(n): # Padding is 96 (random token length) - 32 (decoded voucher # length) - 4 (fixed-width counter) b64encode( - v + u"{:0>4}{:0>60}".format(counter, n).encode("ascii"), + v + "{:0>4}{:0>60}".format(counter, n).encode("ascii"), ), ) @@ -340,7 +347,7 @@ class DummyRedeemer(object): really redeeming them, it makes up some fake ZKAPs and pretends those are the result. - :ivar unicode _public_key: The base64-encoded public key to return with + :ivar str _public_key: The base64-encoded public key to return with all successful redemption results. As with the tokens returned by this redeemer, chances are this is not actually a valid public key. Its corresponding private key certainly has not been used to sign @@ -348,7 +355,7 @@ class DummyRedeemer(object): """ _public_key = attr.ib( - validator=attr.validators.instance_of(unicode), + validator=attr.validators.instance_of(str), ) @classmethod @@ -356,8 +363,8 @@ def make(cls, section_name, node_config, announcement, reactor): return cls( node_config.get_config( section=section_name, - option=u"issuer-public-key", - ).decode(u"utf-8"), + option="issuer-public-key", + ).decode("utf-8"), ) def random_tokens_for_voucher(self, voucher, counter, count): @@ -461,7 +468,7 @@ class RistrettoRedeemer(object): def make(cls, section_name, node_config, announcement, reactor): configured_issuer = node_config.get_config( section=section_name, - option=u"ristretto-issuer-root-url", + option="ristretto-issuer-root-url", ).decode("ascii") if announcement is not None: # Don't let us talk to a storage server that has a different idea @@ -473,7 +480,7 @@ def make(cls, section_name, node_config, announcement, reactor): # If we aren't given an announcement then we're not being used in # the context of a specific storage server so the check is # unnecessary and impossible. - announced_issuer = announcement[u"ristretto-issuer-root-url"] + announced_issuer = announcement["ristretto-issuer-root-url"] if announced_issuer != configured_issuer: raise IssuerConfigurationMismatch(announced_issuer, configured_issuer) @@ -498,12 +505,12 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): ) blinded_tokens = list(token.blind() for token in random_tokens) response = yield self._treq.post( - self._api_root.child(u"v1", u"redeem").to_text(), + self._api_root.child("v1", "redeem").to_text(), dumps( { - u"redeemVoucher": voucher.number.decode("ascii"), - u"redeemCounter": counter, - u"redeemTokens": list( + "redeemVoucher": voucher.number.decode("ascii"), + "redeemCounter": counter, + "redeemTokens": list( token.encode_base64() for token in blinded_tokens ), } @@ -517,26 +524,26 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): except ValueError: raise UnexpectedResponse(response.code, response_body) - success = result.get(u"success", False) + success = result.get("success", False) if not success: - reason = result.get(u"reason", None) - if reason == u"double-spend": + reason = result.get("reason", None) + if reason == "double-spend": raise AlreadySpent(voucher) - elif reason == u"unpaid": + elif reason == "unpaid": raise Unpaid(voucher) raise UnrecognizedFailureReason(result) self._log.info( "Redeemed: {public_key} {proof} {count}", - public_key=result[u"public-key"], - proof=result[u"proof"], - count=len(result[u"signatures"]), + public_key=result["public-key"], + proof=result["proof"], + count=len(result["signatures"]), ) - marshaled_signed_tokens = result[u"signatures"] - marshaled_proof = result[u"proof"] - marshaled_public_key = result[u"public-key"] + marshaled_signed_tokens = result["signatures"] + marshaled_proof = result["proof"] + marshaled_public_key = result["public-key"] public_key = challenge_bypass_ristretto.PublicKey.decode_base64( marshaled_public_key.encode("ascii"), @@ -660,17 +667,17 @@ class PaymentController(object): redeeming a voucher, if no other count is given when the redemption is started. - :ivar set[unicode] allowed_public_keys: The base64-encoded public keys for + :ivar set[str] allowed_public_keys: The base64-encoded public keys for which to accept tokens. - :ivar dict[unicode, Redeeming] _active: A mapping from voucher identifiers + :ivar dict[str, Redeeming] _active: A mapping from voucher identifiers which currently have redemption attempts in progress to a ``Redeeming`` state representing the attempt. - :ivar dict[unicode, datetime] _error: A mapping from voucher identifiers + :ivar dict[str, datetime] _error: A mapping from voucher identifiers which have recently failed with an unrecognized, transient error. - :ivar dict[unicode, datetime] _unpaid: A mapping from voucher identifiers + :ivar dict[str, datetime] _unpaid: A mapping from voucher identifiers which have recently failed a redemption attempt due to an unpaid response from the redemption server to timestamps when the failure was observed. @@ -732,7 +739,7 @@ def _schedule_retries(self): ) def _retry_redemption(self): - for voucher in self._error.keys() + self._unpaid.keys(): + for voucher in list(self._error.keys()) + list(self._unpaid.keys()): if voucher in self._active: continue if self.get_voucher(voucher).state.should_start_redemption(): @@ -982,22 +989,22 @@ def incorporate_transient_state(self, voucher): def get_redeemer(plugin_name, node_config, announcement, reactor): - section_name = u"storageclient.plugins.{}".format(plugin_name) + section_name = "storageclient.plugins.{}".format(plugin_name) redeemer_kind = node_config.get_config( section=section_name, - option=u"redeemer", - default=u"ristretto", + option="redeemer", + default="ristretto", ) return _REDEEMERS[redeemer_kind](section_name, node_config, announcement, reactor) _REDEEMERS = { - u"non": NonRedeemer.make, - u"dummy": DummyRedeemer.make, - u"double-spend": DoubleSpendRedeemer.make, - u"unpaid": UnpaidRedeemer.make, - u"error": ErrorRedeemer.make, - u"ristretto": RistrettoRedeemer.make, + "non": NonRedeemer.make, + "dummy": DummyRedeemer.make, + "double-spend": DoubleSpendRedeemer.make, + "unpaid": UnpaidRedeemer.make, + "error": ErrorRedeemer.make, + "ristretto": RistrettoRedeemer.make, } diff --git a/src/_zkapauthorizer/eliot.py b/src/_zkapauthorizer/eliot.py index 8f607d8a..ccd84727 100644 --- a/src/_zkapauthorizer/eliot.py +++ b/src/_zkapauthorizer/eliot.py @@ -17,90 +17,97 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from eliot import ActionType, Field, MessageType PRIVACYPASS_MESSAGE = Field( - u"message", - unicode, - u"The PrivacyPass request-binding data associated with a pass.", + "message", + str, + "The PrivacyPass request-binding data associated with a pass.", ) INVALID_REASON = Field( - u"reason", - unicode, - u"The reason given by the server for rejecting a pass as invalid.", + "reason", + str, + "The reason given by the server for rejecting a pass as invalid.", ) PASS_COUNT = Field( - u"count", + "count", int, - u"A number of passes.", + "A number of passes.", ) GET_PASSES = MessageType( - u"zkapauthorizer:get-passes", + "zkapauthorizer:get-passes", [PRIVACYPASS_MESSAGE, PASS_COUNT], - u"An attempt to spend passes is beginning.", + "An attempt to spend passes is beginning.", ) SPENT_PASSES = MessageType( - u"zkapauthorizer:spent-passes", + "zkapauthorizer:spent-passes", [PASS_COUNT], - u"An attempt to spend passes has succeeded.", + "An attempt to spend passes has succeeded.", ) INVALID_PASSES = MessageType( - u"zkapauthorizer:invalid-passes", + "zkapauthorizer:invalid-passes", [INVALID_REASON, PASS_COUNT], - u"An attempt to spend passes has found some to be invalid.", + "An attempt to spend passes has found some to be invalid.", ) RESET_PASSES = MessageType( - u"zkapauthorizer:reset-passes", + "zkapauthorizer:reset-passes", [PASS_COUNT], - u"Some passes involved in a failed spending attempt have not definitely been spent and are being returned for future use.", + "Some passes involved in a failed spending attempt have not definitely been spent and are being returned for future use.", ) SIGNATURE_CHECK_FAILED = MessageType( - u"zkapauthorizer:storage-client:signature-check-failed", + "zkapauthorizer:storage-client:signature-check-failed", [PASS_COUNT], - u"Some passes the client tried to use were rejected for having invalid signatures.", + "Some passes the client tried to use were rejected for having invalid signatures.", ) CALL_WITH_PASSES = ActionType( - u"zkapauthorizer:storage-client:call-with-passes", + "zkapauthorizer:storage-client:call-with-passes", [PASS_COUNT], [], - u"A storage operation is being started which may spend some passes.", + "A storage operation is being started which may spend some passes.", ) CURRENT_SIZES = Field( - u"current_sizes", + "current_sizes", dict, - u"A dictionary mapping the numbers of existing shares to their existing sizes.", + "A dictionary mapping the numbers of existing shares to their existing sizes.", ) TW_VECTORS_SUMMARY = Field( - u"tw_vectors_summary", + "tw_vectors_summary", dict, - u"A dictionary mapping share numbers from tw_vectors to test and write vector summaries.", + "A dictionary mapping share numbers from tw_vectors to test and write vector summaries.", ) NEW_SIZES = Field( - u"new_sizes", + "new_sizes", dict, - u"A dictionary like that of CURRENT_SIZES but for the sizes computed for the shares after applying tw_vectors.", + "A dictionary like that of CURRENT_SIZES but for the sizes computed for the shares after applying tw_vectors.", ) NEW_PASSES = Field( - u"new_passes", + "new_passes", int, - u"The number of passes computed as being required for the change in size.", + "The number of passes computed as being required for the change in size.", ) MUTABLE_PASSES_REQUIRED = MessageType( - u"zkapauthorizer:storage:mutable-passes-required", + "zkapauthorizer:storage:mutable-passes-required", [CURRENT_SIZES, TW_VECTORS_SUMMARY, NEW_SIZES, NEW_PASSES], - u"Some number of passes has been computed as the cost of updating a mutable.", + "Some number of passes has been computed as the cost of updating a mutable.", ) diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index 7530d2b2..57b7cb4a 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -17,6 +17,16 @@ the storage plugin. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long + from datetime import datetime from functools import wraps from json import dumps, loads @@ -41,12 +51,12 @@ def parse_datetime(s, **kw): """ - Like ``aniso8601.parse_datetime`` but accept unicode as well. + Like ``aniso8601.parse_datetime`` but accept str as well. """ - if isinstance(s, unicode): + if isinstance(s, str): s = s.encode("utf-8") assert isinstance(s, bytes) - if "delimiter" in kw and isinstance(kw["delimiter"], unicode): + if "delimiter" in kw and isinstance(kw["delimiter"], str): kw["delimiter"] = kw["delimiter"].encode("utf-8") return _parse_datetime(s, **kw) @@ -86,7 +96,7 @@ class NotEnoughTokens(Exception): """ -CONFIG_DB_NAME = u"privatestorageio-zkapauthz-v1.sqlite3" +CONFIG_DB_NAME = "privatestorageio-zkapauthz-v1.sqlite3" def open_and_initialize(path, connect=None): @@ -380,7 +390,7 @@ def insert_unblinded_tokens(self, cursor, unblinded_tokens, group_id): Store some unblinded tokens, for example as part of a backup-restore process. - :param list[unicode] unblinded_tokens: The unblinded tokens to store. + :param list[str] unblinded_tokens: The unblinded tokens to store. :param int group_id: The unique identifier of the redemption group to which these tokens belong. @@ -398,7 +408,7 @@ def insert_unblinded_tokens_for_voucher( tokens. This voucher will be marked as redeemed to indicate it has fulfilled its purpose and has no further use for us. - :param unicode public_key: The encoded public key for the private key + :param str public_key: The encoded public key for the private key which was used to sign these tokens. :param list[UnblindedToken] unblinded_tokens: The unblinded tokens to @@ -411,9 +421,9 @@ def insert_unblinded_tokens_for_voucher( inserted tokens, ``False`` otherwise. """ if completed: - voucher_state = u"redeemed" + voucher_state = "redeemed" else: - voucher_state = u"pending" + voucher_state = "pending" if spendable: token_count_increase = len(unblinded_tokens) @@ -683,7 +693,7 @@ def backup(self, cursor): ) tokens = cursor.fetchall() return { - u"unblinded-tokens": list(token for (token,) in tokens), + "unblinded-tokens": list(token for (token,) in tokens), } def start_lease_maintenance(self): @@ -720,9 +730,9 @@ def get_latest_lease_maintenance_activity(self, cursor): return None [(started, count, finished)] = activity return LeaseMaintenanceActivity( - parse_datetime(started, delimiter=u" "), + parse_datetime(started, delimiter=" "), count, - parse_datetime(finished, delimiter=u" "), + parse_datetime(finished, delimiter=" "), ) @@ -923,8 +933,8 @@ def should_start_redemption(self): def to_json_v1(self): return { - u"name": u"pending", - u"counter": self.counter, + "name": "pending", + "counter": self.counter, } @@ -944,9 +954,9 @@ def should_start_redemption(self): def to_json_v1(self): return { - u"name": u"redeeming", - u"started": self.started.isoformat(), - u"counter": self.counter, + "name": "redeeming", + "started": self.started.isoformat(), + "counter": self.counter, } @@ -969,9 +979,9 @@ def should_start_redemption(self): def to_json_v1(self): return { - u"name": u"redeemed", - u"finished": self.finished.isoformat(), - u"token-count": self.token_count, + "name": "redeemed", + "finished": self.finished.isoformat(), + "token-count": self.token_count, } @@ -984,8 +994,8 @@ def should_start_redemption(self): def to_json_v1(self): return { - u"name": u"double-spend", - u"finished": self.finished.isoformat(), + "name": "double-spend", + "finished": self.finished.isoformat(), } @@ -1004,8 +1014,8 @@ def should_start_redemption(self): def to_json_v1(self): return { - u"name": u"unpaid", - u"finished": self.finished.isoformat(), + "name": "unpaid", + "finished": self.finished.isoformat(), } @@ -1018,16 +1028,16 @@ class Error(object): """ finished = attr.ib(validator=attr.validators.instance_of(datetime)) - details = attr.ib(validator=attr.validators.instance_of(unicode)) + details = attr.ib(validator=attr.validators.instance_of(str)) def should_start_redemption(self): return True def to_json_v1(self): return { - u"name": u"error", - u"finished": self.finished.isoformat(), - u"details": self.details, + "name": "error", + "finished": self.finished.isoformat(), + "details": self.details, } @@ -1089,15 +1099,15 @@ class Voucher(object): @classmethod def from_row(cls, row): def state_from_row(state, row): - if state == u"pending": + if state == "pending": return Pending(counter=row[3]) - if state == u"double-spend": + if state == "double-spend": return DoubleSpend( - parse_datetime(row[0], delimiter=u" "), + parse_datetime(row[0], delimiter=" "), ) - if state == u"redeemed": + if state == "redeemed": return Redeemed( - parse_datetime(row[0], delimiter=u" "), + parse_datetime(row[0], delimiter=" "), row[1], ) raise ValueError("Unknown voucher state {}".format(state)) @@ -1112,54 +1122,54 @@ def state_from_row(state, row): # value represents a leap second. However, since we also use # Python to generate the data in the first place, it should never # represent a leap second... I hope. - created=parse_datetime(created, delimiter=u" "), + created=parse_datetime(created, delimiter=" "), state=state_from_row(state, row[4:]), ) @classmethod def from_json(cls, json): values = loads(json) - version = values.pop(u"version") + version = values.pop("version") return getattr(cls, "from_json_v{}".format(version))(values) @classmethod def from_json_v1(cls, values): - state_json = values[u"state"] - state_name = state_json[u"name"] - if state_name == u"pending": - state = Pending(counter=state_json[u"counter"]) - elif state_name == u"redeeming": + state_json = values["state"] + state_name = state_json["name"] + if state_name == "pending": + state = Pending(counter=state_json["counter"]) + elif state_name == "redeeming": state = Redeeming( - started=parse_datetime(state_json[u"started"]), - counter=state_json[u"counter"], + started=parse_datetime(state_json["started"]), + counter=state_json["counter"], ) - elif state_name == u"double-spend": + elif state_name == "double-spend": state = DoubleSpend( - finished=parse_datetime(state_json[u"finished"]), + finished=parse_datetime(state_json["finished"]), ) - elif state_name == u"redeemed": + elif state_name == "redeemed": state = Redeemed( - finished=parse_datetime(state_json[u"finished"]), - token_count=state_json[u"token-count"], + finished=parse_datetime(state_json["finished"]), + token_count=state_json["token-count"], ) - elif state_name == u"unpaid": + elif state_name == "unpaid": state = Unpaid( - finished=parse_datetime(state_json[u"finished"]), + finished=parse_datetime(state_json["finished"]), ) - elif state_name == u"error": + elif state_name == "error": state = Error( - finished=parse_datetime(state_json[u"finished"]), - details=state_json[u"details"], + finished=parse_datetime(state_json["finished"]), + details=state_json["details"], ) else: raise ValueError("Unrecognized state {!r}".format(state_json)) return cls( - number=values[u"number"].encode("ascii"), - expected_tokens=values[u"expected-tokens"], + number=values["number"].encode("ascii"), + expected_tokens=values["expected-tokens"], created=None - if values[u"created"] is None - else parse_datetime(values[u"created"]), + if values["created"] is None + else parse_datetime(values["created"]), state=state, ) @@ -1172,9 +1182,9 @@ def marshal(self): def to_json_v1(self): state = self.state.to_json_v1() return { - u"number": self.number.decode("ascii"), - u"expected-tokens": self.expected_tokens, - u"created": None if self.created is None else self.created.isoformat(), - u"state": state, - u"version": 1, + "number": self.number.decode("ascii"), + "expected-tokens": self.expected_tokens, + "created": None if self.created is None else self.created.isoformat(), + "state": state, + "version": 1, } diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index f5db0531..89b0a11f 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -21,6 +21,16 @@ In the future it should also allow users to read statistics about token usage. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long + from itertools import islice from json import dumps, load, loads from sys import maxint @@ -65,18 +75,18 @@ def get_token_count( Retrieve the configured voucher value, in number of tokens, from the given configuration. - :param unicode plugin_name: The plugin name to use to choose a + :param str plugin_name: The plugin name to use to choose a configuration section. :param _Config node_config: See ``from_configuration``. :param int default: The value to return if none is configured. """ - section_name = u"storageclient.plugins.{}".format(plugin_name) + section_name = "storageclient.plugins.{}".format(plugin_name) return int( node_config.get_config( section=section_name, - option=u"default-token-count", + option="default-token-count", default=NUM_TOKENS, ) ) @@ -109,7 +119,7 @@ def from_configuration( :return IZKAPRoot: The root of the resource hierarchy presented by the client side of the plugin. """ - plugin_name = u"privatestorageio-zkapauthz-v1" + plugin_name = "privatestorageio-zkapauthz-v1" if redeemer is None: redeemer = get_redeemer( plugin_name, @@ -219,7 +229,7 @@ def render_POST(self, request): Calculate the price in ZKAPs to store or continue storing files specified sizes. """ - if wrong_content_type(request, u"application/json"): + if wrong_content_type(request, "application/json"): return NOT_DONE_YET application_json(request) @@ -235,8 +245,8 @@ def render_POST(self, request): ) try: - version = body_object[u"version"] - sizes = body_object[u"sizes"] + version = body_object["version"] + sizes = body_object["sizes"] except (TypeError, KeyError): request.setResponseCode(BAD_REQUEST) return dumps( @@ -268,8 +278,8 @@ def render_POST(self, request): price = self._price_calculator.calculate(sizes) return dumps( { - u"price": price, - u"period": self._lease_period, + "price": price, + "period": self._lease_period, } ) @@ -280,14 +290,14 @@ def wrong_content_type(request, required_type): :param request: The request object to check. - :param unicode required_type: The required content-type (eg - ``u"application/json"``). + :param str required_type: The required content-type (eg + ``"application/json"``). :return bool: ``True`` if the content-type is wrong and an error response has been generated. ``False`` otherwise. """ actual_type = request.requestHeaders.getRawHeaders( - u"content-type", + "content-type", [None], )[0] if actual_type != required_type: @@ -303,7 +313,7 @@ def application_json(request): :param twisted.web.iweb.IRequest request: The request to modify. """ - request.responseHeaders.setRawHeaders(u"content-type", [u"application/json"]) + request.responseHeaders.setRawHeaders("content-type", ["application/json"]) class _ProjectVersion(Resource): @@ -339,7 +349,7 @@ def render_GET(self, request): """ application_json(request) state = self._store.backup() - unblinded_tokens = state[u"unblinded-tokens"] + unblinded_tokens = state["unblinded-tokens"] limit = request.args.get(b"limit", [None])[0] if limit is not None: @@ -349,14 +359,14 @@ def render_GET(self, request): return dumps( { - u"total": len(unblinded_tokens), - u"spendable": self._store.count_unblinded_tokens(), - u"unblinded-tokens": list( + "total": len(unblinded_tokens), + "spendable": self._store.count_unblinded_tokens(), + "unblinded-tokens": list( islice( (token for token in unblinded_tokens if token > position), limit ) ), - u"lease-maintenance-spending": self._lease_maintenance_activity(), + "lease-maintenance-spending": self._lease_maintenance_activity(), } ) @@ -365,7 +375,7 @@ def render_POST(self, request): Store some unblinded tokens. """ application_json(request) - unblinded_tokens = load(request.content)[u"unblinded-tokens"] + unblinded_tokens = load(request.content)["unblinded-tokens"] self._store.insert_unblinded_tokens(unblinded_tokens, group_id=0) return dumps({}) @@ -374,8 +384,8 @@ def _lease_maintenance_activity(self): if activity is None: return activity return { - u"when": activity.finished.isoformat(), - u"count": activity.passes_required, + "when": activity.finished.isoformat(), + "count": activity.passes_required, } @@ -401,14 +411,14 @@ def render_PUT(self, request): try: payload = loads(request.content.read()) except Exception: - return bad_request(u"json request body required").render(request) - if payload.keys() != [u"voucher"]: + return bad_request("json request body required").render(request) + if payload.keys() != ["voucher"]: return bad_request( - u"request object must have exactly one key: 'voucher'" + "request object must have exactly one key: 'voucher'" ).render(request) - voucher = payload[u"voucher"] + voucher = payload["voucher"] if not is_syntactic_voucher(voucher): - return bad_request(u"submitted voucher is syntactically invalid").render( + return bad_request("submitted voucher is syntactically invalid").render( request ) @@ -422,7 +432,7 @@ def render_GET(self, request): application_json(request) return dumps( { - u"vouchers": list( + "vouchers": list( self._controller.incorporate_transient_state(voucher).marshal() for voucher in self._store.list() ), @@ -444,12 +454,12 @@ def is_syntactic_voucher(voucher): """ :param voucher: A candidate object to inspect. - :return bool: ``True`` if and only if ``voucher`` is a unicode string + :return bool: ``True`` if and only if ``voucher`` is a text string containing a syntactically valid voucher. This says **nothing** about the validity of the represented voucher itself. A ``True`` result - only means the unicode string can be **interpreted** as a voucher. + only means the string can be **interpreted** as a voucher. """ - if not isinstance(voucher, unicode): + if not isinstance(voucher, str): return False if len(voucher) != 44: # TODO. 44 is the length of 32 bytes base64 encoded. This model @@ -480,7 +490,7 @@ def render_GET(self, request): return self._voucher.to_json() -def bad_request(reason=u"Bad Request"): +def bad_request(reason="Bad Request"): """ :return IResource: A resource which can be rendered to produce a **BAD REQUEST** response. diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index bbe326a2..aeb86184 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -16,7 +16,15 @@ Functionality shared between the storage client and server. """ +from __future__ import absolute_import from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long from base64 import b64encode @@ -50,7 +58,7 @@ class MorePassesRequired(Exception): def _message_maker(label): def make_message(storage_index): - return u"{label} {storage_index}".format( + return "{label} {storage_index}".format( label=label, storage_index=b64encode(storage_index), ) @@ -60,10 +68,10 @@ def make_message(storage_index): # Functions to construct the PrivacyPass request-binding message for pass # construction for different Tahoe-LAFS storage operations. -allocate_buckets_message = _message_maker(u"allocate_buckets") -add_lease_message = _message_maker(u"add_lease") +allocate_buckets_message = _message_maker("allocate_buckets") +add_lease_message = _message_maker("add_lease") slot_testv_and_readv_and_writev_message = _message_maker( - u"slot_testv_and_readv_and_writev" + "slot_testv_and_readv_and_writev" ) # The number of bytes we're willing to store for a lease period for each pass @@ -80,8 +88,8 @@ def get_configured_shares_needed(node_config): """ return int( node_config.get_config( - section=u"client", - option=u"shares.needed", + section="client", + option="shares.needed", default=3, ) ) @@ -96,8 +104,8 @@ def get_configured_shares_total(node_config): """ return int( node_config.get_config( - section=u"client", - option=u"shares.total", + section="client", + option="shares.total", default=10, ) ) @@ -111,11 +119,11 @@ def get_configured_pass_value(node_config): value is read from the **pass-value** option of the ZKAPAuthorizer plugin client section. """ - section_name = u"storageclient.plugins.privatestorageio-zkapauthz-v1" + section_name = "storageclient.plugins.privatestorageio-zkapauthz-v1" return int( node_config.get_config( section=section_name, - option=u"pass-value", + option="pass-value", default=BYTES_PER_PASS, ) ) @@ -125,17 +133,19 @@ def get_configured_allowed_public_keys(node_config): """ Read the set of allowed issuer public keys from the given configuration. """ - section_name = u"storageclient.plugins.privatestorageio-zkapauthz-v1" + section_name = "storageclient.plugins.privatestorageio-zkapauthz-v1" return set( node_config.get_config( section=section_name, - option=u"allowed-public-keys", + option="allowed-public-keys", ) .strip() .split(",") ) +_dict_values = type(dict().values()) + def required_passes(bytes_per_pass, share_sizes): """ Calculate the number of passes that are required to store shares of the @@ -148,9 +158,9 @@ def required_passes(bytes_per_pass, share_sizes): :return int: The number of passes required to cover the storage cost. """ - if not isinstance(share_sizes, list): + if not isinstance(share_sizes, (list, _dict_values)): raise TypeError( - "Share sizes must be a list of integers, got {!r} instead".format( + "Share sizes must be a list (or dict_values) of integers, got {!r} instead".format( share_sizes, ), ) @@ -253,7 +263,7 @@ def get_required_new_passes_for_mutable_write(pass_value, current_sizes, tw_vect """ current_passes = required_passes( pass_value, - current_sizes.values(), + list(current_sizes.values()), ) new_sizes = current_sizes.copy() diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 61b40196..6ce17b93 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -16,6 +16,15 @@ Hypothesis strategies for property testing. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta from urllib import quote @@ -142,7 +151,7 @@ def tahoe_config_texts(storage_client_plugins, shares): def merge_shares(shares, the_rest): for (k, v) in zip(("needed", "happy", "total"), shares): if v is not None: - the_rest["shares." + k] = u"{}".format(v) + the_rest["shares." + k] = "{}".format(v) return the_rest client_section = builds( @@ -151,7 +160,7 @@ def merge_shares(shares, the_rest): fixed_dictionaries( { "storage.plugins": just( - u",".join(storage_client_plugins.keys()), + ",".join(storage_client_plugins.keys()), ), }, ), @@ -186,8 +195,8 @@ def minimal_tahoe_configs(storage_client_plugins=None, shares=just((None, None, :param shares: See ``tahoe_config_texts``. - :return SearchStrategy[unicode]: A strategy that builds unicode strings - which are Tahoe-LAFS configuration file contents. + :return SearchStrategy[str]: A strategy that builds text strings which are + Tahoe-LAFS configuration file contents. """ if storage_client_plugins is None: storage_client_plugins = {} @@ -207,9 +216,9 @@ def node_nicknames(): alphabet=characters( blacklist_categories={ # Surrogates - u"Cs", + "Cs", # Unnamed and control characters - u"Cc", + "Cc", }, ), ) @@ -241,23 +250,23 @@ def server_configurations(signing_key_path): """ Build configuration values for the server-side plugin. - :param unicode signing_key_path: A value to insert for the + :param str signing_key_path: A value to insert for the **ristretto-signing-key-path** item. """ return one_of( fixed_dictionaries( { - u"pass-value": + "pass-value": # The configuration is ini so everything is always a byte string! - integers(min_value=1).map(bytes), + integers(min_value=1).map(lambda v: u"{}".format(v).encode("ascii")), } ), just({}), ).map( lambda config: config.update( { - u"ristretto-issuer-root-url": u"https://issuer.example.invalid/", - u"ristretto-signing-key-path": signing_key_path.path, + "ristretto-issuer-root-url": "https://issuer.example.invalid/", + "ristretto-signing-key-path": signing_key_path.path, } ) or config, @@ -294,8 +303,8 @@ def merge( allowed_public_keys, ): config = { - u"default-token-count": u"32", - u"allowed-public-keys": u",".join(allowed_public_keys), + "default-token-count": "32", + "allowed-public-keys": ",".join(allowed_public_keys), } config.update(extra_configuration) return config @@ -314,8 +323,8 @@ def client_ristrettoredeemer_configurations(): return zkapauthz_configuration( just( { - u"ristretto-issuer-root-url": u"https://issuer.example.invalid/", - u"redeemer": u"ristretto", + "ristretto-issuer-root-url": "https://issuer.example.invalid/", + "redeemer": "ristretto", } ) ) @@ -351,10 +360,10 @@ def share_a_key(allowed_keys): extra_config = lease_configs.map( lambda config: config.update( { - u"redeemer": u"dummy", + "redeemer": "dummy", # Pick out one of the allowed public keys so that the dummy # appears to produce usable tokens. - u"issuer-public-key": next(iter(allowed_keys)), + "issuer-public-key": next(iter(allowed_keys)), } ) or config, @@ -382,7 +391,7 @@ def client_doublespendredeemer_configurations(default_token_counts=token_counts( return zkapauthz_configuration( just( { - u"redeemer": u"double-spend", + "redeemer": "double-spend", } ) ) @@ -395,7 +404,7 @@ def client_unpaidredeemer_configurations(): return zkapauthz_configuration( just( { - u"redeemer": u"unpaid", + "redeemer": "unpaid", } ) ) @@ -408,7 +417,7 @@ def client_nonredeemer_configurations(): return zkapauthz_configuration( just( { - u"redeemer": u"non", + "redeemer": "non", } ) ) @@ -421,8 +430,8 @@ def client_errorredeemer_configurations(details): return zkapauthz_configuration( just( { - u"redeemer": u"error", - u"details": details, + "redeemer": "error", + "details": details, } ) ) @@ -492,14 +501,14 @@ def direct_tahoe_configs( """ config_texts = minimal_tahoe_configs( { - u"privatestorageio-zkapauthz-v1": zkapauthz_v1_configuration, + "privatestorageio-zkapauthz-v1": zkapauthz_v1_configuration, }, shares, ) return config_texts.map( lambda config_text: config_from_string( - u"/dev/null/illegal", - u"", + "/dev/null/illegal", + "", config_text.encode("utf-8"), ), ) @@ -845,7 +854,7 @@ def bytes_for_share(sharenum, size): given share number """ if 0 <= sharenum <= 255: - return (unichr(sharenum) * size).encode("latin-1") + return (chr(sharenum) * size).encode("latin-1") raise ValueError("Sharenum must be between 0 and 255 inclusive.") @@ -949,7 +958,7 @@ def announcements(): """ return just( { - u"ristretto-issuer-root-url": u"https://issuer.example.invalid/", + "ristretto-issuer-root-url": "https://issuer.example.invalid/", } ) @@ -977,7 +986,7 @@ class _DirectoryNode(object): _storage_index = attr.ib() _children = attr.ib() - def list(self): + def list(self): # noqa: F811 return succeed(self._children) def get_storage_index(self): diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 7e3e9abf..26f0786a 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -18,6 +18,13 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import datetime from io import BytesIO @@ -110,7 +117,7 @@ vouchers, ) -TRANSIENT_ERROR = u"something went wrong, who knows what" +TRANSIENT_ERROR = "something went wrong, who knows what" # Helper to work-around https://github.com/twisted/treq/issues/161 def uncooperator(started=True): @@ -137,7 +144,7 @@ def is_not_json(bytestring): def not_vouchers(): """ - Builds unicode strings which are not legal vouchers. + Builds text strings which are not legal vouchers. """ return one_of( text().filter( @@ -147,7 +154,7 @@ def not_vouchers(): # Turn a valid voucher into a voucher that is invalid only by # containing a character from the base64 alphabet in place of one # from the urlsafe-base64 alphabet. - lambda voucher: u"/" + lambda voucher: "/" + voucher[1:], ), ) @@ -155,7 +162,7 @@ def not_vouchers(): def is_urlsafe_base64(text): """ - :param unicode text: A candidate unicode string to inspect. + :param str text: A candidate text string to inspect. :return bool: ``True`` if and only if ``text`` is urlsafe-base64 encoded """ @@ -174,13 +181,13 @@ def invalid_bodies(): # The wrong key but the right kind of value. fixed_dictionaries( { - u"some-key": vouchers(), + "some-key": vouchers(), } ).map(dumps), # The right key but the wrong kind of value. fixed_dictionaries( { - u"voucher": one_of( + "voucher": one_of( integers(), not_vouchers(), ), @@ -242,7 +249,7 @@ def authorized_request(api_auth_token, agent, method, uri, headers=None, data=No else: headers = Headers(headers) headers.setRawHeaders( - u"authorization", + "authorization", [b"tahoe-lafs {}".format(api_auth_token)], ) return agent.request( @@ -323,24 +330,24 @@ def test_get_token_count(self, token_count): ``get_token_count`` returns the integer value of the ``default-token-count`` item from the given configuration object. """ - plugin_name = u"hello-world" + plugin_name = "hello-world" if token_count is None: expected_count = NUM_TOKENS token_config = {} else: expected_count = token_count - token_config = {u"default-token-count": u"{}".format(expected_count)} + token_config = {"default-token-count": "{}".format(expected_count)} config_text = config_string_from_sections( [ { - u"storageclient.plugins." + plugin_name: token_config, + "storageclient.plugins." + plugin_name: token_config, } ] ) node_config = config_from_string( self.useFixture(TempDir()).join(b"tahoe"), - u"tub.port", + "tub.port", config_text.encode("utf-8"), ) self.assertThat( @@ -492,7 +499,7 @@ def test_post(self, get_config, api_auth_token, voucher, unblinded_tokens): data = BytesIO( dumps( { - u"unblinded-tokens": list( + "unblinded-tokens": list( token.unblinded_token.decode("ascii") for token in unblinded_tokens ) @@ -514,7 +521,7 @@ def test_post(self, get_config, api_auth_token, voucher, unblinded_tokens): ), ) - stored_tokens = root.controller.store.backup()[u"unblinded-tokens"] + stored_tokens = root.controller.store.backup()["unblinded-tokens"] self.assertThat( stored_tokens, @@ -560,8 +567,8 @@ def test_get(self, get_config, api_auth_token, voucher, extra_tokens): b"http://127.0.0.1/unblinded-token", ) self.addDetail( - u"requesting result", - text_content(u"{}".format(vars(requesting.result))), + "requesting result", + text_content("{}".format(vars(requesting.result))), ) self.assertThat( requesting, @@ -608,8 +615,8 @@ def test_get_limit(self, get_config, api_auth_token, voucher, extra_tokens, limi b"http://127.0.0.1/unblinded-token?limit={}".format(limit), ) self.addDetail( - u"requesting result", - text_content(u"{}".format(vars(requesting.result))), + "requesting result", + text_content("{}".format(vars(requesting.result))), ) self.assertThat( requesting, @@ -663,8 +670,8 @@ def test_get_position( ), ) self.addDetail( - u"requesting result", - text_content(u"{}".format(vars(requesting.result))), + "requesting result", + text_content("{}".format(vars(requesting.result))), ) self.assertThat( requesting, @@ -674,7 +681,7 @@ def test_get_position( AllMatch( MatchesAll( GreaterThan(position), - IsInstance(unicode), + IsInstance(str), ), ), matches_lease_maintenance_spending(), @@ -715,7 +722,7 @@ def get_tokens(): ) d.addCallback(readBody) d.addCallback( - lambda body: loads(body)[u"unblinded-tokens"], + lambda body: loads(body)["unblinded-tokens"], ) return d @@ -756,7 +763,7 @@ def check_tokens(before_and_after): succeeded( MatchesPredicate( check_tokens, - u"initial, after (%s): initial[1:] != after", + "initial, after (%s): initial[1:] != after", ), ), ) @@ -803,7 +810,7 @@ def test_latest_lease_maintenance_spending( ) d.addCallback(readBody) d.addCallback( - lambda body: loads(body)[u"lease-maintenance-spending"], + lambda body: loads(body)["lease-maintenance-spending"], ) self.assertThat( d, @@ -845,10 +852,10 @@ def succeeded_with_unblinded_tokens_with_matcher( succeeded( ContainsDict( { - u"total": Equals(all_token_count), - u"spendable": match_spendable_token_count, - u"unblinded-tokens": match_unblinded_tokens, - u"lease-maintenance-spending": match_lease_maint_spending, + "total": Equals(all_token_count), + "spendable": match_spendable_token_count, + "unblinded-tokens": match_unblinded_tokens, + "lease-maintenance-spending": match_lease_maint_spending, } ), ), @@ -873,7 +880,7 @@ def succeeded_with_unblinded_tokens(all_token_count, returned_token_count): match_spendable_token_count=Equals(all_token_count), match_unblinded_tokens=MatchesAll( HasLength(returned_token_count), - AllMatch(IsInstance(unicode)), + AllMatch(IsInstance(str)), ), match_lease_maint_spending=matches_lease_maintenance_spending(), ) @@ -889,8 +896,8 @@ def matches_lease_maintenance_spending(): Is(None), ContainsDict( { - u"when": matches_iso8601_datetime(), - u"amount": matches_positive_integer(), + "when": matches_iso8601_datetime(), + "amount": matches_positive_integer(), } ), ) @@ -905,11 +912,11 @@ def matches_positive_integer(): def matches_iso8601_datetime(): """ - :return: A matcher which matches unicode strings which can be parsed as an + :return: A matcher which matches text strings which can be parsed as an ISO8601 datetime string. """ return MatchesAll( - IsInstance(unicode), + IsInstance(str), AfterPreprocessing( parse_datetime, lambda d: Always(), @@ -942,7 +949,7 @@ def test_put_voucher(self, get_config, api_auth_token, voucher): ) root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) - data = BytesIO(dumps({u"voucher": voucher.decode("ascii")})) + data = BytesIO(dumps({"voucher": voucher.decode("ascii")})) requesting = authorized_request( api_auth_token, agent, @@ -951,8 +958,8 @@ def test_put_voucher(self, get_config, api_auth_token, voucher): data=data, ) self.addDetail( - u"requesting result", - text_content(u"{}".format(vars(requesting.result))), + "requesting result", + text_content("{}".format(vars(requesting.result))), ) self.assertThat( requesting, @@ -983,8 +990,8 @@ def test_put_invalid_body(self, get_config, api_auth_token, body): data=BytesIO(body), ) self.addDetail( - u"requesting result", - text_content(u"{}".format(vars(requesting.result))), + "requesting result", + text_content("{}".format(vars(requesting.result))), ) self.assertThat( requesting, @@ -1006,7 +1013,7 @@ def test_get_invalid_voucher(self, get_config, api_auth_token, not_voucher): ) root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) - url = u"http://127.0.0.1/voucher/{}".format( + url = "http://127.0.0.1/voucher/{}".format( quote( not_voucher.encode("utf-8"), safe=b"", @@ -1043,7 +1050,7 @@ def test_get_unknown_voucher(self, get_config, api_auth_token, voucher): api_auth_token, agent, b"GET", - u"http://127.0.0.1/voucher/{}".format(voucher).encode("ascii"), + "http://127.0.0.1/voucher/{}".format(voucher).encode("ascii"), ) self.assertThat( requesting, @@ -1235,7 +1242,7 @@ def _test_get_known_voucher( agent, b"PUT", b"http://127.0.0.1/voucher", - data=BytesIO(dumps({u"voucher": voucher.decode("ascii")})), + data=BytesIO(dumps({"voucher": voucher.decode("ascii")})), ) self.assertThat( putting, @@ -1248,7 +1255,7 @@ def _test_get_known_voucher( api_auth_token, agent, b"GET", - u"http://127.0.0.1/voucher/{}".format( + "http://127.0.0.1/voucher/{}".format( quote( voucher.encode("utf-8"), safe=b"", @@ -1292,7 +1299,7 @@ def test_list_vouchers(self, config, api_auth_token, now, vouchers): vouchers, Equals( { - u"vouchers": list( + "vouchers": list( Voucher( number=voucher, expected_tokens=count, @@ -1329,7 +1336,7 @@ def test_list_vouchers_transient_states( vouchers, Equals( { - u"vouchers": list( + "vouchers": list( Voucher( number=voucher, expected_tokens=count, @@ -1362,7 +1369,7 @@ def _test_list_vouchers( note("{} vouchers".format(len(vouchers))) for voucher in vouchers: - data = BytesIO(dumps({u"voucher": voucher.decode("ascii")})) + data = BytesIO(dumps({"voucher": voucher.decode("ascii")})) putting = authorized_request( api_auth_token, agent, @@ -1479,22 +1486,22 @@ def bad_calculate_price_requests(): good_data = fixed_dictionaries( { - u"version": good_version, - u"sizes": good_sizes, + "version": good_version, + "sizes": good_sizes, } ).map(dumps) bad_data_version = fixed_dictionaries( { - u"version": bad_version, - u"sizes": good_sizes, + "version": bad_version, + "sizes": good_sizes, } ).map(dumps) bad_data_sizes = fixed_dictionaries( { - u"version": good_version, - u"sizes": bad_sizes, + "version": good_version, + "sizes": bad_sizes, } ).map(dumps) @@ -1638,7 +1645,7 @@ def test_calculated_price(self, encoding_params_and_config, api_auth_token, size b"POST", self.url, headers={b"content-type": [b"application/json"]}, - data=BytesIO(dumps({u"version": 1, u"sizes": sizes})), + data=BytesIO(dumps({"version": 1, "sizes": sizes})), ), succeeded( matches_response( @@ -1648,8 +1655,8 @@ def test_calculated_price(self, encoding_params_and_config, api_auth_token, size loads, Equals( { - u"price": expected_price, - u"period": 60 * 60 * 24 * 31 - min_time_remaining, + "price": expected_price, + "period": 60 * 60 * 24 * 31 - min_time_remaining, } ), ), @@ -1660,8 +1667,8 @@ def test_calculated_price(self, encoding_params_and_config, api_auth_token, size def application_json(): return AfterPreprocessing( - lambda h: h.getRawHeaders(u"content-type"), - Equals([u"application/json"]), + lambda h: h.getRawHeaders("content-type"), + Equals(["application/json"]), ) @@ -1701,8 +1708,8 @@ class _MatchResponse(object): def match(self, response): self._details.update( { - u"code": response.code, - u"headers": response.headers.getAllRawHeaders(), + "code": response.code, + "headers": response.headers.getAllRawHeaders(), } ) return MatchesStructure( diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index e4f08b1a..1dc17948 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -43,6 +43,7 @@ Equals, HasLength, IsInstance, + Is, MatchesAll, MatchesStructure, ) @@ -73,6 +74,7 @@ UnpaidRedeemer, UnrecognizedFailureReason, token_count_for_group, + bracket, ) from ..model import DoubleSpend as model_DoubleSpend from ..model import Error as model_Error @@ -1289,3 +1291,54 @@ def bad_content_type(request): b"Unsupported media type", b"Unsupported media type", ).render(request) + + +class BracketTests(TestCase): + """ + Tests for ``bracket``. + """ + def test_success(self): + """ + ``bracket`` calls ``first`` then ``between`` then ``last`` and returns a + ``Deferred`` that fires with the result of ``between``. + """ + result = object() + actions = [] + first = partial(actions.append, "first") + def between(): + actions.append("between") + return result + last = partial(actions.append, "last") + self.assertThat( + bracket(first, last, between), + succeeded( + Is(result), + ), + ) + self.assertThat( + actions, + Equals(["first", "between", "last"]), + ) + + def test_failure(self): + """ + ``bracket`` calls ``first`` then ``between`` then ``last`` and returns a + ``Deferred`` that fires with the failure result of ``between``. + """ + class SomeException(Exception): + pass + actions = [] + first = partial(actions.append, "first") + def between(): + actions.append("between") + raise SomeException() + last = partial(actions.append, "last") + self.assertThat( + bracket(first, last, between), + failed( + AfterPreprocessing( + lambda failure: failure.value, + IsInstance(SomeException), + ), + ), + ) diff --git a/src/_zkapauthorizer/tests/test_model.py b/src/_zkapauthorizer/tests/test_model.py index 93605cd8..6c0e360c 100644 --- a/src/_zkapauthorizer/tests/test_model.py +++ b/src/_zkapauthorizer/tests/test_model.py @@ -269,7 +269,7 @@ def test_uncreateable_store_directory(self, get_config, now): ), Raises( AfterPreprocessing( - lambda (type, exc, tb): exc, + lambda exc_info: exc_info[1], MatchesAll( IsInstance(StoreOpenError), MatchesStructure( diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index f300d3a6..56c63aa6 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -17,6 +17,13 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import timedelta from functools import partial @@ -99,7 +106,7 @@ vouchers, ) -SIGNING_KEY_PATH = FilePath(__file__).sibling(u"testing-signing.key") +SIGNING_KEY_PATH = FilePath(__file__).sibling("testing-signing.key") def get_rref(interface=None): @@ -281,12 +288,12 @@ def test_metrics_written(self, metrics_interval, clock): and an interval how often to do so, test that metrics are actually written there after the configured interval. """ - metrics_path = self.useFixture(TempDir()).join(u"metrics") + metrics_path = self.useFixture(TempDir()).join("metrics") configuration = { - u"prometheus-metrics-path": metrics_path, - u"prometheus-metrics-interval": str(int(metrics_interval.total_seconds())), - u"ristretto-issuer-root-url": "foo", - u"ristretto-signing-key-path": SIGNING_KEY_PATH.path, + "prometheus-metrics-path": metrics_path, + "prometheus-metrics-interval": str(int(metrics_interval.total_seconds())), + "ristretto-issuer-root-url": "foo", + "ristretto-signing-key-path": SIGNING_KEY_PATH.path, } announceable = extract_result( storage_server.get_storage_server( @@ -341,8 +348,8 @@ def read_metrics(path): tahoe_configs_with_mismatched_issuer = minimal_tahoe_configs( { - u"privatestorageio-zkapauthz-v1": just( - {u"ristretto-issuer-root-url": u"https://another-issuer.example.invalid/"} + "privatestorageio-zkapauthz-v1": just( + {"ristretto-issuer-root-url": "https://another-issuer.example.invalid/"} ), } ) @@ -401,8 +408,8 @@ def test_mismatched_ristretto_issuer(self, config_text, announcement): # switch to an io.StringIO here. config_text = StringIO() node_config.config.write(config_text) - self.addDetail(u"config", text_content(config_text.getvalue())) - self.addDetail(u"announcement", text_content(unicode(announcement))) + self.addDetail("config", text_content(config_text.getvalue())) + self.addDetail("announcement", text_content(str(announcement))) self.assertThat( lambda: storage_server.get_storage_client( node_config, @@ -521,12 +528,12 @@ def test_unblinded_tokens_spent( # tests, at least until creating a real server doesn't involve so much # complex setup. So avoid using any of the client APIs that make a # remote call ... which is all of them. - pass_group = storage_client._get_passes(u"request binding message", num_passes) + pass_group = storage_client._get_passes("request binding message", num_passes) pass_group.mark_spent() # There should be no unblinded tokens left to extract. self.assertThat( - lambda: storage_client._get_passes(u"request binding message", 1), + lambda: storage_client._get_passes("request binding message", 1), raises(NotEnoughTokens), ) @@ -540,8 +547,8 @@ def test_unblinded_tokens_spent( lambda logged_message: logged_message.message, ContainsDict( { - u"message": Equals(u"request binding message"), - u"count": Equals(num_passes), + "message": Equals("request binding message"), + "count": Equals(num_passes), } ), ), diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index baadd911..2dd7ab91 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -17,6 +17,16 @@ Tests for ``_zkapauthorizer.pricecalculator``. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long + from functools import partial from hypothesis import given From fc5684e2b0726264f577c7cf4b2025b00be718ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:21:17 -0500 Subject: [PATCH 15/59] black --- src/_zkapauthorizer/_storage_server.py | 25 ++++++++++++++++- src/_zkapauthorizer/eliot.py | 25 ++++++++++++++++- src/_zkapauthorizer/model.py | 25 ++++++++++++++++- src/_zkapauthorizer/resource.py | 25 ++++++++++++++++- src/_zkapauthorizer/storage_common.py | 26 +++++++++++++++++- src/_zkapauthorizer/tests/strategies.py | 27 +++++++++++++++++-- .../tests/test_client_resource.py | 25 ++++++++++++++++- src/_zkapauthorizer/tests/test_controller.py | 7 +++++ src/_zkapauthorizer/tests/test_plugin.py | 25 ++++++++++++++++- .../tests/test_pricecalculator.py | 25 ++++++++++++++++- 10 files changed, 225 insertions(+), 10 deletions(-) diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 2d774aca..f81f75bd 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -27,8 +27,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from datetime import timedelta from errno import ENOENT diff --git a/src/_zkapauthorizer/eliot.py b/src/_zkapauthorizer/eliot.py index ccd84727..2adc410b 100644 --- a/src/_zkapauthorizer/eliot.py +++ b/src/_zkapauthorizer/eliot.py @@ -22,8 +22,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from eliot import ActionType, Field, MessageType diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index 57b7cb4a..14306b15 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -23,8 +23,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from past.builtins import long from datetime import datetime diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 89b0a11f..9436b191 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -27,8 +27,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from past.builtins import long from itertools import islice diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index aeb86184..4239ee55 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -22,8 +22,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from past.builtins import long from base64 import b64encode @@ -146,6 +169,7 @@ def get_configured_allowed_public_keys(node_config): _dict_values = type(dict().values()) + def required_passes(bytes_per_pass, share_sizes): """ Calculate the number of passes that are required to store shares of the diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 6ce17b93..1b6528cd 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -22,8 +22,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta @@ -258,7 +281,7 @@ def server_configurations(signing_key_path): { "pass-value": # The configuration is ini so everything is always a byte string! - integers(min_value=1).map(lambda v: u"{}".format(v).encode("ascii")), + integers(min_value=1).map(lambda v: "{}".format(v).encode("ascii")), } ), just({}), diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 26f0786a..6b0d677f 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -23,8 +23,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from datetime import datetime from io import BytesIO diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index 1dc17948..fa15e3cc 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -1297,6 +1297,7 @@ class BracketTests(TestCase): """ Tests for ``bracket``. """ + def test_success(self): """ ``bracket`` calls ``first`` then ``between`` then ``last`` and returns a @@ -1305,9 +1306,11 @@ def test_success(self): result = object() actions = [] first = partial(actions.append, "first") + def between(): actions.append("between") return result + last = partial(actions.append, "last") self.assertThat( bracket(first, last, between), @@ -1325,13 +1328,17 @@ def test_failure(self): ``bracket`` calls ``first`` then ``between`` then ``last`` and returns a ``Deferred`` that fires with the failure result of ``between``. """ + class SomeException(Exception): pass + actions = [] first = partial(actions.append, "first") + def between(): actions.append("between") raise SomeException() + last = partial(actions.append, "last") self.assertThat( bracket(first, last, between), diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 56c63aa6..10f25b43 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -22,8 +22,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from datetime import timedelta from functools import partial diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index 2dd7ab91..b2b39697 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -23,8 +23,31 @@ from __future__ import unicode_literals from future.utils import PY2 + if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from past.builtins import long from functools import partial From ed8e0a0075f47e3370d8749d3d4da85331ca94ba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:21:54 -0500 Subject: [PATCH 16/59] isort --- src/_zkapauthorizer/_storage_server.py | 5 +---- src/_zkapauthorizer/controller.py | 6 ++---- src/_zkapauthorizer/eliot.py | 5 +---- src/_zkapauthorizer/model.py | 7 ++----- src/_zkapauthorizer/resource.py | 7 ++----- src/_zkapauthorizer/storage_common.py | 7 ++----- src/_zkapauthorizer/tests/strategies.py | 5 +---- src/_zkapauthorizer/tests/test_client_resource.py | 5 +---- src/_zkapauthorizer/tests/test_controller.py | 4 ++-- src/_zkapauthorizer/tests/test_plugin.py | 5 +---- src/_zkapauthorizer/tests/test_pricecalculator.py | 7 ++----- 11 files changed, 17 insertions(+), 46 deletions(-) diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index f81f75bd..aebccdcf 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -21,10 +21,7 @@ implemented in ``_storage_client.py``. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index c4f202ee..b30ed608 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -17,12 +17,10 @@ for the client side of the storage plugin. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 + if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/_zkapauthorizer/eliot.py b/src/_zkapauthorizer/eliot.py index 2adc410b..8daabaa7 100644 --- a/src/_zkapauthorizer/eliot.py +++ b/src/_zkapauthorizer/eliot.py @@ -16,10 +16,7 @@ Eliot field, message, and action definitions for ZKAPAuthorizer. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index 14306b15..e6834f11 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -17,10 +17,7 @@ the storage plugin. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 @@ -48,7 +45,6 @@ max, min, ) # noqa: F401 -from past.builtins import long from datetime import datetime from functools import wraps @@ -58,6 +54,7 @@ import attr from aniso8601 import parse_datetime as _parse_datetime +from past.builtins import long from twisted.logger import Logger from twisted.python.filepath import FilePath from zope.interface import Interface, implementer diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 9436b191..556983ed 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -21,10 +21,7 @@ In the future it should also allow users to read statistics about token usage. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 @@ -52,12 +49,12 @@ max, min, ) # noqa: F401 -from past.builtins import long from itertools import islice from json import dumps, load, loads from sys import maxint +from past.builtins import long from twisted.logger import Logger from twisted.web.http import BAD_REQUEST from twisted.web.resource import ErrorPage, IResource, NoResource, Resource diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index 4239ee55..2ddd2a84 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -16,10 +16,7 @@ Functionality shared between the storage client and server. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 @@ -47,11 +44,11 @@ max, min, ) # noqa: F401 -from past.builtins import long from base64 import b64encode import attr +from past.builtins import long from pyutil.mathutil import div_ceil from .eliot import MUTABLE_PASSES_REQUIRED diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 1b6528cd..eef58788 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -16,10 +16,7 @@ Hypothesis strategies for property testing. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 6b0d677f..c9ef7675 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -17,10 +17,7 @@ plugin. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index fa15e3cc..4742a146 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -42,8 +42,8 @@ Always, Equals, HasLength, - IsInstance, Is, + IsInstance, MatchesAll, MatchesStructure, ) @@ -73,8 +73,8 @@ Unpaid, UnpaidRedeemer, UnrecognizedFailureReason, - token_count_for_group, bracket, + token_count_for_group, ) from ..model import DoubleSpend as model_DoubleSpend from ..model import Error as model_Error diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 10f25b43..e3d1f71c 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -16,10 +16,7 @@ Tests for the Tahoe-LAFS plugin. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index b2b39697..ab8990ff 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -17,10 +17,7 @@ Tests for ``_zkapauthorizer.pricecalculator``. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals from future.utils import PY2 @@ -48,12 +45,12 @@ max, min, ) # noqa: F401 -from past.builtins import long from functools import partial from hypothesis import given from hypothesis.strategies import integers, lists, tuples +from past.builtins import long from testtools import TestCase from testtools.matchers import Equals, GreaterThan, IsInstance, MatchesAll From c123df9c5bf41cf613b7ead1e7b128f1bd7e2da5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 14:22:59 -0500 Subject: [PATCH 17/59] add python3.9 to the ci testing matrix --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a7a8f08..b74c07e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,7 +209,8 @@ jobs: nix-build tests.nix \ --argstr hypothesisProfile ci \ --arg collectCoverage true \ - --argstr tahoe-lafs-source << parameters.tahoe-lafs-source >> + --argstr tahoe-lafs-source << parameters.tahoe-lafs-source >> \ + --argstr python python<< parameters.py-version >> - run: name: "Push to Cachix" @@ -251,6 +252,7 @@ workflows: parameters: py-version: - "2.7" + - "3.9" xcode-version: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions From f296f9e6b9de414ff65140cfb9cc7c9cda6c2d6d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Dec 2021 16:17:33 -0500 Subject: [PATCH 18/59] fix all the noqa comments --- src/_zkapauthorizer/_storage_server.py | 4 ++-- src/_zkapauthorizer/controller.py | 24 ++++++++++++++++++- src/_zkapauthorizer/eliot.py | 4 ++-- src/_zkapauthorizer/model.py | 4 ++-- src/_zkapauthorizer/resource.py | 4 ++-- src/_zkapauthorizer/storage_common.py | 4 ++-- src/_zkapauthorizer/tests/strategies.py | 4 ++-- .../tests/test_client_resource.py | 4 ++-- src/_zkapauthorizer/tests/test_plugin.py | 4 ++-- .../tests/test_pricecalculator.py | 4 ++-- 10 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index aebccdcf..5d94598f 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -26,7 +26,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -48,7 +48,7 @@ str, max, min, - ) # noqa: F401 + ) from datetime import timedelta from errno import ENOENT diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index b30ed608..f9536b26 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -22,7 +22,29 @@ from future.utils import PY2 if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 from base64 import b64decode, b64encode from datetime import timedelta diff --git a/src/_zkapauthorizer/eliot.py b/src/_zkapauthorizer/eliot.py index 8daabaa7..82676056 100644 --- a/src/_zkapauthorizer/eliot.py +++ b/src/_zkapauthorizer/eliot.py @@ -21,7 +21,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -43,7 +43,7 @@ str, max, min, - ) # noqa: F401 + ) from eliot import ActionType, Field, MessageType diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index e6834f11..7caa8efa 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -22,7 +22,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -44,7 +44,7 @@ str, max, min, - ) # noqa: F401 + ) from datetime import datetime from functools import wraps diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 556983ed..9dbb51e2 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -26,7 +26,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -48,7 +48,7 @@ str, max, min, - ) # noqa: F401 + ) from itertools import islice from json import dumps, load, loads diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index 2ddd2a84..b847073a 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -21,7 +21,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -43,7 +43,7 @@ str, max, min, - ) # noqa: F401 + ) from base64 import b64encode diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index eef58788..7f29860e 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -21,7 +21,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -43,7 +43,7 @@ str, max, min, - ) # noqa: F401 + ) from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index c9ef7675..f559b3e1 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -22,7 +22,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -44,7 +44,7 @@ str, max, min, - ) # noqa: F401 + ) from datetime import datetime from io import BytesIO diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index e3d1f71c..831f4236 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -21,7 +21,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -43,7 +43,7 @@ str, max, min, - ) # noqa: F401 + ) from datetime import timedelta from functools import partial diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index ab8990ff..b69135bf 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -22,7 +22,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -44,7 +44,7 @@ str, max, min, - ) # noqa: F401 + ) from functools import partial From 71658b3af0c60caad692329d6ebcd4a7aab1e040 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 31 Dec 2021 16:20:03 -0500 Subject: [PATCH 19/59] more packaging fixes --- default.nix | 6 ++++++ setup.cfg | 4 ++++ shell.nix | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 5166f554..18dba97f 100644 --- a/default.nix +++ b/default.nix @@ -53,11 +53,17 @@ in # this another way, dependencies have undetected dependencies, easier # to just use the wheel. collections-extended = "wheel"; + # same as collections-extended + isort = "wheel"; # From nixpkgs or sdist, fails with # cp: cannot stat 'benchmark/': No such file or directory # cp: cannot stat 'tests/': No such file or directory tomli = "wheel"; + + # repo re-org or something? + # find: ‘hypothesis-6.32.1/hypothesis-python’: No such file or directory + hypothesis = "wheel"; }; in rec { diff --git a/setup.cfg b/setup.cfg index 65fd066c..fe4b45cd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,10 @@ install_requires = # install cross-platform. colorama + # Python 3 transition + six + future + [options.extras_require] test = coverage; fixtures; testtools; hypothesis diff --git a/shell.nix b/shell.nix index 5f5d6489..023fc1bd 100644 --- a/shell.nix +++ b/shell.nix @@ -6,7 +6,7 @@ let inherit (tests) pkgs; in pkgs.mkShell { - packages = [ + buildInputs = [ tests.python tests.lint-python pkgs.niv From 4b5824be354bece1fabd158a4d90d98e6891d8f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 31 Dec 2021 16:20:20 -0500 Subject: [PATCH 20/59] changes for a passing test suite on 2.7 and 3.8 --- src/_zkapauthorizer/_json.py | 34 ++++ src/_zkapauthorizer/_plugin.py | 2 +- src/_zkapauthorizer/_storage_client.py | 29 +++- src/_zkapauthorizer/_storage_server.py | 14 +- src/_zkapauthorizer/controller.py | 40 +++-- src/_zkapauthorizer/foolscap.py | 13 +- src/_zkapauthorizer/lease_maintenance.py | 36 ++++- src/_zkapauthorizer/model.py | 39 +++-- src/_zkapauthorizer/resource.py | 12 +- src/_zkapauthorizer/spending.py | 58 +++++-- src/_zkapauthorizer/storage_common.py | 4 +- src/_zkapauthorizer/tests/fixtures.py | 35 ++++- src/_zkapauthorizer/tests/foolscap.py | 2 +- src/_zkapauthorizer/tests/privacypass.py | 4 +- src/_zkapauthorizer/tests/storage_common.py | 35 +++-- src/_zkapauthorizer/tests/strategies.py | 13 +- .../tests/test_client_resource.py | 79 +++++----- src/_zkapauthorizer/tests/test_controller.py | 145 ++++++++++++++++-- src/_zkapauthorizer/tests/test_foolscap.py | 43 ++++-- .../tests/test_lease_maintenance.py | 40 ++++- src/_zkapauthorizer/tests/test_model.py | 51 ++++-- src/_zkapauthorizer/tests/test_plugin.py | 52 ++++--- src/_zkapauthorizer/tests/test_private.py | 34 +++- src/_zkapauthorizer/tests/test_spending.py | 35 ++++- .../tests/test_storage_client.py | 49 ++++-- .../tests/test_storage_protocol.py | 4 +- .../tests/test_storage_server.py | 24 +-- src/_zkapauthorizer/tests/test_strategies.py | 33 +++- 28 files changed, 734 insertions(+), 225 deletions(-) create mode 100644 src/_zkapauthorizer/_json.py diff --git a/src/_zkapauthorizer/_json.py b/src/_zkapauthorizer/_json.py new file mode 100644 index 00000000..ff8c0c20 --- /dev/null +++ b/src/_zkapauthorizer/_json.py @@ -0,0 +1,34 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) + +from six import ensure_binary +from json import dumps as _dumps + +def dumps(o): + return ensure_binary(_dumps(o)) diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index f6105e63..5deab0dd 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -291,7 +291,7 @@ def get_now(): get_now=get_now, ) last_run_path = FilePath( - node_config.get_private_path(b"last-lease-maintenance-run") + node_config.get_private_path(u"last-lease-maintenance-run") ) # Create the service to periodically run the lease maintenance operation. return lease_maintenance_service( diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index 28670659..b47c5b3b 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -20,7 +20,34 @@ implemented in ``_storage_server.py``. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) from functools import partial, wraps diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 5d94598f..6e0ea94e 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -149,13 +149,13 @@ def _is_invalid_pass(cls, message, pass_, signing_key): """ Cryptographically check the validity of a single pass. - :param str message: The shared message for pass validation. + :param bytes message: The shared message for pass validation. :param Pass pass_: The pass to validate. :return bool: ``False`` (invalid) if the pass includes a valid signature, ``True`` (valid) otherwise. """ - assert isinstance(message, str), "message %r not str" % (message,) + assert isinstance(message, bytes), "message %r not bytes" % (message,) assert isinstance(pass_, Pass), "pass %r not a Pass" % (pass_,) try: preimage = TokenPreimage.decode_base64(pass_.preimage) @@ -163,7 +163,7 @@ def _is_invalid_pass(cls, message, pass_, signing_key): unblinded_token = signing_key.rederive_unblinded_token(preimage) verification_key = unblinded_token.derive_verification_key_sha512() invalid_pass = verification_key.invalid_sha512( - proposed_signature, message.encode("utf-8") + proposed_signature, message, ) return invalid_pass except Exception: @@ -175,7 +175,7 @@ def validate_passes(cls, message, passes, signing_key): """ Check all of the given passes for validity. - :param str message: The shared message for pass validation. + :param bytes message: The shared message for pass validation. :param list[bytes] passes: The encoded passes to validate. :param SigningKey signing_key: The signing key to use to check the passes. @@ -315,7 +315,7 @@ def remote_allocate_buckets( storage for immutable shares if they present valid passes. """ validation = _ValidationResult.validate_passes( - allocate_buckets_message(storage_index), + allocate_buckets_message(storage_index).encode("utf-8"), passes, self._signing_key, ) @@ -398,7 +398,7 @@ def remote_add_lease(self, passes, storage_index, *a, **kw): duration of share storage if they present valid passes. """ validation = _ValidationResult.validate_passes( - add_lease_message(storage_index), + add_lease_message(storage_index).encode("utf-8"), passes, self._signing_key, ) @@ -506,7 +506,7 @@ def _slot_testv_and_readv_and_writev( # Check passes for cryptographic validity. validation = _ValidationResult.validate_passes( - slot_testv_and_readv_and_writev_message(storage_index), + slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), passes, self._signing_key, ) diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index f9536b26..e11fd5f8 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -22,7 +22,7 @@ from future.utils import PY2 if PY2: - from future.builtins import ( + from future.builtins import ( # noqa: F401 filter, map, zip, @@ -44,7 +44,7 @@ str, max, min, - ) # noqa: F401 + ) from base64 import b64decode, b64encode from datetime import timedelta @@ -54,6 +54,7 @@ from operator import delitem, setitem from sys import exc_info +from six import ensure_text import attr import challenge_bypass_ristretto from treq import content @@ -105,7 +106,7 @@ class Unpaid(Exception): """ -@attr.s(frozen=True) +@attr.s class UnrecognizedFailureReason(Exception): """ An attempt was made to redeem a voucher and the response contained an unknown reason. @@ -121,15 +122,18 @@ class RedemptionResult(object): """ Contain the results of an attempt to redeem a voucher for ZKAP material. - :ivar list[UnblindedToken] unblinded_tokens: The tokens which resulted - from the redemption. + :ivar unblinded_tokens: The tokens which resulted from the redemption. - :ivar str public_key: The public key which the server proved was - involved in the redemption process. + :ivar public_key: The public key which the server proved was involved in + the redemption process. """ - unblinded_tokens = attr.ib() - public_key = attr.ib() + unblinded_tokens = attr.ib( # type: List[UnblindedToken] + validator=attr.validators.instance_of(list), + ) + public_key = attr.ib( # type: str + validator=attr.validators.instance_of(str), + ) class IRedeemer(Interface): @@ -269,10 +273,10 @@ class ErrorRedeemer(object): @classmethod def make(cls, section_name, node_config, announcement, reactor): - details = node_config.get_config( + details = ensure_text(node_config.get_config( section=section_name, option="details", - ).decode("ascii") + )) return cls(details) def random_tokens_for_voucher(self, voucher, counter, count): @@ -384,7 +388,7 @@ def make(cls, section_name, node_config, announcement, reactor): node_config.get_config( section=section_name, option="issuer-public-key", - ).decode("utf-8"), + ), ) def random_tokens_for_voucher(self, voucher, counter, count): @@ -486,10 +490,10 @@ class RistrettoRedeemer(object): @classmethod def make(cls, section_name, node_config, announcement, reactor): - configured_issuer = node_config.get_config( + configured_issuer = ensure_text(node_config.get_config( section=section_name, option="ristretto-issuer-root-url", - ).decode("ascii") + )) if announcement is not None: # Don't let us talk to a storage server that has a different idea # about who issues ZKAPs. We should lift this limitation (that is, we @@ -531,7 +535,7 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): "redeemVoucher": voucher.number.decode("ascii"), "redeemCounter": counter, "redeemTokens": list( - token.encode_base64() for token in blinded_tokens + ensure_text(token.encode_base64()) for token in blinded_tokens ), } ), @@ -965,7 +969,7 @@ def _redeem_failure(self, voucher, reason): ) self._error[voucher] = model_Error( finished=self.store.now(), - details=reason.getErrorMessage().decode("utf-8", "replace"), + details=ensure_text(reason.getErrorMessage()), ) return False @@ -1053,7 +1057,9 @@ def bracket(first, last, between): except: info = exc_info() yield last() - raise info[0], info[1], info[2] + if PY2: + exec("raise info[0], info[1], info[2]") + raise else: yield last() returnValue(result) diff --git a/src/_zkapauthorizer/foolscap.py b/src/_zkapauthorizer/foolscap.py index 20ba99fd..c2d814ec 100644 --- a/src/_zkapauthorizer/foolscap.py +++ b/src/_zkapauthorizer/foolscap.py @@ -17,14 +17,15 @@ to communicate between storage clients and servers. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals +from future.utils import PY2 import attr from allmydata.interfaces import Offset, RIStorageServer, StorageIndex from foolscap.api import Any, Copyable, DictOf, ListOf, RemoteCopy from foolscap.constraint import ByteStringConstraint from foolscap.remoteinterface import RemoteInterface, RemoteMethodSchema - +from six import ensure_str @attr.s class ShareStat(Copyable, RemoteCopy): @@ -37,7 +38,7 @@ class ShareStat(Copyable, RemoteCopy): lease on this share expires, or None if there is no lease. """ - typeToCopy = copytype = "ShareStat" + typeToCopy = copytype = ensure_str("ShareStat") # To be a RemoteCopy it must be possible to instantiate this with no # arguments. :/ So supply defaults for these attributes. @@ -90,7 +91,7 @@ def add_passes(schema): :return foolscap.remoteinterface.RemoteMethodSchema: A schema like ``schema`` but with one additional required argument. """ - return add_arguments(schema, [(b"passes", _PassList)]) + return add_arguments(schema, [("passes", _PassList)]) def add_arguments(schema, kwargs): @@ -136,7 +137,9 @@ class RIPrivacyPassAuthorizedStorageServer(RemoteInterface): validated is service provided. """ - __remote_name__ = "RIPrivacyPassAuthorizedStorageServer.tahoe.privatestorage.io" + __remote_name__ = ensure_str( + "RIPrivacyPassAuthorizedStorageServer.tahoe.privatestorage.io" + ) get_version = RIStorageServer["get_version"] diff --git a/src/_zkapauthorizer/lease_maintenance.py b/src/_zkapauthorizer/lease_maintenance.py index 49adebf5..4916768b 100644 --- a/src/_zkapauthorizer/lease_maintenance.py +++ b/src/_zkapauthorizer/lease_maintenance.py @@ -17,6 +17,35 @@ refresh leases on all shares reachable from a root. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) + from datetime import datetime, timedelta from errno import ENOENT from functools import partial @@ -26,6 +55,7 @@ except ImportError: pass +from six import ensure_binary import attr from allmydata.interfaces import IDirectoryNode, IFilesystemNode from allmydata.util.hashutil import ( @@ -395,7 +425,7 @@ def lease_maintenance_service( """ interval_mean = lease_maint_config.crawl_interval_mean interval_range = lease_maint_config.crawl_interval_range - halfrange = interval_range / 2 + halfrange = interval_range // 2 def sample_interval_distribution(): return timedelta( @@ -507,7 +537,7 @@ def write_time_to_path(path, when): :param datetime when: The datetime to write. """ - path.setContent(when.isoformat()) + path.setContent(ensure_binary(when.isoformat())) def read_time_from_path(path): @@ -526,7 +556,7 @@ def read_time_from_path(path): return None raise else: - return parse_datetime(when) + return parse_datetime(when.decode("ascii")) def visit_storage_indexes_from_root(visitor, get_root_nodes): diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index 7caa8efa..fba2112d 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -48,7 +48,7 @@ from datetime import datetime from functools import wraps -from json import dumps, loads +from json import loads from sqlite3 import OperationalError from sqlite3 import connect as _connect @@ -58,7 +58,9 @@ from twisted.logger import Logger from twisted.python.filepath import FilePath from zope.interface import Interface, implementer +from six import ensure_text +from ._json import dumps from ._base64 import urlsafe_b64decode from .schema import get_schema_upgrades, get_schema_version, run_schema_upgrades from .storage_common import ( @@ -69,16 +71,19 @@ from .validators import greater_than, has_length, is_base64_encoded -def parse_datetime(s, **kw): - """ - Like ``aniso8601.parse_datetime`` but accept str as well. - """ - if isinstance(s, str): - s = s.encode("utf-8") - assert isinstance(s, bytes) - if "delimiter" in kw and isinstance(kw["delimiter"], str): - kw["delimiter"] = kw["delimiter"].encode("utf-8") - return _parse_datetime(s, **kw) +if PY2: + def parse_datetime(s, **kw): + """ + Like ``aniso8601.parse_datetime`` but accept str as well. + """ + if isinstance(s, str): + s = s.encode("utf-8") + assert isinstance(s, bytes) + if "delimiter" in kw and isinstance(kw["delimiter"], str): + kw["delimiter"] = kw["delimiter"].encode("utf-8") + return _parse_datetime(s, **kw) +else: + parse_datetime = _parse_datetime class ILeaseMaintenanceObserver(Interface): @@ -975,7 +980,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "redeeming", - "started": self.started.isoformat(), + "started": ensure_text(self.started.isoformat()), "counter": self.counter, } @@ -1000,7 +1005,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "redeemed", - "finished": self.finished.isoformat(), + "finished": ensure_text(self.finished.isoformat()), "token-count": self.token_count, } @@ -1015,7 +1020,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "double-spend", - "finished": self.finished.isoformat(), + "finished": ensure_text(self.finished.isoformat()), } @@ -1035,7 +1040,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "unpaid", - "finished": self.finished.isoformat(), + "finished": ensure_text(self.finished.isoformat()), } @@ -1056,7 +1061,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "error", - "finished": self.finished.isoformat(), + "finished": ensure_text(self.finished.isoformat()), "details": self.details, } @@ -1204,7 +1209,7 @@ def to_json_v1(self): return { "number": self.number.decode("ascii"), "expected-tokens": self.expected_tokens, - "created": None if self.created is None else self.created.isoformat(), + "created": None if self.created is None else ensure_text(self.created.isoformat()), "state": state, "version": 1, } diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 9dbb51e2..82d787e5 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -51,9 +51,10 @@ ) from itertools import islice -from json import dumps, load, loads -from sys import maxint +from json import load, loads +from sys import maxsize +from six import ensure_str, ensure_binary from past.builtins import long from twisted.logger import Logger from twisted.web.http import BAD_REQUEST @@ -61,6 +62,7 @@ from twisted.web.server import NOT_DONE_YET from zope.interface import Attribute +from ._json import dumps from . import __version__ as _zkapauthorizer_version from ._base64 import urlsafe_b64decode from .config import get_configured_lease_duration @@ -170,7 +172,7 @@ def from_configuration( ) root = create_private_tree( - lambda: node_config.get_private_config(b"api_auth_token"), + lambda: ensure_binary(node_config.get_private_config(ensure_str("api_auth_token"))), authorizationless_resource_tree( store, controller, @@ -373,7 +375,7 @@ def render_GET(self, request): limit = request.args.get(b"limit", [None])[0] if limit is not None: - limit = min(maxint, int(limit)) + limit = min(maxsize, int(limit)) position = request.args.get(b"position", [b""])[0].decode("utf-8") @@ -432,7 +434,7 @@ def render_PUT(self, request): payload = loads(request.content.read()) except Exception: return bad_request("json request body required").render(request) - if payload.keys() != ["voucher"]: + if set(payload) != {"voucher"}: return bad_request( "request object must have exactly one key: 'voucher'" ).render(request) diff --git a/src/_zkapauthorizer/spending.py b/src/_zkapauthorizer/spending.py index 19b96ecb..49983f4e 100644 --- a/src/_zkapauthorizer/spending.py +++ b/src/_zkapauthorizer/spending.py @@ -16,6 +16,35 @@ A module for logic controlling the manner in which ZKAPs are spent. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) + import attr from zope.interface import Attribute, Interface, implementer @@ -101,7 +130,7 @@ class PassGroup(object): """ Track the state of a group of passes intended as payment for an operation. - :ivar unicode _message: The request binding message for this group of + :ivar _message: The request binding message for this group of passes. :ivar IPassFactory _factory: The factory which created this pass group. @@ -109,19 +138,22 @@ class PassGroup(object): :ivar list[Pass] passes: The passes of which this group consists. """ - _message = attr.ib() - _factory = attr.ib() - _tokens = attr.ib() + _message = attr.ib(validator=attr.validators.instance_of(bytes)) # type: bytes + _factory = attr.ib(validator=attr.validators.provides(IPassFactory)) # type: IPassFactory + _tokens = attr.ib(validator=attr.validators.instance_of(list)) # type: List[(UnblinidedToken, Pass)] @property def passes(self): + # type: () -> List[Pass] return list(pass_ for (unblinded_token, pass_) in self._tokens) @property def unblinded_tokens(self): + # type: () -> List[UnblindedToken] return list(unblinded_token for (unblinded_token, pass_) in self._tokens) def split(self, select_indices): + # type: (List[int]) -> (PassGroup, PassGroup) selected = [] unselected = [] for idx, t in enumerate(self._tokens): @@ -135,18 +167,22 @@ def split(self, select_indices): ) def expand(self, by_amount): + # type: (int) -> PassGroup return attr.evolve( self, tokens=self._tokens + self._factory.get(self._message, by_amount)._tokens, ) def mark_spent(self): + # type: () -> None self._factory._mark_spent(self.unblinded_tokens) def mark_invalid(self, reason): + # type: () -> None self._factory._mark_invalid(reason, self.unblinded_tokens) def reset(self): + # tye: () -> None self._factory._reset(self.unblinded_tokens) @@ -158,12 +194,12 @@ class SpendingController(object): attempts when necessary. """ - get_unblinded_tokens = attr.ib() - discard_unblinded_tokens = attr.ib() - invalidate_unblinded_tokens = attr.ib() - reset_unblinded_tokens = attr.ib() + get_unblinded_tokens = attr.ib() # type: (int) -> [UnblindedToken] + discard_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None + invalidate_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None + reset_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None - tokens_to_passes = attr.ib() + tokens_to_passes = attr.ib() # type: (bytes, [UnblindedTokens]) -> [Pass] @classmethod def for_store(cls, tokens_to_passes, store): @@ -179,10 +215,10 @@ def get(self, message, num_passes): unblinded_tokens = self.get_unblinded_tokens(num_passes) passes = self.tokens_to_passes(message, unblinded_tokens) GET_PASSES.log( - message=message, + message=message.decode("utf-8"), count=num_passes, ) - return PassGroup(message, self, zip(unblinded_tokens, passes)) + return PassGroup(message, self, list(zip(unblinded_tokens, passes))) def _mark_spent(self, unblinded_tokens): SPENT_PASSES.log( diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index b847073a..b9c4b2cf 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -55,7 +55,7 @@ from .validators import greater_than -@attr.s(frozen=True, str=True) +@attr.s(str=True) class MorePassesRequired(Exception): """ Storage operations fail with ``MorePassesRequired`` when they are not @@ -80,7 +80,7 @@ def _message_maker(label): def make_message(storage_index): return "{label} {storage_index}".format( label=label, - storage_index=b64encode(storage_index), + storage_index=b64encode(storage_index).decode("utf-8"), ) return make_message diff --git a/src/_zkapauthorizer/tests/fixtures.py b/src/_zkapauthorizer/tests/fixtures.py index e017eadd..2bc62c8e 100644 --- a/src/_zkapauthorizer/tests/fixtures.py +++ b/src/_zkapauthorizer/tests/fixtures.py @@ -16,7 +16,34 @@ Common fixtures to let the test suite focus on application logic. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) from base64 import b64encode @@ -49,9 +76,9 @@ class AnonymousStorageServer(Fixture): clock = attr.ib() def _setUp(self): - self.tempdir = FilePath(self.useFixture(TempDir()).join(b"storage")) + self.tempdir = FilePath(self.useFixture(TempDir()).join(u"storage")) self.storage_server = StorageServer( - self.tempdir.asBytesMode().path, + self.tempdir.asTextMode().path, b"x" * 20, clock=self.clock, ) @@ -75,7 +102,7 @@ class TemporaryVoucherStore(Fixture): def _setUp(self): self.tempdir = self.useFixture(TempDir()) - self.config = self.get_config(self.tempdir.join(b"node"), b"tub.port") + self.config = self.get_config(self.tempdir.join(u"node"), u"tub.port") self.store = VoucherStore.from_node_config( self.config, self.get_now, diff --git a/src/_zkapauthorizer/tests/foolscap.py b/src/_zkapauthorizer/tests/foolscap.py index 3a984bea..e2d1c43f 100644 --- a/src/_zkapauthorizer/tests/foolscap.py +++ b/src/_zkapauthorizer/tests/foolscap.py @@ -121,7 +121,7 @@ def callRemote(self, methname, *args, **kwargs): schema = self._referenceable.getInterface()[methname] if self.check_args: schema.checkAllArgs(args, kwargs, inbound=True) - _check_copyables(list(args) + kwargs.values()) + _check_copyables(list(args) + list(kwargs.values())) result = self._referenceable.doRemoteCall( methname, args, diff --git a/src/_zkapauthorizer/tests/privacypass.py b/src/_zkapauthorizer/tests/privacypass.py index a2c29cc4..0f428773 100644 --- a/src/_zkapauthorizer/tests/privacypass.py +++ b/src/_zkapauthorizer/tests/privacypass.py @@ -30,7 +30,7 @@ def make_passes(signing_key, for_message, random_tokens): :param challenge_bypass_ristretto.SigningKey signing_key: The key to use to sign the passes. - :param unicode for_message: The request-binding message with which to + :param bytes for_message: The request-binding message with which to associate the passes. :param list[challenge_bypass_ristretto.RandomToken] random_tokens: The @@ -62,7 +62,7 @@ def make_passes(signing_key, for_message, random_tokens): for unblinded_signature in unblinded_signatures ) message_signatures = list( - verification_key.sign_sha512(for_message.encode("utf-8")) + verification_key.sign_sha512(for_message) for verification_key in verification_keys ) passes = list( diff --git a/src/_zkapauthorizer/tests/storage_common.py b/src/_zkapauthorizer/tests/storage_common.py index db95a71c..61c061c5 100644 --- a/src/_zkapauthorizer/tests/storage_common.py +++ b/src/_zkapauthorizer/tests/storage_common.py @@ -154,7 +154,7 @@ def get_passes(message, num_passes): def get_passes(message, count, signing_key): """ - :param unicode message: Request-binding message for PrivacyPass. + :param bytes message: Request-binding message for PrivacyPass. :param int count: The number of passes to get. @@ -163,6 +163,7 @@ def get_passes(message, count, signing_key): :return list[Pass]: ``count`` new random passes signed with the given key and bound to the given message. """ + assert isinstance(message, bytes) return make_passes( signing_key, message, @@ -200,33 +201,33 @@ class _PassFactory(object): """ A stateful pass issuer. - :ivar (unicode -> int -> [bytes]) _get_passes: A function for getting - passes. + :ivar _get_passes: A function for getting passes. - :ivar set[int] in_use: All of the passes given out without a confirmed + :ivar in_use: All of the passes given out without a confirmed terminal state. - :ivar dict[int, unicode] invalid: All of the passes given out and returned - using ``IPassGroup.invalid`` mapped to the reason given. + :ivar invalid: All of the passes given out and returned using + ``IPassGroup.invalid`` mapped to the reason given. - :ivar set[int] spent: All of the passes given out and returned via + :ivar spent: All of the passes given out and returned via ``IPassGroup.mark_spent``. - :ivar set[int] issued: All of the passes ever given out. + :ivar issued: All of the passes ever given out. - :ivar list[int] returned: A list of passes which were given out but then - returned via ``IPassGroup.reset``. + :ivar returned: A list of passes which were given out but then returned + via ``IPassGroup.reset``. """ - _get_passes = attr.ib() + _get_passes = attr.ib() # type: (bytes, int) -> List[bytes] - returned = attr.ib(default=attr.Factory(list), init=False) - in_use = attr.ib(default=attr.Factory(set), init=False) - invalid = attr.ib(default=attr.Factory(dict), init=False) - spent = attr.ib(default=attr.Factory(set), init=False) - issued = attr.ib(default=attr.Factory(set), init=False) + returned = attr.ib(default=attr.Factory(list), init=False) # type: List[int] + in_use = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] + invalid = attr.ib(default=attr.Factory(dict), init=False) # type: Dict[int, unicode] + spent = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] + issued = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] def get(self, message, num_passes): + # type: (bytes, int) -> PassGroup passes = [] if self.returned: passes.extend(self.returned[:num_passes]) @@ -235,7 +236,7 @@ def get(self, message, num_passes): passes.extend(self._get_passes(message, num_passes)) self.issued.update(passes) self.in_use.update(passes) - return PassGroup(message, self, zip(passes, passes)) + return PassGroup(message, self, list(zip(passes, passes))) def _clear(self): """ diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 7f29860e..54306ea1 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -47,7 +47,6 @@ from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta -from urllib import quote import attr from allmydata.client import config_from_string @@ -70,6 +69,8 @@ text, tuples, ) +from six.moves.urllib.parse import quote +from six import ensure_binary from twisted.internet.defer import succeed from twisted.internet.task import Clock from twisted.web.test.requesthelper import DummyRequest @@ -554,7 +555,7 @@ def tahoe_configs( def path_setter(config): def set_paths(basedir, portnumfile): - config._basedir = basedir.decode("ascii") + config._basedir = basedir config.portnum_fname = portnumfile return config @@ -757,7 +758,13 @@ def request_paths(): :see: ``requests`` """ - return lists(text().map(lambda x: quote(x.encode("utf-8"), safe=b""))) + def quote_segment(seg): + if PY2: + return quote(seg.encode("utf-8"), safe=b"") + else: + return quote(seg, safe="").encode("utf-8") + + return lists(text().map(quote_segment)) def requests(paths=request_paths()): diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index f559b3e1..471a41a7 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -48,8 +48,8 @@ from datetime import datetime from io import BytesIO -from json import dumps -from urllib import quote +from six.moves.urllib.parse import quote +from six import ensure_binary, ensure_text import attr from allmydata.client import config_from_string @@ -98,6 +98,7 @@ from twisted.web.http_headers import Headers from twisted.web.resource import IResource, getChildForRequest +from .. _json import dumps from .. import __version__ as zkapauthorizer_version from .._base64 import urlsafe_b64decode from ..configutil import config_string_from_sections @@ -164,17 +165,17 @@ def is_not_json(bytestring): def not_vouchers(): """ - Builds text strings which are not legal vouchers. + Builds byte strings which are not legal vouchers. """ return one_of( text().filter( lambda t: (not is_urlsafe_base64(t)), - ), + ).map(lambda t: t.encode("utf-8")), vouchers().map( # Turn a valid voucher into a voucher that is invalid only by # containing a character from the base64 alphabet in place of one # from the urlsafe-base64 alphabet. - lambda voucher: "/" + lambda voucher: b"/" + voucher[1:], ), ) @@ -201,7 +202,7 @@ def invalid_bodies(): # The wrong key but the right kind of value. fixed_dictionaries( { - "some-key": vouchers(), + "some-key": vouchers().map(ensure_text), } ).map(dumps), # The right key but the wrong kind of value. @@ -209,7 +210,7 @@ def invalid_bodies(): { "voucher": one_of( integers(), - not_vouchers(), + not_vouchers().map(ensure_text), ), } ).map(dumps), @@ -270,7 +271,7 @@ def authorized_request(api_auth_token, agent, method, uri, headers=None, data=No headers = Headers(headers) headers.setRawHeaders( "authorization", - [b"tahoe-lafs {}".format(api_auth_token)], + [b"tahoe-lafs " + api_auth_token], ) return agent.request( method, @@ -294,8 +295,8 @@ def get_config_with_api_token(tempdir, get_config, api_auth_token): :param bytes api_auth_token: The HTTP API authorization token to write to the node directory. """ - basedir = tempdir.join(b"tahoe") - config = get_config(basedir, b"tub.port") + basedir = tempdir.join(u"tahoe") + config = get_config(basedir, u"tub.port") add_api_token_to_config( basedir, config, @@ -309,9 +310,9 @@ def add_api_token_to_config(basedir, config, api_auth_token): Create a private directory beneath the given base directory, point the given config at it, and write the given API auth token to it. """ - FilePath(basedir).child(b"private").makedirs() + FilePath(basedir).child(u"private").makedirs() config._basedir = basedir - config.write_private_config(b"api_auth_token", api_auth_token) + config.write_private_config(u"api_auth_token", api_auth_token) class FromConfigurationTests(TestCase): @@ -326,7 +327,7 @@ def test_allowed_public_keys(self, get_config): the public keys found in the configuration. """ tempdir = self.useFixture(TempDir()) - config = get_config(tempdir.join(b"tahoe"), b"tub.port") + config = get_config(tempdir.join("tahoe"), "tub.port") allowed_public_keys = get_configured_allowed_public_keys(config) # root_from_config is just an easier way to call from_configuration @@ -366,7 +367,7 @@ def test_get_token_count(self, token_count): ] ) node_config = config_from_string( - self.useFixture(TempDir()).join(b"tahoe"), + self.useFixture(TempDir()).join("tahoe"), "tub.port", config_text.encode("utf-8"), ) @@ -391,7 +392,7 @@ def test_unauthorized(self, get_config, path): receives a 401 response. """ tempdir = self.useFixture(TempDir()) - config = get_config(tempdir.join(b"tahoe"), b"tub.port") + config = get_config(tempdir.join("tahoe"), "tub.port") root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) requesting = agent.request( @@ -429,7 +430,7 @@ def test_reachable(self, get_config, request): ``from_configuration``. """ tempdir = self.useFixture(TempDir()) - config = get_config(tempdir.join(b"tahoe"), b"tub.port") + config = get_config(tempdir.join("tahoe"), "tub.port") root = root_from_config(config, datetime.now) self.assertThat( getChildForRequest(root, request), @@ -545,7 +546,7 @@ def test_post(self, get_config, api_auth_token, voucher, unblinded_tokens): self.assertThat( stored_tokens, - Equals(list(token.unblinded_token for token in unblinded_tokens)), + Equals(list(token.unblinded_token.decode("ascii") for token in unblinded_tokens)), ) @given( @@ -632,7 +633,7 @@ def test_get_limit(self, get_config, api_auth_token, voucher, extra_tokens, limi api_auth_token, agent, b"GET", - b"http://127.0.0.1/unblinded-token?limit={}".format(limit), + u"http://127.0.0.1/unblinded-token?limit={}".format(limit).encode("utf-8"), ) self.addDetail( "requesting result", @@ -685,9 +686,9 @@ def test_get_position( api_auth_token, agent, b"GET", - b"http://127.0.0.1/unblinded-token?position={}".format( + u"http://127.0.0.1/unblinded-token?position={}".format( quote(position.encode("utf-8"), safe=b""), - ), + ).encode("utf-8"), ) self.addDetail( "requesting result", @@ -1033,11 +1034,11 @@ def test_get_invalid_voucher(self, get_config, api_auth_token, not_voucher): ) root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) - url = "http://127.0.0.1/voucher/{}".format( + url = u"http://127.0.0.1/voucher/{}".format( quote( - not_voucher.encode("utf-8"), + not_voucher, safe=b"", - ).decode("utf-8"), + ), ).encode("ascii") requesting = authorized_request( api_auth_token, @@ -1070,7 +1071,7 @@ def test_get_unknown_voucher(self, get_config, api_auth_token, voucher): api_auth_token, agent, b"GET", - "http://127.0.0.1/voucher/{}".format(voucher).encode("ascii"), + b"http://127.0.0.1/voucher/" + voucher, ) self.assertThat( requesting, @@ -1251,7 +1252,7 @@ def _test_get_known_voucher( to be returned by the ``GET``. """ add_api_token_to_config( - self.useFixture(TempDir()).join(b"tahoe"), + self.useFixture(TempDir()).join("tahoe"), config, api_auth_token, ) @@ -1275,11 +1276,11 @@ def _test_get_known_voucher( api_auth_token, agent, b"GET", - "http://127.0.0.1/voucher/{}".format( + u"http://127.0.0.1/voucher/{}".format( quote( - voucher.encode("utf-8"), - safe=b"", - ).decode("utf-8"), + voucher, + safe=u"", + ), ).encode("ascii"), ) self.assertThat( @@ -1379,7 +1380,7 @@ def _test_list_vouchers( # times between setUp and tearDown. Avoid re-using the same # temporary directory for every Hypothesis iteration because this # test leaves state behind that invalidates future iterations. - self.useFixture(TempDir()).join(b"tahoe"), + self.useFixture(TempDir()).join("tahoe"), config, api_auth_token, ) @@ -1431,9 +1432,10 @@ def mime_types(blacklist=None): """ Build MIME types as b"major/minor" byte strings. - :param set|None blacklist: If not ``None``, MIME types to exclude from the - result. + :param blacklist: If not ``None``, MIME types to + exclude from the result. """ + # type: Optional[Set[unicode]] -> Strategy[unicode] if blacklist is None: blacklist = set() return ( @@ -1442,7 +1444,7 @@ def mime_types(blacklist=None): text(), ) .map( - b"/".join, + u"/".join, ) .filter( lambda content_type: content_type not in blacklist, @@ -1482,7 +1484,7 @@ def bad_calculate_price_requests(): bad_headers = fixed_dictionaries( { b"content-type": mime_types(blacklist={b"application/json"},).map( - lambda content_type: [content_type], + lambda content_type: [content_type.encode("utf-8")], ), } ) @@ -1645,7 +1647,7 @@ def test_calculated_price(self, encoding_params_and_config, api_auth_token, size (encoding_params, min_time_remaining), config = encoding_params_and_config shares_needed, shares_happy, shares_total = encoding_params add_api_token_to_config( - self.useFixture(TempDir()).join(b"tahoe"), + self.useFixture(TempDir()).join("tahoe"), config, api_auth_token, ) @@ -1699,7 +1701,7 @@ def json_content(response): def ok_response(headers=None): - return match_response(OK, headers) + return match_response(OK, headers, phrase=Equals(b"OK")) def not_found_response(headers=None): @@ -1710,12 +1712,13 @@ def bad_request_response(headers=None): return match_response(BAD_REQUEST, headers) -def match_response(code, headers): +def match_response(code, headers, phrase=Always()): if headers is None: headers = Always() return _MatchResponse( code=Equals(code), headers=headers, + phrase=phrase, ) @@ -1723,6 +1726,7 @@ def match_response(code, headers): class _MatchResponse(object): code = attr.ib() headers = attr.ib() + phrase = attr.ib() _details = attr.ib(default=attr.Factory(dict)) def match(self, response): @@ -1735,6 +1739,7 @@ def match(self, response): return MatchesStructure( code=self.code, headers=self.headers, + phrase=self.phrase, ).match(response) def get_details(self): diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index 4742a146..4b4f399b 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -20,9 +20,10 @@ from datetime import datetime, timedelta from functools import partial -from json import dumps, loads +from json import loads import attr +from six import ensure_text from challenge_bypass_ristretto import ( BatchDLEQProof, BlindedToken, @@ -49,7 +50,7 @@ ) from testtools.twistedsupport import failed, has_no_result, succeeded from treq.testing import StubTreq -from twisted.internet.defer import fail +from twisted.internet.defer import fail, succeed from twisted.internet.task import Clock from twisted.python.url import URL from twisted.web.http import BAD_REQUEST, INTERNAL_SERVER_ERROR, UNSUPPORTED_MEDIA_TYPE @@ -58,6 +59,7 @@ from twisted.web.resource import ErrorPage, Resource from zope.interface import implementer +from .._json import dumps from ..controller import ( AlreadySpent, DoubleSpendRedeemer, @@ -794,7 +796,7 @@ def test_good_ristretto_redemption(self, voucher, counter, num_tokens): HasLength(num_tokens), ), public_key=Equals( - PublicKey.from_signing_key(signing_key).encode_base64(), + ensure_text(PublicKey.from_signing_key(signing_key).encode_base64()), ), ), ), @@ -1156,9 +1158,9 @@ def render_POST(self, request): return dumps( { u"success": True, - u"public-key": self.public_key.encode_base64(), - u"signatures": marshaled_signed_tokens, - u"proof": marshaled_proof, + u"public-key": ensure_text(self.public_key.encode_base64()), + u"signatures": list(ensure_text(t) for t in marshaled_signed_tokens), + u"proof": ensure_text(marshaled_proof), } ) @@ -1250,7 +1252,7 @@ def check_redemption_request(request): Verify that the given request conforms to the redemption server's public interface. """ - if request.requestHeaders.getRawHeaders(b"content-type") != ["application/json"]: + if request.requestHeaders.getRawHeaders(b"content-type") != [b"application/json"]: return bad_content_type(request) p = request.content.tell() @@ -1293,10 +1295,15 @@ def bad_content_type(request): ).render(request) -class BracketTests(TestCase): +class _BracketTestMixin: """ Tests for ``bracket``. """ + def wrap_success(self, result): + raise NotImplemented() + + def wrap_failure(self, result): + raise NotImplemented() def test_success(self): """ @@ -1309,7 +1316,7 @@ def test_success(self): def between(): actions.append("between") - return result + return self.wrap_success(result) last = partial(actions.append, "last") self.assertThat( @@ -1337,7 +1344,7 @@ class SomeException(Exception): def between(): actions.append("between") - raise SomeException() + return self.wrap_failure(SomeException()) last = partial(actions.append, "last") self.assertThat( @@ -1349,3 +1356,121 @@ def between(): ), ), ) + self.assertThat( + actions, + Equals(["first", "between", "last"]), + ) + + def test_success_with_failing_last(self): + """ + If the ``between`` action succeeds and the ``last`` action fails then + ``bracket`` fails the same way as the ``last`` action. + """ + + class SomeException(Exception): + pass + + actions = [] + first = partial(actions.append, "first") + def between(): + actions.append("between") + return self.wrap_success(None) + + def last(): + actions.append("last") + return self.wrap_failure(SomeException()) + + self.assertThat( + bracket(first, last, between), + failed( + AfterPreprocessing( + lambda failure: failure.value, + IsInstance(SomeException), + ), + ), + ) + self.assertThat( + actions, + Equals(["first", "between", "last"]), + ) + + def test_failure_with_failing_last(self): + """ + If both the ``between`` and ``last`` actions fail then ``bracket`` fails + the same way as the ``last`` action. + """ + + class SomeException(Exception): + pass + + class AnotherException(Exception): + pass + + actions = [] + first = partial(actions.append, "first") + + def between(): + actions.append("between") + return self.wrap_failure(SomeException()) + + def last(): + actions.append("last") + return self.wrap_failure(AnotherException()) + + self.assertThat( + bracket(first, last, between), + failed( + AfterPreprocessing( + lambda failure: failure.value, + IsInstance(AnotherException), + ), + ), + ) + self.assertThat( + actions, + Equals(["first", "between", "last"]), + ) + + def test_first_failure(self): + """ + If the ``first`` action fails then ``bracket`` fails the same way and + runs neither the ``between`` nor ``last`` actions. + """ + + class SomeException(Exception): + pass + + actions = [] + + def first(): + actions.append("first") + return self.wrap_failure(SomeException()) + + between = partial(actions.append, "between") + last = partial(actions.append, "last") + + self.assertThat( + bracket(first, last, between), + failed( + AfterPreprocessing( + lambda failure: failure.value, + IsInstance(SomeException), + ), + ), + ) + self.assertThat( + actions, + Equals(["first"]), + ) + +class BracketTests(_BracketTestMixin, TestCase): + def wrap_success(self, result): + return result + def wrap_failure(self, exception): + raise exception + +class SynchronousDeferredBracketTests(_BracketTestMixin, TestCase): + def wrap_success(self, result): + return succeed(result) + def wrap_failure(self, exception): + return fail(exception) diff --git a/src/_zkapauthorizer/tests/test_foolscap.py b/src/_zkapauthorizer/tests/test_foolscap.py index 3a313b87..1dee261c 100644 --- a/src/_zkapauthorizer/tests/test_foolscap.py +++ b/src/_zkapauthorizer/tests/test_foolscap.py @@ -16,8 +16,36 @@ Tests for Foolscap-related test helpers. """ -from __future__ import absolute_import - +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +# if PY2: +# from future.builtins import ( # noqa: F401 +# filter, +# map, +# zip, +# ascii, +# chr, +# hex, +# input, +# next, +# oct, +# open, +# pow, +# round, +# super, +# bytes, +# dict, +# list, +# object, +# range, +# str, +# max, +# min, +# ) + +from six import ensure_str from fixtures import Fixture from foolscap.api import Any, RemoteInterface, Violation from foolscap.furl import decode_furl @@ -55,7 +83,7 @@ def whatever_method(arg=Any()): def remote_reference(): tub = Tub() tub.setLocation("127.0.0.1:12345") - url = tub.buildURL(b"efgh") + url = tub.buildURL(ensure_str("efgh")) # Ugh ugh ugh. Skip over the extra correctness checking in # RemoteReferenceTracker.__init__ that requires having a broker by passing @@ -85,12 +113,9 @@ def test_tracker_url(self, ref): """ self.assertThat( ref.tracker.getURL(), - MatchesAll( - IsInstance(bytes), - AfterPreprocessing( - decode_furl, - Always(), - ), + AfterPreprocessing( + decode_furl, + Always(), ), ) diff --git a/src/_zkapauthorizer/tests/test_lease_maintenance.py b/src/_zkapauthorizer/tests/test_lease_maintenance.py index 02e2230c..15d9ca97 100644 --- a/src/_zkapauthorizer/tests/test_lease_maintenance.py +++ b/src/_zkapauthorizer/tests/test_lease_maintenance.py @@ -16,7 +16,34 @@ Tests for ``_zkapauthorizer.lease_maintenance``. """ -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) from datetime import datetime, timedelta @@ -37,6 +64,7 @@ randoms, sets, ) +from six import ensure_binary from testtools import TestCase from testtools.matchers import ( AfterPreprocessing, @@ -284,8 +312,8 @@ def test_initial_interval(self, random, mean): [maintenance_call] = clock.getDelayedCalls() datetime_now = datetime.utcfromtimestamp(clock.seconds()) - low = datetime_now + mean - (range_ / 2) - high = datetime_now + mean + (range_ / 2) + low = datetime_now + mean - (range_ // 2) + high = datetime_now + mean + (range_ // 2) self.assertThat( datetime.utcfromtimestamp(maintenance_call.getTime()), between(low, high), @@ -313,7 +341,7 @@ def test_initial_interval_with_last_run(self, random, clock, mean, since_last_ru # Figure out the absolute last run time. last_run = datetime_now - since_last_run last_run_path = FilePath(self.useFixture(TempDir()).join("last-run")) - last_run_path.setContent(last_run.isoformat()) + last_run_path.setContent(ensure_binary(last_run.isoformat())) service = lease_maintenance_service( dummy_maintain_leases, @@ -331,14 +359,14 @@ def test_initial_interval_with_last_run(self, random, clock, mean, since_last_ru low = datetime_now + max( timedelta(0), - mean - (range_ / 2) - since_last_run, + mean - (range_ // 2) - since_last_run, ) high = max( # If since_last_run is one microsecond (precision of timedelta) # then the range is indivisible. Avoid putting the expected high # below the expected low. low, - datetime_now + mean + (range_ / 2) - since_last_run, + datetime_now + mean + (range_ // 2) - since_last_run, ) note( diff --git a/src/_zkapauthorizer/tests/test_model.py b/src/_zkapauthorizer/tests/test_model.py index 6c0e360c..0102ee70 100644 --- a/src/_zkapauthorizer/tests/test_model.py +++ b/src/_zkapauthorizer/tests/test_model.py @@ -17,7 +17,34 @@ Tests for ``_zkapauthorizer.model``. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +# if PY2: +# from future.builtins import ( # noqa: F401 +# filter, +# map, +# zip, +# ascii, +# chr, +# hex, +# input, +# next, +# oct, +# open, +# pow, +# round, +# super, +# bytes, +# dict, +# list, +# object, +# range, +# str, +# max, +# min, +# ) from datetime import datetime, timedelta from errno import EACCES @@ -141,8 +168,8 @@ def test_add_with_distinct_counters( """ counter_a = counters[0] counter_b = counters[1] - tokens_a = tokens[: len(tokens) / 2] - tokens_b = tokens[len(tokens) / 2 :] + tokens_a = tokens[: len(tokens) // 2] + tokens_b = tokens[len(tokens) // 2 :] store = self.useFixture(TemporaryVoucherStore(get_config, lambda: now)).store # We only have to get the expected_tokens value (len(tokens)) right on @@ -253,13 +280,13 @@ def test_uncreateable_store_directory(self, get_config, now): then ``VoucherStore.from_node_config`` raises ``StoreOpenError``. """ tempdir = self.useFixture(TempDir()) - nodedir = tempdir.join(b"node") + nodedir = tempdir.join(u"node") # Create the node directory without permission to create the # underlying directory. mkdir(nodedir, 0o500) - config = get_config(nodedir, b"tub.port") + config = get_config(nodedir, u"tub.port") self.assertThat( lambda: VoucherStore.from_node_config( @@ -295,9 +322,9 @@ def test_unopenable_store(self, get_config, now): ``VoucherStore.from_node_config`` raises ``StoreOpenError``. """ tempdir = self.useFixture(TempDir()) - nodedir = tempdir.join(b"node") + nodedir = tempdir.join(u"node") - config = get_config(nodedir, b"tub.port") + config = get_config(nodedir, u"tub.port") # Create the underlying database file. store = VoucherStore.from_node_config(config, lambda: now) @@ -358,9 +385,9 @@ def _spend_order_test(self, get_config, voucher_value, public_key, now, data): :return: A three-tuple of (backed up tokens, extracted tokens, inserted tokens). """ tempdir = self.useFixture(TempDir()) - nodedir = tempdir.join(b"node") + nodedir = tempdir.join(u"node") - config = get_config(nodedir, b"tub.port") + config = get_config(nodedir, u"tub.port") # Create the underlying database file. store = VoucherStore.from_node_config(config, lambda: now) @@ -384,14 +411,14 @@ def _spend_order_test(self, get_config, voucher_value, public_key, now, data): while tokens_remaining > 0: to_spend = data.draw(integers(min_value=1, max_value=tokens_remaining)) extracted_tokens.extend( - token.unblinded_token for token in store.get_unblinded_tokens(to_spend) + token.unblinded_token.decode("ascii") for token in store.get_unblinded_tokens(to_spend) ) tokens_remaining -= to_spend return ( backed_up_tokens, extracted_tokens, - list(token.unblinded_token for token in unblinded_tokens), + list(token.unblinded_token.decode("ascii") for token in unblinded_tokens), ) @@ -946,7 +973,7 @@ def store_for_test(testcase, get_config, get_now): :return VoucherStore: A newly created temporary store. """ tempdir = testcase.useFixture(TempDir()) - config = get_config(tempdir.join(b"node"), b"tub.port") + config = get_config(tempdir.join(u"node"), u"tub.port") store = VoucherStore.from_node_config( config, get_now, diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 831f4236..0baa1124 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -48,6 +48,7 @@ from datetime import timedelta from functools import partial from os import makedirs +import os.path from allmydata.client import config_from_string, create_client_from_config from allmydata.interfaces import ( @@ -66,7 +67,8 @@ from hypothesis.strategies import datetimes, just, sampled_from, timedeltas from prometheus_client import Gauge from prometheus_client.parser import text_string_to_metric_families -from StringIO import StringIO +from six.moves import StringIO +from six import ensure_binary from testtools import TestCase from testtools.content import text_content from testtools.matchers import ( @@ -389,8 +391,8 @@ def test_interface(self, get_config, announcement): """ tempdir = self.useFixture(TempDir()) node_config = get_config( - tempdir.join(b"node"), - b"tub.port", + tempdir.join(u"node"), + u"tub.port", ) storage_client = storage_server.get_storage_client( @@ -412,8 +414,8 @@ def test_mismatched_ristretto_issuer(self, config_text, announcement): """ tempdir = self.useFixture(TempDir()) node_config = config_from_string( - tempdir.join(b"node"), - b"tub.port", + tempdir.join(u"node"), + u"tub.port", config_text.encode("utf-8"), ) # On Tahoe-LAFS <1.16, the config is written as bytes. @@ -466,8 +468,8 @@ def test_mismatch_storage_server_furl( """ tempdir = self.useFixture(TempDir()) node_config = get_config( - tempdir.join(b"node"), - b"tub.port", + tempdir.join(u"node"), + u"tub.port", ) storage_client = storage_server.get_storage_client( @@ -516,8 +518,8 @@ def test_unblinded_tokens_spent( """ tempdir = self.useFixture(TempDir()) node_config = get_config( - tempdir.join(b"node"), - b"tub.port", + tempdir.join(u"node"), + u"tub.port", ) store = VoucherStore.from_node_config(node_config, lambda: now) @@ -548,12 +550,12 @@ def test_unblinded_tokens_spent( # tests, at least until creating a real server doesn't involve so much # complex setup. So avoid using any of the client APIs that make a # remote call ... which is all of them. - pass_group = storage_client._get_passes("request binding message", num_passes) + pass_group = storage_client._get_passes(b"request binding message", num_passes) pass_group.mark_spent() # There should be no unblinded tokens left to extract. self.assertThat( - lambda: storage_client._get_passes("request binding message", 1), + lambda: storage_client._get_passes(b"request binding message", 1), raises(NotEnoughTokens), ) @@ -567,7 +569,7 @@ def test_unblinded_tokens_spent( lambda logged_message: logged_message.message, ContainsDict( { - "message": Equals("request binding message"), + "message": Equals(u"request binding message"), "count": Equals(num_passes), } ), @@ -589,8 +591,8 @@ def test_interface(self, get_config): ``get_client_resource`` returns an object that provides ``IResource``. """ tempdir = self.useFixture(TempDir()) - nodedir = tempdir.join(b"node") - config = get_config(nodedir, b"tub.port") + nodedir = tempdir.join(u"node") + config = get_config(nodedir, u"tub.port") self.assertThat( storage_server.get_client_resource( config, @@ -652,20 +654,28 @@ def _create(self, get_config, servers_yaml, rootcap): file, ``False`` otherwise. """ tempdir = self.useFixture(TempDir()) - nodedir = tempdir.join(b"node") - privatedir = tempdir.join(b"node", b"private") + nodedir = tempdir.join(u"node") + privatedir = tempdir.join(u"node", u"private") makedirs(privatedir) - config = get_config(nodedir, b"tub.port") + config = get_config(nodedir, u"tub.port") + + # In Tahoe-LAFS 1.17 write_private_config is broken. It mixes bytes + # and unicode in an os.path.join() call that always fails with a + # TypeError. + def write_private_config(name, value): + privname = os.path.join(config._basedir, u"private", name) + with open(privname, "wb") as f: + f.write(value) if servers_yaml is not None: # Provide it a statically configured server to connect to. - config.write_private_config( - b"servers.yaml", + write_private_config( + u"servers.yaml", servers_yaml, ) if rootcap: config.write_private_config( - b"rootcap", + u"rootcap", b"dddddddd", ) @@ -780,7 +790,7 @@ def test_valid(self, key_bytes): :param bytes key: A base64-encoded Ristretto signing key. """ - p = FilePath(self.useFixture(TempDir()).join(b"key")) + p = FilePath(self.useFixture(TempDir()).join(u"key")) p.setContent(key_bytes) key = load_signing_key(p) self.assertThat(key, IsInstance(SigningKey)) diff --git a/src/_zkapauthorizer/tests/test_private.py b/src/_zkapauthorizer/tests/test_private.py index 568cc1eb..effc6dc7 100644 --- a/src/_zkapauthorizer/tests/test_private.py +++ b/src/_zkapauthorizer/tests/test_private.py @@ -10,6 +10,32 @@ """ from __future__ import absolute_import, division, print_function, unicode_literals +from future.utils import PY2 +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) + from allmydata.test.web.matchers import has_response_code from testtools import TestCase @@ -39,7 +65,9 @@ def setUp(self): def _authorization(self, scheme, value): return Headers( { - "authorization": ["{} {}".format(scheme, value)], + u"authorization": [ + u"{} {}".format(scheme.decode("ascii"), value.decode("ascii")), + ], } ) @@ -60,7 +88,7 @@ def test_wrong_scheme(self): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization("basic", self.token), + headers=self._authorization(b"basic", self.token), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) @@ -73,7 +101,7 @@ def test_wrong_token(self): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, "foo bar"), + headers=self._authorization(SCHEME, b"foo bar"), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) diff --git a/src/_zkapauthorizer/tests/test_spending.py b/src/_zkapauthorizer/tests/test_spending.py index 6833e089..55f9aa92 100644 --- a/src/_zkapauthorizer/tests/test_spending.py +++ b/src/_zkapauthorizer/tests/test_spending.py @@ -16,6 +16,35 @@ Tests for ``_zkapauthorizer.spending``. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) + from hypothesis import given from hypothesis.strategies import data, integers, randoms from testtools import TestCase @@ -62,7 +91,7 @@ def test_get(self, voucher, num_passes, now): store=configless.store, ) - group = pass_factory.get(u"message", num_passes) + group = pass_factory.get(b"message", num_passes) self.assertThat( group, MatchesAll( @@ -97,7 +126,7 @@ def _test_token_group_operation( # Figure out some subset, maybe empty, of passes from the group that # we will try to operate on. group_size = data.draw(integers(min_value=0, max_value=num_passes)) - indices = range(num_passes) + indices = list(range(num_passes)) random.shuffle(indices) spent_indices = indices[:group_size] @@ -106,7 +135,7 @@ def _test_token_group_operation( tokens_to_passes=configless.redeemer.tokens_to_passes, store=configless.store, ) - group = pass_factory.get(u"message", num_passes) + group = pass_factory.get(b"message", num_passes) spent, rest = group.split(spent_indices) operation(spent) diff --git a/src/_zkapauthorizer/tests/test_storage_client.py b/src/_zkapauthorizer/tests/test_storage_client.py index 1075884f..8da2975c 100644 --- a/src/_zkapauthorizer/tests/test_storage_client.py +++ b/src/_zkapauthorizer/tests/test_storage_client.py @@ -16,7 +16,34 @@ Tests for ``_zkapauthorizer._storage_client``. """ -from __future__ import division +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) from functools import partial @@ -180,7 +207,7 @@ def test_success_result(self, num_passes): call_with_passes( lambda group: succeed(result), num_passes, - partial(pass_factory(integer_passes(num_passes)).get, u"message"), + partial(pass_factory(integer_passes(num_passes)).get, b"message"), ), succeeded(Is(result)), ) @@ -197,7 +224,7 @@ def test_failure_result(self, num_passes): call_with_passes( lambda group: fail(result), num_passes, - partial(pass_factory(integer_passes(num_passes)).get, u"message"), + partial(pass_factory(integer_passes(num_passes)).get, b"message"), ), failed( AfterPreprocessing( @@ -220,7 +247,7 @@ def test_passes_issued(self, num_passes): call_with_passes( lambda group: succeed(group.passes), num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), succeeded( Equals( @@ -241,7 +268,7 @@ def test_passes_spent_on_success(self, num_passes): call_with_passes( lambda group: None, num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), succeeded(Always()), ) @@ -261,7 +288,7 @@ def test_passes_returned_on_failure(self, num_passes): call_with_passes( lambda group: fail(Exception("Anything")), num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), failed(Always()), ) @@ -301,7 +328,7 @@ def reject_even_pass_values(group): call_with_passes( reject_even_pass_values, num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), succeeded(Always()), ) @@ -342,7 +369,7 @@ def reject_passes(group): call_with_passes( reject_passes, num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), failed( AfterPreprocessing( @@ -406,7 +433,7 @@ def reject_half_passes(group): # out of passes no matter how many we start with. reject_half_passes, num_passes, - partial(passes.get, u"message"), + partial(passes.get, b"message"), ), failed( AfterPreprocessing( @@ -458,7 +485,7 @@ def test_returned_passes_reused(self, num_passes_a, num_passes_b): ``IPassGroup.reset`` makes passes available to be returned by ``IPassGroup.get`` again. """ - message = u"message" + message = b"message" min_passes = min(num_passes_a, num_passes_b) max_passes = max(num_passes_a, num_passes_b) @@ -486,7 +513,7 @@ def _test_disallowed_transition(self, num_passes, setup_op, invalid_op): :param (IPassGroup -> None) invalid_op: Some follow-up operation to perform with the pass group and to assert raises an exception. """ - message = u"message" + message = b"message" factory = pass_factory(integer_passes(num_passes)) group = factory.get(message, num_passes) setup_op(group) diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index 89654633..ffae2fff 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -235,14 +235,14 @@ def test_rejected_passes_reported( # Make some passes with a key untrusted by the server. bad_passes = get_passes( - allocate_buckets_message(storage_index), + allocate_buckets_message(storage_index).encode("utf-8"), len(bad_pass_indexes), random_signing_key(), ) # Make some passes with a key trusted by the server. good_passes = get_passes( - allocate_buckets_message(storage_index), + allocate_buckets_message(storage_index).encode("utf-8"), num_passes - len(bad_passes), self.signing_key, ) diff --git a/src/_zkapauthorizer/tests/test_storage_server.py b/src/_zkapauthorizer/tests/test_storage_server.py index 7e334d79..b40e8263 100644 --- a/src/_zkapauthorizer/tests/test_storage_server.py +++ b/src/_zkapauthorizer/tests/test_storage_server.py @@ -81,7 +81,7 @@ def test_validation_result(self, valid_count, invalid_passes): ``validate_passes`` returns a ``_ValidationResult`` instance which describes the valid and invalid passes. """ - message = u"hello world" + message = b"hello world" valid_passes = get_passes( message, valid_count, @@ -136,7 +136,7 @@ def test_raise_for(self): AfterPreprocessing( str, Equals( - "MorePassesRequired(valid_count=4, required_count=10, signature_check_failed=frozenset([4]))" + "MorePassesRequired(valid_count=4, required_count=10, signature_check_failed={})".format(str(frozenset([4]))), ), ), ), @@ -243,7 +243,7 @@ def test_allocate_buckets_fails_without_enough_passes(self): renew_secret = b"x" * 32 cancel_secret = b"y" * 32 valid_passes = get_passes( - allocate_buckets_message(storage_index), + allocate_buckets_message(storage_index).encode("utf-8"), required_passes - 1, self.signing_key, ) @@ -345,7 +345,7 @@ def _test_extend_mutable_fails_without_passes( ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index), + slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), required_pass_count, self.signing_key, ) @@ -481,7 +481,7 @@ def test_mutable_new_length_rejected( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index), + slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), required_pass_count, self.signing_key, ) @@ -550,7 +550,7 @@ def test_add_lease_fails_without_passes( # Attempt the lease operation with one fewer pass than is required. passes = get_passes( - add_lease_message(storage_index), + add_lease_message(storage_index).encode("utf-8"), required_count - 1, self.signing_key, ) @@ -614,7 +614,7 @@ def test_mutable_share_sizes( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(slot), + slot_testv_and_readv_and_writev_message(slot).encode("utf-8"), required_pass_count, self.signing_key, ) @@ -678,7 +678,7 @@ def test_mutable_spending_metrics( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index), + slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), num_passes, self.signing_key, ) @@ -739,7 +739,7 @@ def test_mutable_failure_spending_metrics( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index), + slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), num_passes, self.signing_key, ) @@ -820,7 +820,7 @@ def test_immutable_spending_metrics( [size] * len(new_sharenums - existing_sharenums), ) valid_passes = get_passes( - allocate_buckets_message(storage_index), + allocate_buckets_message(storage_index).encode("utf-8"), num_passes, self.signing_key, ) @@ -887,7 +887,7 @@ def test_add_lease_metrics( self.storage_server._pass_value, [allocated_size] * len(sharenums) ) valid_passes = get_passes( - add_lease_message(storage_index), + add_lease_message(storage_index).encode("utf-8"), num_passes, self.signing_key, ) @@ -948,7 +948,7 @@ def test_add_lease_metrics_on_failure( self.storage_server._pass_value, [allocated_size] * len(sharenums) ) valid_passes = get_passes( - add_lease_message(storage_index), + add_lease_message(storage_index).encode("utf-8"), num_passes, self.signing_key, ) diff --git a/src/_zkapauthorizer/tests/test_strategies.py b/src/_zkapauthorizer/tests/test_strategies.py index b046450c..da975705 100644 --- a/src/_zkapauthorizer/tests/test_strategies.py +++ b/src/_zkapauthorizer/tests/test_strategies.py @@ -16,7 +16,34 @@ Tests for our custom Hypothesis strategies. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( # noqa: F401 + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) from allmydata.client import config_from_string from fixtures import TempDir @@ -49,7 +76,7 @@ def test_parses(self, data): ) note(config_text) config_from_string( - tempdir.join(b"tahoe.ini"), - b"tub.port", + tempdir.join(u"tahoe.ini"), + u"tub.port", config_text.encode("utf-8"), ) From 890e67dd39448cfc084881dd2f4cbdff96f3188e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jan 2022 17:05:17 -0500 Subject: [PATCH 21/59] add a test for using the test vector feature and fix it --- src/_zkapauthorizer/_storage_client.py | 21 ++++++- .../tests/test_storage_protocol.py | 57 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index b47c5b3b..ecb785f9 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -450,6 +450,25 @@ def slot_testv_and_readv_and_writev( tw_vectors, r_vector, ): + # type: ( + # Any, + # bytes, + # Tuple[bytes, bytes, bytes], + # Dict[ + # int, + # Tuple[ + # List[ + # Tuple[int, int, bytes, bytes], + # ], + # List[ + # Tuple[int, bytes], + # ], + # Optional[int], + # ], + # ], + # List[Tuple[int, int]], + # ) -> Deferred + # Read operations are free. num_passes = 0 @@ -458,7 +477,7 @@ def slot_testv_and_readv_and_writev( tw_vectors = { sharenum: ( [ - (offset, length, "eq", specimen) + (offset, length, b"eq", specimen) for (offset, length, specimen) in test_vector ], data_vectors, diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index ffae2fff..7e5fb3e6 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -72,6 +72,7 @@ ) from .strategies import bytes_for_share # Not really a strategy... from .strategies import ( + TestAndWriteVectors, lease_cancel_secrets, lease_renew_secrets, posix_timestamps, @@ -79,6 +80,7 @@ sharenum_sets, sharenums, sizes, + slot_data_vectors, slot_test_and_write_vectors_for_shares, storage_indexes, write_enabler_secrets, @@ -1073,6 +1075,61 @@ def write(): ), ) + @given( + storage_index=storage_indexes(), + secrets=tuples( + write_enabler_secrets(), + lease_renew_secrets(), + lease_cancel_secrets(), + ), + sharenum=sharenums(), + data_vector=slot_data_vectors(), + replacement_data_vector=slot_data_vectors(), + ) + def test_test_vectors_match(self, storage_index, secrets, sharenum, data_vector, replacement_data_vector): + """ + If test vectors are given then the write is allowed if they match the + existing data. + """ + empty_test_vector = [] + + def write(tw_vectors): + return self.client.slot_testv_and_readv_and_writev( + storage_index, + secrets=secrets, + tw_vectors=tw_vectors, + r_vector=[], + ) + + def read(sharenum, readv): + d = self.client.slot_readv(storage_index, [sharenum], readv) + d.addCallback(lambda data: data[sharenum]) + return d + + def equal_test_vector(data_vector): + return list( + (offset, len(data), data) + for (offset, data) + in data_vector + ) + + # Create the share + d = write({ + sharenum: (empty_test_vector, data_vector, None), + }) + self.assertThat(d, is_successful_write()) + + # Write some new data to with a correct test vector. We can only be + # sure we know data from the last element of the test vector since + # earlier elements may have been overwritten. + d = write({ + sharenum: (equal_test_vector(data_vector)[-1:], replacement_data_vector, None), + }) + self.assertThat(d, is_successful_write()) + + # Check that the new data is present + assert_read_back_data(self, storage_index, secrets, {sharenum: TestAndWriteVectors(None, replacement_data_vector, None)}) + def assert_read_back_data( self, storage_index, secrets, test_and_write_vectors_for_shares From 4dc239c3494518a53d92f8c1cf0af0f42e7131f1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jan 2022 19:56:31 -0500 Subject: [PATCH 22/59] provide a shell where we can run tests, not just linters --- shell.nix | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/shell.nix b/shell.nix index 023fc1bd..9388b735 100644 --- a/shell.nix +++ b/shell.nix @@ -2,13 +2,55 @@ # work on Nix 2.3. It works with Nix 2.5. I'm not sure about 2.4. { ... }@args: let - tests = import ./tests.nix args; - inherit (tests) pkgs; + privatestorage = import ./. args; + inherit (privatestorage) pkgs mach-nix tahoe-lafs zkapauthorizer; + + pythonVersion = zkapauthorizer.meta.mach-nix.python; + + python = mach-nix.mkPython { + inherit (zkapauthorizer.meta.mach-nix) providers; + python = pythonVersion; + overridesPre = [ + ( + self: super: { + inherit tahoe-lafs; + } + ) + ]; + + requirements = let + py3 = pythonVersion > "python3"; + lint = pkgs.lib.optionalString (builtins.trace py3 py3) '' +# lint +black +isort +flake8 +''; + workarounds = '' +# Mitigate for undetected cryptography dependency +setuptools_rust +# And for tomli +flit_core +''; + testing = '' +coverage +fixtures +testtools +hypothesis +''; + in + '' +${lint} +${workarounds} +${testing} +${zkapauthorizer.requirements} +''; + }; in pkgs.mkShell { + PYTHONDONTWRITEBYTECODE = "1"; + buildInputs = [ - tests.python - tests.lint-python - pkgs.niv + python ]; } From 61dc80d9e58065ffaaa30061ebf830a041addf61 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 11:52:44 -0500 Subject: [PATCH 23/59] strip explicit Python 2 support from the Nix packaging --- default.nix | 28 +++++----------------------- shell.nix | 48 ++++++++++++++---------------------------------- tests.nix | 2 +- 3 files changed, 20 insertions(+), 58 deletions(-) diff --git a/default.nix b/default.nix index 18dba97f..48bdab38 100644 --- a/default.nix +++ b/default.nix @@ -1,21 +1,9 @@ let sources = import nix/sources.nix; - - # nixpkgs 21.11 packages a version of pkgconfig that is broken on Python 2.7 - # so supply our own. mach-nix depends on this to set up its own build - # environment and cannot discover a working version from pypi the way it - # works for other dependencies. - fixPkgconfig = self: super: { - python27 = super.python27.override { - packageOverrides = python-self: python-super: { - pkgconfig = python-super.pythonPackages.callPackage ./pkgconfig.nix {}; - }; - }; - }; in -{ pkgs ? import sources.release2111 { overlays = [ fixPkgconfig ]; } +{ pkgs ? import sources.release2111 { } , pypiData ? sources.pypi-deps-db -, python ? "python27" +, python ? "python39" , mach-nix ? import sources.mach-nix { inherit pkgs pypiData python; } , tahoe-lafs-source ? "tahoe-lafs" , tahoe-lafs-repo ? sources.${tahoe-lafs-source} @@ -83,18 +71,12 @@ in version = "1.17.0.post999"; # See https://github.com/DavHau/mach-nix/issues/190 requirementsExtra = - let - foolscap-version = - if python <= "python30" - then "foolscap == 0.13.1" - else "foolscap >= 21.7.0"; - in - '' + '' # See https://github.com/DavHau/mach-nix/issues/190 pyrsistent < 0.17 configparser eliot - ${foolscap-version} + foolscap >= 21.7.0 # undetected cryptography build dependency # https://github.com/DavHau/mach-nix/issues/305 @@ -102,7 +84,7 @@ in # undetected tomli build dependency # probably same underlying cause as cryptography issue flit_core - ''; + ''; postPatch = '' cat > src/allmydata/_version.py < "python3"; - lint = pkgs.lib.optionalString (builtins.trace py3 py3) '' -# lint -black -isort -flake8 -''; - workarounds = '' -# Mitigate for undetected cryptography dependency -setuptools_rust -# And for tomli -flit_core -''; - testing = '' -coverage -fixtures -testtools -hypothesis -''; - in + requirements = '' -${lint} -${workarounds} -${testing} -${zkapauthorizer.requirements} -''; + ${builtins.readFile ./requirements/test.in} + ${zkapauthorizer.requirements} + ''; }; in pkgs.mkShell { + # Avoid leaving .pyc all over the source tree when manually triggering tests + # runs. PYTHONDONTWRITEBYTECODE = "1"; buildInputs = [ - python + # Provide the linting tools for interactive usage. + lint-python + # Supply all of the runtime and testing dependencies. + python-env ]; } diff --git a/tests.nix b/tests.nix index f51d5727..0d7f6a77 100644 --- a/tests.nix +++ b/tests.nix @@ -70,5 +70,5 @@ let ''; in { - inherit pkgs python lint-python tests; + inherit privatestorage lint-python tests; } From 4c29995ca14413e2cb801d5e4cd389429537d578 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 11:55:48 -0500 Subject: [PATCH 24/59] Drop Python 2 from the CI matrix --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b74c07e7..59b22d6f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -251,7 +251,6 @@ workflows: matrix: parameters: py-version: - - "2.7" - "3.9" xcode-version: From bf7c3126b52699c4bb03cf25dec7016f95c75b58 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 12:11:40 -0500 Subject: [PATCH 25/59] commit to Python 3 for json.dumps --- src/_zkapauthorizer/_json.py | 50 +++++++------------ src/_zkapauthorizer/controller.py | 5 +- src/_zkapauthorizer/model.py | 4 +- src/_zkapauthorizer/resource.py | 20 ++++---- .../tests/test_client_resource.py | 24 ++++----- src/_zkapauthorizer/tests/test_controller.py | 8 +-- 6 files changed, 50 insertions(+), 61 deletions(-) diff --git a/src/_zkapauthorizer/_json.py b/src/_zkapauthorizer/_json.py index ff8c0c20..898f6b56 100644 --- a/src/_zkapauthorizer/_json.py +++ b/src/_zkapauthorizer/_json.py @@ -1,34 +1,22 @@ -from __future__ import absolute_import, division, print_function, unicode_literals +# Copyright 2022 PrivateStorage.io, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - -from six import ensure_binary from json import dumps as _dumps +from typing import Any -def dumps(o): - return ensure_binary(_dumps(o)) +def dumps_utf8(o: Any) -> bytes: + """ + Serialize an object to a UTF-8-encoded JSON byte string. + """ + return _dumps(o).encode("utf-8") diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index e11fd5f8..4be61aa6 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -50,7 +50,7 @@ from datetime import timedelta from functools import partial from hashlib import sha256 -from json import dumps, loads +from json import loads from operator import delitem, setitem from sys import exc_info @@ -67,6 +67,7 @@ from twisted.web.client import Agent from zope.interface import Interface, implementer +from ._json import dumps_utf8 from ._base64 import urlsafe_b64decode from ._stack import less_limited_stack from .model import Error as model_Error @@ -530,7 +531,7 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): blinded_tokens = list(token.blind() for token in random_tokens) response = yield self._treq.post( self._api_root.child("v1", "redeem").to_text(), - dumps( + dumps_utf8( { "redeemVoucher": voucher.number.decode("ascii"), "redeemCounter": counter, diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index fba2112d..0aec9d38 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -60,7 +60,7 @@ from zope.interface import Interface, implementer from six import ensure_text -from ._json import dumps +from ._json import dumps_utf8 from ._base64 import urlsafe_b64decode from .schema import get_schema_upgrades, get_schema_version, run_schema_upgrades from .storage_common import ( @@ -1199,7 +1199,7 @@ def from_json_v1(cls, values): ) def to_json(self): - return dumps(self.marshal()) + return dumps_utf8(self.marshal()) def marshal(self): return self.to_json_v1() diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 82d787e5..b2b97520 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -62,7 +62,7 @@ from twisted.web.server import NOT_DONE_YET from zope.interface import Attribute -from ._json import dumps +from ._json import dumps_utf8 from . import __version__ as _zkapauthorizer_version from ._base64 import urlsafe_b64decode from .config import get_configured_lease_duration @@ -260,7 +260,7 @@ def render_POST(self, request): body_object = loads(payload) except ValueError: request.setResponseCode(BAD_REQUEST) - return dumps( + return dumps_utf8( { "error": "could not parse request body", } @@ -271,7 +271,7 @@ def render_POST(self, request): sizes = body_object["sizes"] except (TypeError, KeyError): request.setResponseCode(BAD_REQUEST) - return dumps( + return dumps_utf8( { "error": "could not read `version` and `sizes` properties", } @@ -279,7 +279,7 @@ def render_POST(self, request): if version != 1: request.setResponseCode(BAD_REQUEST) - return dumps( + return dumps_utf8( { "error": "did not find required version number 1 in request", } @@ -289,7 +289,7 @@ def render_POST(self, request): isinstance(size, (int, long)) and size >= 0 for size in sizes ): request.setResponseCode(BAD_REQUEST) - return dumps( + return dumps_utf8( { "error": "did not find required positive integer sizes list in request", } @@ -298,7 +298,7 @@ def render_POST(self, request): application_json(request) price = self._price_calculator.calculate(sizes) - return dumps( + return dumps_utf8( { "price": price, "period": self._lease_period, @@ -345,7 +345,7 @@ class _ProjectVersion(Resource): def render_GET(self, request): application_json(request) - return dumps( + return dumps_utf8( { "version": _zkapauthorizer_version, } @@ -379,7 +379,7 @@ def render_GET(self, request): position = request.args.get(b"position", [b""])[0].decode("utf-8") - return dumps( + return dumps_utf8( { "total": len(unblinded_tokens), "spendable": self._store.count_unblinded_tokens(), @@ -399,7 +399,7 @@ def render_POST(self, request): application_json(request) unblinded_tokens = load(request.content)["unblinded-tokens"] self._store.insert_unblinded_tokens(unblinded_tokens, group_id=0) - return dumps({}) + return dumps_utf8({}) def _lease_maintenance_activity(self): activity = self._store.get_latest_lease_maintenance_activity() @@ -452,7 +452,7 @@ def render_PUT(self, request): def render_GET(self, request): application_json(request) - return dumps( + return dumps_utf8( { "vouchers": list( self._controller.incorporate_transient_state(voucher).marshal() diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 471a41a7..63acf4c4 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -98,7 +98,7 @@ from twisted.web.http_headers import Headers from twisted.web.resource import IResource, getChildForRequest -from .. _json import dumps +from .. _json import dumps_utf8 from .. import __version__ as zkapauthorizer_version from .._base64 import urlsafe_b64decode from ..configutil import config_string_from_sections @@ -204,7 +204,7 @@ def invalid_bodies(): { "some-key": vouchers().map(ensure_text), } - ).map(dumps), + ).map(dumps_utf8), # The right key but the wrong kind of value. fixed_dictionaries( { @@ -213,7 +213,7 @@ def invalid_bodies(): not_vouchers().map(ensure_text), ), } - ).map(dumps), + ).map(dumps_utf8), # Not even JSON binary().filter(is_not_json), ) @@ -518,7 +518,7 @@ def test_post(self, get_config, api_auth_token, voucher, unblinded_tokens): root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) data = BytesIO( - dumps( + dumps_utf8( { "unblinded-tokens": list( token.unblinded_token.decode("ascii") @@ -970,7 +970,7 @@ def test_put_voucher(self, get_config, api_auth_token, voucher): ) root = root_from_config(config, datetime.now) agent = RequestTraversalAgent(root) - data = BytesIO(dumps({"voucher": voucher.decode("ascii")})) + data = BytesIO(dumps_utf8({"voucher": voucher.decode("ascii")})) requesting = authorized_request( api_auth_token, agent, @@ -1263,7 +1263,7 @@ def _test_get_known_voucher( agent, b"PUT", b"http://127.0.0.1/voucher", - data=BytesIO(dumps({"voucher": voucher.decode("ascii")})), + data=BytesIO(dumps_utf8({"voucher": voucher.decode("ascii")})), ) self.assertThat( putting, @@ -1390,7 +1390,7 @@ def _test_list_vouchers( note("{} vouchers".format(len(vouchers))) for voucher in vouchers: - data = BytesIO(dumps({"voucher": voucher.decode("ascii")})) + data = BytesIO(dumps_utf8({"voucher": voucher.decode("ascii")})) putting = authorized_request( api_auth_token, agent, @@ -1511,26 +1511,26 @@ def bad_calculate_price_requests(): "version": good_version, "sizes": good_sizes, } - ).map(dumps) + ).map(dumps_utf8) bad_data_version = fixed_dictionaries( { "version": bad_version, "sizes": good_sizes, } - ).map(dumps) + ).map(dumps_utf8) bad_data_sizes = fixed_dictionaries( { "version": good_version, "sizes": bad_sizes, } - ).map(dumps) + ).map(dumps_utf8) bad_data_other = dictionaries( text(), integers(), - ).map(dumps) + ).map(dumps_utf8) bad_data_junk = binary() @@ -1667,7 +1667,7 @@ def test_calculated_price(self, encoding_params_and_config, api_auth_token, size b"POST", self.url, headers={b"content-type": [b"application/json"]}, - data=BytesIO(dumps({"version": 1, "sizes": sizes})), + data=BytesIO(dumps_utf8({"version": 1, "sizes": sizes})), ), succeeded( matches_response( diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index 4b4f399b..1ffac55a 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -59,7 +59,7 @@ from twisted.web.resource import ErrorPage, Resource from zope.interface import implementer -from .._json import dumps +from .._json import dumps_utf8 from ..controller import ( AlreadySpent, DoubleSpendRedeemer, @@ -1155,7 +1155,7 @@ def render_POST(self, request): finally: servers_proof.destroy() - return dumps( + return dumps_utf8( { u"success": True, u"public-key": ensure_text(self.public_key.encode_base64()), @@ -1233,7 +1233,7 @@ def test_missing_properties(self, properties): treq = treq_for_loopback_ristretto(issuer) d = treq.post( NOWHERE.child(u"v1", u"redeem").to_text().encode("ascii"), - dumps(dict.fromkeys(properties)), + dumps_utf8(dict.fromkeys(properties)), headers=Headers({u"content-type": [u"application/json"]}), ) self.assertThat( @@ -1283,7 +1283,7 @@ def check_redemption_request(request): def bad_request(request, body_object): request.setResponseCode(BAD_REQUEST) request.setHeader(b"content-type", b"application/json") - request.write(dumps(body_object)) + request.write(dumps_utf8(body_object)) return b"" From 36667f6536dd69b69bfe6fe3c2685573cf877831 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 12:42:34 -0500 Subject: [PATCH 26/59] drop __future__ and future imports They're all redundant on Python 3 --- src/_zkapauthorizer/_base64.py | 2 - src/_zkapauthorizer/_plugin.py | 8 +- src/_zkapauthorizer/_storage_client.py | 77 ++++++------------- src/_zkapauthorizer/_storage_server.py | 34 +------- src/_zkapauthorizer/configutil.py | 3 - src/_zkapauthorizer/controller.py | 34 +------- src/_zkapauthorizer/eliot.py | 29 ------- src/_zkapauthorizer/foolscap.py | 8 +- src/_zkapauthorizer/lease_maintenance.py | 29 ------- src/_zkapauthorizer/model.py | 46 +---------- src/_zkapauthorizer/private.py | 2 - src/_zkapauthorizer/resource.py | 33 +------- src/_zkapauthorizer/schema.py | 3 - src/_zkapauthorizer/server/spending.py | 7 +- src/_zkapauthorizer/spending.py | 41 ++-------- src/_zkapauthorizer/storage_common.py | 36 +-------- src/_zkapauthorizer/tests/eliot.py | 2 - src/_zkapauthorizer/tests/fixtures.py | 29 ------- src/_zkapauthorizer/tests/foolscap.py | 4 +- src/_zkapauthorizer/tests/json.py | 2 - src/_zkapauthorizer/tests/privacypass.py | 2 - src/_zkapauthorizer/tests/storage_common.py | 3 +- src/_zkapauthorizer/tests/strategies.py | 37 +-------- src/_zkapauthorizer/tests/test_base64.py | 2 - .../tests/test_client_resource.py | 43 ++--------- src/_zkapauthorizer/tests/test_controller.py | 6 +- src/_zkapauthorizer/tests/test_foolscap.py | 41 ++-------- .../tests/test_lease_maintenance.py | 29 ------- src/_zkapauthorizer/tests/test_matchers.py | 2 - src/_zkapauthorizer/tests/test_model.py | 29 ------- src/_zkapauthorizer/tests/test_plugin.py | 42 +--------- .../tests/test_pricecalculator.py | 29 ------- src/_zkapauthorizer/tests/test_private.py | 28 ------- src/_zkapauthorizer/tests/test_schema.py | 2 - src/_zkapauthorizer/tests/test_spending.py | 29 ------- .../tests/test_storage_client.py | 29 ------- .../tests/test_storage_protocol.py | 2 - .../tests/test_storage_server.py | 4 +- src/_zkapauthorizer/tests/test_strategies.py | 29 ------- 39 files changed, 64 insertions(+), 753 deletions(-) diff --git a/src/_zkapauthorizer/_base64.py b/src/_zkapauthorizer/_base64.py index 473cb41c..604797ec 100644 --- a/src/_zkapauthorizer/_base64.py +++ b/src/_zkapauthorizer/_base64.py @@ -16,8 +16,6 @@ This module implements base64 encoding-related functionality. """ -from __future__ import absolute_import - from base64 import b64decode as _b64decode from binascii import Error from re import compile as _compile diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index 5deab0dd..68a5d19d 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -17,17 +17,11 @@ Tahoe-LAFS. """ -from __future__ import absolute_import - import random from datetime import datetime from functools import partial from weakref import WeakValueDictionary - -try: - from typing import Callable -except ImportError: - pass +from typing import Callable import attr from allmydata.client import _Client diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index ecb785f9..1f9d19ad 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -20,43 +20,15 @@ implemented in ``_storage_server.py``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from functools import partial, wraps +from typing import Any, Tuple, Dict, List, Optional import attr from allmydata.interfaces import IStorageServer from allmydata.util.eliotutil import log_call_deferred from attr.validators import provides from eliot.twisted import inline_callbacks -from twisted.internet.defer import returnValue +from twisted.internet.defer import returnValue, Deferred from twisted.internet.interfaces import IReactorTime from twisted.python.reflect import namedAny from zope.interface import implementer @@ -73,6 +45,20 @@ slot_testv_and_readv_and_writev_message, ) +Secrets = Tuple[bytes, bytes, bytes] +TestWriteVectors = Dict[ + int, + Tuple[ + List[ + Tuple[int, int, bytes, bytes], + ], + List[ + Tuple[int, bytes], + ], + Optional[int], + ], +] +ReadVector = List[Tuple[int, int]] class IncorrectStorageServerReference(Exception): """ @@ -444,31 +430,12 @@ def advise_corrupt_share( @with_rref def slot_testv_and_readv_and_writev( self, - rref, - storage_index, - secrets, - tw_vectors, - r_vector, - ): - # type: ( - # Any, - # bytes, - # Tuple[bytes, bytes, bytes], - # Dict[ - # int, - # Tuple[ - # List[ - # Tuple[int, int, bytes, bytes], - # ], - # List[ - # Tuple[int, bytes], - # ], - # Optional[int], - # ], - # ], - # List[Tuple[int, int]], - # ) -> Deferred - + rref: Any, + storage_index: bytes, + secrets: Secrets, + tw_vectors: TestWriteVectors, + r_vector: ReadVector, + ) -> Deferred: # Read operations are free. num_passes = 0 diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 6e0ea94e..2aedd909 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -21,41 +21,13 @@ implemented in ``_storage_client.py``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import timedelta from errno import ENOENT from functools import partial from os import listdir, stat from os.path import join from struct import calcsize, unpack +from typing import Dict, List, Optional import attr from allmydata.interfaces import RIStorageServer, TestAndWriteVectorsForShares @@ -96,10 +68,6 @@ slot_testv_and_readv_and_writev_message, ) -try: - from typing import Dict, List, Optional -except ImportError: - pass # The last Python 2-supporting prometheus_client nevertheless tries to use # FileNotFoundError, an exception type from Python 3. Since that release, diff --git a/src/_zkapauthorizer/configutil.py b/src/_zkapauthorizer/configutil.py index 2df26fc6..82709368 100644 --- a/src/_zkapauthorizer/configutil.py +++ b/src/_zkapauthorizer/configutil.py @@ -16,9 +16,6 @@ Basic utilities related to the Tahoe configuration file. """ -from __future__ import absolute_import, division, print_function, unicode_literals - - def _merge_dictionaries(dictionaries): """ Collapse a sequence of dictionaries into one, with collisions resolved by diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index 4be61aa6..a1147a79 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -17,42 +17,13 @@ for the client side of the storage plugin. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - +from typing import List from base64 import b64decode, b64encode from datetime import timedelta from functools import partial from hashlib import sha256 from json import loads from operator import delitem, setitem -from sys import exc_info from six import ensure_text import attr @@ -1056,10 +1027,7 @@ def bracket(first, last, between): except GeneratorExit: raise except: - info = exc_info() yield last() - if PY2: - exec("raise info[0], info[1], info[2]") raise else: yield last() diff --git a/src/_zkapauthorizer/eliot.py b/src/_zkapauthorizer/eliot.py index 82676056..e9f4d493 100644 --- a/src/_zkapauthorizer/eliot.py +++ b/src/_zkapauthorizer/eliot.py @@ -16,35 +16,6 @@ Eliot field, message, and action definitions for ZKAPAuthorizer. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from eliot import ActionType, Field, MessageType PRIVACYPASS_MESSAGE = Field( diff --git a/src/_zkapauthorizer/foolscap.py b/src/_zkapauthorizer/foolscap.py index c2d814ec..efd2e74f 100644 --- a/src/_zkapauthorizer/foolscap.py +++ b/src/_zkapauthorizer/foolscap.py @@ -17,15 +17,11 @@ to communicate between storage clients and servers. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 import attr from allmydata.interfaces import Offset, RIStorageServer, StorageIndex from foolscap.api import Any, Copyable, DictOf, ListOf, RemoteCopy from foolscap.constraint import ByteStringConstraint from foolscap.remoteinterface import RemoteInterface, RemoteMethodSchema -from six import ensure_str @attr.s class ShareStat(Copyable, RemoteCopy): @@ -38,7 +34,7 @@ class ShareStat(Copyable, RemoteCopy): lease on this share expires, or None if there is no lease. """ - typeToCopy = copytype = ensure_str("ShareStat") + typeToCopy = copytype = "ShareStat" # To be a RemoteCopy it must be possible to instantiate this with no # arguments. :/ So supply defaults for these attributes. @@ -137,7 +133,7 @@ class RIPrivacyPassAuthorizedStorageServer(RemoteInterface): validated is service provided. """ - __remote_name__ = ensure_str( + __remote_name__ = ( "RIPrivacyPassAuthorizedStorageServer.tahoe.privatestorage.io" ) diff --git a/src/_zkapauthorizer/lease_maintenance.py b/src/_zkapauthorizer/lease_maintenance.py index 4916768b..48f6ecf9 100644 --- a/src/_zkapauthorizer/lease_maintenance.py +++ b/src/_zkapauthorizer/lease_maintenance.py @@ -17,35 +17,6 @@ refresh leases on all shares reachable from a root. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import datetime, timedelta from errno import ENOENT from functools import partial diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index 0aec9d38..a568ba20 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -17,35 +17,6 @@ the storage plugin. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import datetime from functools import wraps from json import loads @@ -53,7 +24,7 @@ from sqlite3 import connect as _connect import attr -from aniso8601 import parse_datetime as _parse_datetime +from aniso8601 import parse_datetime from past.builtins import long from twisted.logger import Logger from twisted.python.filepath import FilePath @@ -71,21 +42,6 @@ from .validators import greater_than, has_length, is_base64_encoded -if PY2: - def parse_datetime(s, **kw): - """ - Like ``aniso8601.parse_datetime`` but accept str as well. - """ - if isinstance(s, str): - s = s.encode("utf-8") - assert isinstance(s, bytes) - if "delimiter" in kw and isinstance(kw["delimiter"], str): - kw["delimiter"] = kw["delimiter"].encode("utf-8") - return _parse_datetime(s, **kw) -else: - parse_datetime = _parse_datetime - - class ILeaseMaintenanceObserver(Interface): """ An object which is interested in receiving events related to the progress diff --git a/src/_zkapauthorizer/private.py b/src/_zkapauthorizer/private.py index 535c0020..7e2adb00 100644 --- a/src/_zkapauthorizer/private.py +++ b/src/_zkapauthorizer/private.py @@ -10,8 +10,6 @@ Twisted Web resource hierarchy. """ -from __future__ import absolute_import, division, print_function, unicode_literals - # https://github.com/twisted/nevow/issues/106 may affect this code but if so # then the hotfix Tahoe-LAFS applies should deal with it. # diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index b2b97520..68ef2b7a 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -21,40 +21,11 @@ In the future it should also allow users to read statistics about token usage. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from itertools import islice from json import load, loads from sys import maxsize -from six import ensure_str, ensure_binary +from six import ensure_binary from past.builtins import long from twisted.logger import Logger from twisted.web.http import BAD_REQUEST @@ -172,7 +143,7 @@ def from_configuration( ) root = create_private_tree( - lambda: ensure_binary(node_config.get_private_config(ensure_str("api_auth_token"))), + lambda: ensure_binary(node_config.get_private_config("api_auth_token")), authorizationless_resource_tree( store, controller, diff --git a/src/_zkapauthorizer/schema.py b/src/_zkapauthorizer/schema.py index 9fe72b69..c25dfa9a 100644 --- a/src/_zkapauthorizer/schema.py +++ b/src/_zkapauthorizer/schema.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from __future__ import unicode_literals - """ This module defines the database schema used by the model interface. """ diff --git a/src/_zkapauthorizer/server/spending.py b/src/_zkapauthorizer/server/spending.py index f93f1561..66fe819a 100644 --- a/src/_zkapauthorizer/server/spending.py +++ b/src/_zkapauthorizer/server/spending.py @@ -1,9 +1,4 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -try: - from typing import Any -except ImportError: - pass +from typing import Any import attr from challenge_bypass_ristretto import PublicKey diff --git a/src/_zkapauthorizer/spending.py b/src/_zkapauthorizer/spending.py index 49983f4e..0543fac4 100644 --- a/src/_zkapauthorizer/spending.py +++ b/src/_zkapauthorizer/spending.py @@ -16,40 +16,13 @@ A module for logic controlling the manner in which ZKAPs are spent. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) +from typing import List, Tuple import attr from zope.interface import Attribute, Interface, implementer from .eliot import GET_PASSES, INVALID_PASSES, RESET_PASSES, SPENT_PASSES - +from .model import UnblindedToken, Pass class IPassGroup(Interface): """ @@ -140,7 +113,7 @@ class PassGroup(object): _message = attr.ib(validator=attr.validators.instance_of(bytes)) # type: bytes _factory = attr.ib(validator=attr.validators.provides(IPassFactory)) # type: IPassFactory - _tokens = attr.ib(validator=attr.validators.instance_of(list)) # type: List[(UnblinidedToken, Pass)] + _tokens = attr.ib(validator=attr.validators.instance_of(list)) # type: List[Tuple[UnblindedToken, Pass]] @property def passes(self): @@ -195,11 +168,11 @@ class SpendingController(object): """ get_unblinded_tokens = attr.ib() # type: (int) -> [UnblindedToken] - discard_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None - invalidate_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None - reset_unblinded_tokens = attr.ib() # type: ([UnblindedTokens]) -> None + discard_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None + invalidate_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None + reset_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None - tokens_to_passes = attr.ib() # type: (bytes, [UnblindedTokens]) -> [Pass] + tokens_to_passes = attr.ib() # type: (bytes, [UnblindedToken]) -> [Pass] @classmethod def for_store(cls, tokens_to_passes, store): diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index b9c4b2cf..5313b315 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -16,39 +16,9 @@ Functionality shared between the storage client and server. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from base64 import b64encode import attr -from past.builtins import long from pyutil.mathutil import div_ceil from .eliot import MUTABLE_PASSES_REQUIRED @@ -71,8 +41,8 @@ class MorePassesRequired(Exception): passes indicating passes which failed the signature check. """ - valid_count = attr.ib(validator=attr.validators.instance_of((int, long))) - required_count = attr.ib(validator=attr.validators.instance_of((int, long))) + valid_count = attr.ib(validator=attr.validators.instance_of(int)) + required_count = attr.ib(validator=attr.validators.instance_of(int)) signature_check_failed = attr.ib(converter=frozenset) @@ -332,7 +302,7 @@ def pass_value_attribute(): """ return attr.ib( validator=attr.validators.and_( - attr.validators.instance_of((int, long)), + attr.validators.instance_of(int), greater_than(0), ), ) diff --git a/src/_zkapauthorizer/tests/eliot.py b/src/_zkapauthorizer/tests/eliot.py index ba010cf2..094b64dd 100644 --- a/src/_zkapauthorizer/tests/eliot.py +++ b/src/_zkapauthorizer/tests/eliot.py @@ -16,8 +16,6 @@ Eliot testing helpers. """ -from __future__ import absolute_import - from functools import wraps from unittest import SkipTest diff --git a/src/_zkapauthorizer/tests/fixtures.py b/src/_zkapauthorizer/tests/fixtures.py index 2bc62c8e..9ed90e1f 100644 --- a/src/_zkapauthorizer/tests/fixtures.py +++ b/src/_zkapauthorizer/tests/fixtures.py @@ -16,35 +16,6 @@ Common fixtures to let the test suite focus on application logic. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from base64 import b64encode import attr diff --git a/src/_zkapauthorizer/tests/foolscap.py b/src/_zkapauthorizer/tests/foolscap.py index e2d1c43f..8ad3345c 100644 --- a/src/_zkapauthorizer/tests/foolscap.py +++ b/src/_zkapauthorizer/tests/foolscap.py @@ -16,8 +16,6 @@ Testing helpers related to Foolscap. """ -from __future__ import absolute_import - import attr from allmydata.interfaces import RIStorageServer from foolscap.api import Any, Copyable, Referenceable, RemoteInterface @@ -80,7 +78,7 @@ def __attrs_post_init__(self): self.interfaceName = self.interface.__remote_name__ def getURL(self): - return b"pb://abcd@127.0.0.1:12345/efgh" + return "pb://abcd@127.0.0.1:12345/efgh" @attr.s diff --git a/src/_zkapauthorizer/tests/json.py b/src/_zkapauthorizer/tests/json.py index b8aa7c74..72964128 100644 --- a/src/_zkapauthorizer/tests/json.py +++ b/src/_zkapauthorizer/tests/json.py @@ -16,8 +16,6 @@ A better JSON module. """ -from __future__ import absolute_import - from json import loads as _loads diff --git a/src/_zkapauthorizer/tests/privacypass.py b/src/_zkapauthorizer/tests/privacypass.py index 0f428773..fb4fcc76 100644 --- a/src/_zkapauthorizer/tests/privacypass.py +++ b/src/_zkapauthorizer/tests/privacypass.py @@ -16,8 +16,6 @@ Ristretto-flavored PrivacyPass helpers for the test suite. """ -from __future__ import absolute_import - from challenge_bypass_ristretto import BatchDLEQProof, PublicKey from ..model import Pass diff --git a/src/_zkapauthorizer/tests/storage_common.py b/src/_zkapauthorizer/tests/storage_common.py index 61c061c5..35d52fa0 100644 --- a/src/_zkapauthorizer/tests/storage_common.py +++ b/src/_zkapauthorizer/tests/storage_common.py @@ -20,6 +20,7 @@ from itertools import islice from os import SEEK_CUR from struct import pack +from typing import List, Set, Dict import attr from challenge_bypass_ristretto import RandomToken @@ -222,7 +223,7 @@ class _PassFactory(object): returned = attr.ib(default=attr.Factory(list), init=False) # type: List[int] in_use = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] - invalid = attr.ib(default=attr.Factory(dict), init=False) # type: Dict[int, unicode] + invalid = attr.ib(default=attr.Factory(dict), init=False) # type: Dict[int, str] spent = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] issued = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 54306ea1..977362a0 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -16,35 +16,6 @@ Hypothesis strategies for property testing. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta @@ -69,8 +40,7 @@ text, tuples, ) -from six.moves.urllib.parse import quote -from six import ensure_binary +from urllib.parse import quote from twisted.internet.defer import succeed from twisted.internet.task import Clock from twisted.web.test.requesthelper import DummyRequest @@ -759,10 +729,7 @@ def request_paths(): :see: ``requests`` """ def quote_segment(seg): - if PY2: - return quote(seg.encode("utf-8"), safe=b"") - else: - return quote(seg, safe="").encode("utf-8") + return quote(seg, safe="").encode("utf-8") return lists(text().map(quote_segment)) diff --git a/src/_zkapauthorizer/tests/test_base64.py b/src/_zkapauthorizer/tests/test_base64.py index d91bbacd..df3111d2 100644 --- a/src/_zkapauthorizer/tests/test_base64.py +++ b/src/_zkapauthorizer/tests/test_base64.py @@ -16,8 +16,6 @@ Tests for ``_zkapauthorizer._base64``. """ -from __future__ import absolute_import - from base64 import urlsafe_b64encode from hypothesis import given diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 63acf4c4..f206bddc 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -17,39 +17,11 @@ plugin. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import datetime from io import BytesIO -from six.moves.urllib.parse import quote -from six import ensure_binary, ensure_text +from urllib.parse import quote +from six import ensure_text +from typing import Set, TypeVar, Optional import attr from allmydata.client import config_from_string @@ -138,6 +110,8 @@ vouchers, ) +Strategy = TypeVar() + TRANSIENT_ERROR = "something went wrong, who knows what" # Helper to work-around https://github.com/twisted/treq/issues/161 @@ -1427,15 +1401,12 @@ def _test_list_vouchers( ), ) - -def mime_types(blacklist=None): +def mime_types(blacklist: Optional[Set[str]] = None) -> Strategy[str]: """ Build MIME types as b"major/minor" byte strings. - :param blacklist: If not ``None``, MIME types to - exclude from the result. + :param blacklist: If not ``None``, MIME types to exclude from the result. """ - # type: Optional[Set[unicode]] -> Strategy[unicode] if blacklist is None: blacklist = set() return ( diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index 1ffac55a..b7efbe82 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -16,8 +16,6 @@ Tests for ``_zkapauthorizer.controller``. """ -from __future__ import absolute_import, division - from datetime import datetime, timedelta from functools import partial from json import loads @@ -1300,10 +1298,10 @@ class _BracketTestMixin: Tests for ``bracket``. """ def wrap_success(self, result): - raise NotImplemented() + raise NotImplementedError() def wrap_failure(self, result): - raise NotImplemented() + raise NotImplementedError() def test_success(self): """ diff --git a/src/_zkapauthorizer/tests/test_foolscap.py b/src/_zkapauthorizer/tests/test_foolscap.py index 1dee261c..f22985f4 100644 --- a/src/_zkapauthorizer/tests/test_foolscap.py +++ b/src/_zkapauthorizer/tests/test_foolscap.py @@ -16,36 +16,6 @@ Tests for Foolscap-related test helpers. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -# if PY2: -# from future.builtins import ( # noqa: F401 -# filter, -# map, -# zip, -# ascii, -# chr, -# hex, -# input, -# next, -# oct, -# open, -# pow, -# round, -# super, -# bytes, -# dict, -# list, -# object, -# range, -# str, -# max, -# min, -# ) - -from six import ensure_str from fixtures import Fixture from foolscap.api import Any, RemoteInterface, Violation from foolscap.furl import decode_furl @@ -83,7 +53,7 @@ def whatever_method(arg=Any()): def remote_reference(): tub = Tub() tub.setLocation("127.0.0.1:12345") - url = tub.buildURL(ensure_str("efgh")) + url = tub.buildURL("efgh") # Ugh ugh ugh. Skip over the extra correctness checking in # RemoteReferenceTracker.__init__ that requires having a broker by passing @@ -113,9 +83,12 @@ def test_tracker_url(self, ref): """ self.assertThat( ref.tracker.getURL(), - AfterPreprocessing( - decode_furl, - Always(), + MatchesAll( + IsInstance(str), + AfterPreprocessing( + decode_furl, + Always(), + ), ), ) diff --git a/src/_zkapauthorizer/tests/test_lease_maintenance.py b/src/_zkapauthorizer/tests/test_lease_maintenance.py index 15d9ca97..2f20fcdf 100644 --- a/src/_zkapauthorizer/tests/test_lease_maintenance.py +++ b/src/_zkapauthorizer/tests/test_lease_maintenance.py @@ -16,35 +16,6 @@ Tests for ``_zkapauthorizer.lease_maintenance``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import datetime, timedelta import attr diff --git a/src/_zkapauthorizer/tests/test_matchers.py b/src/_zkapauthorizer/tests/test_matchers.py index e34bb8ab..d97ac1a1 100644 --- a/src/_zkapauthorizer/tests/test_matchers.py +++ b/src/_zkapauthorizer/tests/test_matchers.py @@ -16,8 +16,6 @@ Tests for ``_zkapauthorizer.tests.matchers``. """ -from __future__ import absolute_import - from testtools import TestCase from testtools.matchers import Is, Not from zope.interface import Interface, implementer diff --git a/src/_zkapauthorizer/tests/test_model.py b/src/_zkapauthorizer/tests/test_model.py index 0102ee70..914bb9ab 100644 --- a/src/_zkapauthorizer/tests/test_model.py +++ b/src/_zkapauthorizer/tests/test_model.py @@ -17,35 +17,6 @@ Tests for ``_zkapauthorizer.model``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -# if PY2: -# from future.builtins import ( # noqa: F401 -# filter, -# map, -# zip, -# ascii, -# chr, -# hex, -# input, -# next, -# oct, -# open, -# pow, -# round, -# super, -# bytes, -# dict, -# list, -# object, -# range, -# str, -# max, -# min, -# ) - from datetime import datetime, timedelta from errno import EACCES from os import mkdir diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 0baa1124..80d98139 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -16,35 +16,6 @@ Tests for the Tahoe-LAFS plugin. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from datetime import timedelta from functools import partial from os import makedirs @@ -67,8 +38,7 @@ from hypothesis.strategies import datetimes, just, sampled_from, timedeltas from prometheus_client import Gauge from prometheus_client.parser import text_string_to_metric_families -from six.moves import StringIO -from six import ensure_binary +from io import StringIO from testtools import TestCase from testtools.content import text_content from testtools.matchers import ( @@ -418,16 +388,6 @@ def test_mismatched_ristretto_issuer(self, config_text, announcement): u"tub.port", config_text.encode("utf-8"), ) - # On Tahoe-LAFS <1.16, the config is written as bytes. - # On Tahoe-LAFS >=1.16, the config is written as unicode. - # - # So we'll use `StringIO.StringIO` (not `io.StringIO`) here - which - # will allow either type (it will also implicitly decode bytes to - # unicode if we mix them, though I don't think that should happen - # here). - # - # After support for Tahoe <1.16 support is dropped we probably want to - # switch to an io.StringIO here. config_text = StringIO() node_config.config.write(config_text) self.addDetail("config", text_content(config_text.getvalue())) diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index b69135bf..2d447e3f 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -17,35 +17,6 @@ Tests for ``_zkapauthorizer.pricecalculator``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from functools import partial from hypothesis import given diff --git a/src/_zkapauthorizer/tests/test_private.py b/src/_zkapauthorizer/tests/test_private.py index effc6dc7..ba1fa34d 100644 --- a/src/_zkapauthorizer/tests/test_private.py +++ b/src/_zkapauthorizer/tests/test_private.py @@ -9,34 +9,6 @@ Tests for ``_zkapauthorizer.private``. """ -from __future__ import absolute_import, division, print_function, unicode_literals -from future.utils import PY2 -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - - from allmydata.test.web.matchers import has_response_code from testtools import TestCase from testtools.matchers import Equals diff --git a/src/_zkapauthorizer/tests/test_schema.py b/src/_zkapauthorizer/tests/test_schema.py index 1c535560..f0dc6f87 100644 --- a/src/_zkapauthorizer/tests/test_schema.py +++ b/src/_zkapauthorizer/tests/test_schema.py @@ -17,8 +17,6 @@ Tests for ``_zkapauthorizer.schema``. """ -from __future__ import absolute_import - from testtools import TestCase from testtools.matchers import Equals diff --git a/src/_zkapauthorizer/tests/test_spending.py b/src/_zkapauthorizer/tests/test_spending.py index 55f9aa92..42a5d87f 100644 --- a/src/_zkapauthorizer/tests/test_spending.py +++ b/src/_zkapauthorizer/tests/test_spending.py @@ -16,35 +16,6 @@ Tests for ``_zkapauthorizer.spending``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from hypothesis import given from hypothesis.strategies import data, integers, randoms from testtools import TestCase diff --git a/src/_zkapauthorizer/tests/test_storage_client.py b/src/_zkapauthorizer/tests/test_storage_client.py index 8da2975c..5a0658b6 100644 --- a/src/_zkapauthorizer/tests/test_storage_client.py +++ b/src/_zkapauthorizer/tests/test_storage_client.py @@ -16,35 +16,6 @@ Tests for ``_zkapauthorizer._storage_client``. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from functools import partial from allmydata.client import config_from_string diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index 7e5fb3e6..a231c66b 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -16,8 +16,6 @@ Tests for communication between the client and server components. """ -from __future__ import absolute_import - from allmydata.storage.common import storage_index_to_dir from allmydata.storage.shares import get_share_file from challenge_bypass_ristretto import PublicKey, random_signing_key diff --git a/src/_zkapauthorizer/tests/test_storage_server.py b/src/_zkapauthorizer/tests/test_storage_server.py index b40e8263..d4829914 100644 --- a/src/_zkapauthorizer/tests/test_storage_server.py +++ b/src/_zkapauthorizer/tests/test_storage_server.py @@ -16,8 +16,6 @@ Tests for ``_zkapauthorizer._storage_server``. """ -from __future__ import absolute_import, division - from random import shuffle from time import time @@ -136,7 +134,7 @@ def test_raise_for(self): AfterPreprocessing( str, Equals( - "MorePassesRequired(valid_count=4, required_count=10, signature_check_failed={})".format(str(frozenset([4]))), + "MorePassesRequired(valid_count=4, required_count=10, signature_check_failed=frozenset({4}))", ), ), ), diff --git a/src/_zkapauthorizer/tests/test_strategies.py b/src/_zkapauthorizer/tests/test_strategies.py index da975705..1184dbbe 100644 --- a/src/_zkapauthorizer/tests/test_strategies.py +++ b/src/_zkapauthorizer/tests/test_strategies.py @@ -16,35 +16,6 @@ Tests for our custom Hypothesis strategies. """ -from __future__ import absolute_import, division, print_function, unicode_literals - -from future.utils import PY2 - -if PY2: - from future.builtins import ( # noqa: F401 - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) - from allmydata.client import config_from_string from fixtures import TempDir from hypothesis import given, note From 11a5df57f4240e4256d9d5a2a76ccc7f4eea1aeb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:20:35 -0500 Subject: [PATCH 27/59] remove six and future dependency, also isort and black --- setup.cfg | 4 -- src/_zkapauthorizer/_json.py | 1 + src/_zkapauthorizer/_plugin.py | 2 +- src/_zkapauthorizer/_storage_client.py | 5 ++- src/_zkapauthorizer/_storage_server.py | 3 +- src/_zkapauthorizer/config.py | 6 +-- src/_zkapauthorizer/configutil.py | 1 + src/_zkapauthorizer/controller.py | 18 ++++----- src/_zkapauthorizer/foolscap.py | 5 +-- src/_zkapauthorizer/lease_maintenance.py | 14 +------ src/_zkapauthorizer/model.py | 15 ++++---- src/_zkapauthorizer/resource.py | 5 +-- src/_zkapauthorizer/spending.py | 13 +++++-- src/_zkapauthorizer/tests/storage_common.py | 2 +- src/_zkapauthorizer/tests/strategies.py | 3 +- .../tests/test_client_resource.py | 29 +++++++++------ src/_zkapauthorizer/tests/test_controller.py | 17 ++++++--- .../tests/test_lease_maintenance.py | 10 +---- src/_zkapauthorizer/tests/test_model.py | 3 +- src/_zkapauthorizer/tests/test_plugin.py | 4 +- .../tests/test_storage_protocol.py | 37 ++++++++++++------- 21 files changed, 102 insertions(+), 95 deletions(-) diff --git a/setup.cfg b/setup.cfg index fe4b45cd..65fd066c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,10 +53,6 @@ install_requires = # install cross-platform. colorama - # Python 3 transition - six - future - [options.extras_require] test = coverage; fixtures; testtools; hypothesis diff --git a/src/_zkapauthorizer/_json.py b/src/_zkapauthorizer/_json.py index 898f6b56..cd148c95 100644 --- a/src/_zkapauthorizer/_json.py +++ b/src/_zkapauthorizer/_json.py @@ -15,6 +15,7 @@ from json import dumps as _dumps from typing import Any + def dumps_utf8(o: Any) -> bytes: """ Serialize an object to a UTF-8-encoded JSON byte string. diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index 68a5d19d..ba941158 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -20,8 +20,8 @@ import random from datetime import datetime from functools import partial -from weakref import WeakValueDictionary from typing import Callable +from weakref import WeakValueDictionary import attr from allmydata.client import _Client diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index 1f9d19ad..6ebd3cce 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -21,14 +21,14 @@ """ from functools import partial, wraps -from typing import Any, Tuple, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import attr from allmydata.interfaces import IStorageServer from allmydata.util.eliotutil import log_call_deferred from attr.validators import provides from eliot.twisted import inline_callbacks -from twisted.internet.defer import returnValue, Deferred +from twisted.internet.defer import Deferred, returnValue from twisted.internet.interfaces import IReactorTime from twisted.python.reflect import namedAny from zope.interface import implementer @@ -60,6 +60,7 @@ ] ReadVector = List[Tuple[int, int]] + class IncorrectStorageServerReference(Exception): """ A Foolscap remote object which should reference a ZKAPAuthorizer storage diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 2aedd909..73e4d55a 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -131,7 +131,8 @@ def _is_invalid_pass(cls, message, pass_, signing_key): unblinded_token = signing_key.rederive_unblinded_token(preimage) verification_key = unblinded_token.derive_verification_key_sha512() invalid_pass = verification_key.invalid_sha512( - proposed_signature, message, + proposed_signature, + message, ) return invalid_pass except Exception: diff --git a/src/_zkapauthorizer/config.py b/src/_zkapauthorizer/config.py index b30b43fb..f13e895f 100644 --- a/src/_zkapauthorizer/config.py +++ b/src/_zkapauthorizer/config.py @@ -17,11 +17,7 @@ """ from datetime import timedelta - -try: - from typing import Optional -except ImportError: - pass +from typing import Optional from allmydata.node import _Config diff --git a/src/_zkapauthorizer/configutil.py b/src/_zkapauthorizer/configutil.py index 82709368..b9b167ec 100644 --- a/src/_zkapauthorizer/configutil.py +++ b/src/_zkapauthorizer/configutil.py @@ -16,6 +16,7 @@ Basic utilities related to the Tahoe configuration file. """ + def _merge_dictionaries(dictionaries): """ Collapse a sequence of dictionaries into one, with collisions resolved by diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index a1147a79..70781129 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -17,15 +17,14 @@ for the client side of the storage plugin. """ -from typing import List from base64 import b64decode, b64encode from datetime import timedelta from functools import partial from hashlib import sha256 from json import loads from operator import delitem, setitem +from typing import List -from six import ensure_text import attr import challenge_bypass_ristretto from treq import content @@ -38,8 +37,8 @@ from twisted.web.client import Agent from zope.interface import Interface, implementer -from ._json import dumps_utf8 from ._base64 import urlsafe_b64decode +from ._json import dumps_utf8 from ._stack import less_limited_stack from .model import Error as model_Error from .model import Pass @@ -245,10 +244,10 @@ class ErrorRedeemer(object): @classmethod def make(cls, section_name, node_config, announcement, reactor): - details = ensure_text(node_config.get_config( + details = node_config.get_config( section=section_name, option="details", - )) + ) return cls(details) def random_tokens_for_voucher(self, voucher, counter, count): @@ -462,10 +461,10 @@ class RistrettoRedeemer(object): @classmethod def make(cls, section_name, node_config, announcement, reactor): - configured_issuer = ensure_text(node_config.get_config( + configured_issuer = node_config.get_config( section=section_name, option="ristretto-issuer-root-url", - )) + ) if announcement is not None: # Don't let us talk to a storage server that has a different idea # about who issues ZKAPs. We should lift this limitation (that is, we @@ -507,7 +506,8 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): "redeemVoucher": voucher.number.decode("ascii"), "redeemCounter": counter, "redeemTokens": list( - ensure_text(token.encode_base64()) for token in blinded_tokens + token.encode_base64().decode("utf-8") + for token in blinded_tokens ), } ), @@ -941,7 +941,7 @@ def _redeem_failure(self, voucher, reason): ) self._error[voucher] = model_Error( finished=self.store.now(), - details=ensure_text(reason.getErrorMessage()), + details=reason.getErrorMessage(), ) return False diff --git a/src/_zkapauthorizer/foolscap.py b/src/_zkapauthorizer/foolscap.py index efd2e74f..5ff7ff92 100644 --- a/src/_zkapauthorizer/foolscap.py +++ b/src/_zkapauthorizer/foolscap.py @@ -23,6 +23,7 @@ from foolscap.constraint import ByteStringConstraint from foolscap.remoteinterface import RemoteInterface, RemoteMethodSchema + @attr.s class ShareStat(Copyable, RemoteCopy): """ @@ -133,9 +134,7 @@ class RIPrivacyPassAuthorizedStorageServer(RemoteInterface): validated is service provided. """ - __remote_name__ = ( - "RIPrivacyPassAuthorizedStorageServer.tahoe.privatestorage.io" - ) + __remote_name__ = "RIPrivacyPassAuthorizedStorageServer.tahoe.privatestorage.io" get_version = RIStorageServer["get_version"] diff --git a/src/_zkapauthorizer/lease_maintenance.py b/src/_zkapauthorizer/lease_maintenance.py index 48f6ecf9..ed7d9672 100644 --- a/src/_zkapauthorizer/lease_maintenance.py +++ b/src/_zkapauthorizer/lease_maintenance.py @@ -20,13 +20,8 @@ from datetime import datetime, timedelta from errno import ENOENT from functools import partial +from typing import Any, Dict, Iterable -try: - from typing import Any, Dict -except ImportError: - pass - -from six import ensure_binary import attr from allmydata.interfaces import IDirectoryNode, IFilesystemNode from allmydata.util.hashutil import ( @@ -45,11 +40,6 @@ from .foolscap import ShareStat from .model import ILeaseMaintenanceObserver -try: - from typing import Iterable -except ImportError: - pass - SERVICE_NAME = u"lease maintenance service" @@ -508,7 +498,7 @@ def write_time_to_path(path, when): :param datetime when: The datetime to write. """ - path.setContent(ensure_binary(when.isoformat())) + path.setContent(when.isoformat().encode("utf-8")) def read_time_from_path(path): diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index a568ba20..df00eb63 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -29,10 +29,9 @@ from twisted.logger import Logger from twisted.python.filepath import FilePath from zope.interface import Interface, implementer -from six import ensure_text -from ._json import dumps_utf8 from ._base64 import urlsafe_b64decode +from ._json import dumps_utf8 from .schema import get_schema_upgrades, get_schema_version, run_schema_upgrades from .storage_common import ( get_configured_pass_value, @@ -936,7 +935,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "redeeming", - "started": ensure_text(self.started.isoformat()), + "started": self.started.isoformat(), "counter": self.counter, } @@ -961,7 +960,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "redeemed", - "finished": ensure_text(self.finished.isoformat()), + "finished": self.finished.isoformat(), "token-count": self.token_count, } @@ -976,7 +975,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "double-spend", - "finished": ensure_text(self.finished.isoformat()), + "finished": self.finished.isoformat(), } @@ -996,7 +995,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "unpaid", - "finished": ensure_text(self.finished.isoformat()), + "finished": self.finished.isoformat(), } @@ -1017,7 +1016,7 @@ def should_start_redemption(self): def to_json_v1(self): return { "name": "error", - "finished": ensure_text(self.finished.isoformat()), + "finished": self.finished.isoformat(), "details": self.details, } @@ -1165,7 +1164,7 @@ def to_json_v1(self): return { "number": self.number.decode("ascii"), "expected-tokens": self.expected_tokens, - "created": None if self.created is None else ensure_text(self.created.isoformat()), + "created": None if self.created is None else self.created.isoformat(), "state": state, "version": 1, } diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 68ef2b7a..93993921 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -25,7 +25,6 @@ from json import load, loads from sys import maxsize -from six import ensure_binary from past.builtins import long from twisted.logger import Logger from twisted.web.http import BAD_REQUEST @@ -33,9 +32,9 @@ from twisted.web.server import NOT_DONE_YET from zope.interface import Attribute -from ._json import dumps_utf8 from . import __version__ as _zkapauthorizer_version from ._base64 import urlsafe_b64decode +from ._json import dumps_utf8 from .config import get_configured_lease_duration from .controller import PaymentController, get_redeemer from .pricecalculator import PriceCalculator @@ -143,7 +142,7 @@ def from_configuration( ) root = create_private_tree( - lambda: ensure_binary(node_config.get_private_config("api_auth_token")), + lambda: node_config.get_private_config("api_auth_token").encode("utf-8"), authorizationless_resource_tree( store, controller, diff --git a/src/_zkapauthorizer/spending.py b/src/_zkapauthorizer/spending.py index 0543fac4..89625b56 100644 --- a/src/_zkapauthorizer/spending.py +++ b/src/_zkapauthorizer/spending.py @@ -22,7 +22,8 @@ from zope.interface import Attribute, Interface, implementer from .eliot import GET_PASSES, INVALID_PASSES, RESET_PASSES, SPENT_PASSES -from .model import UnblindedToken, Pass +from .model import Pass, UnblindedToken + class IPassGroup(Interface): """ @@ -112,8 +113,12 @@ class PassGroup(object): """ _message = attr.ib(validator=attr.validators.instance_of(bytes)) # type: bytes - _factory = attr.ib(validator=attr.validators.provides(IPassFactory)) # type: IPassFactory - _tokens = attr.ib(validator=attr.validators.instance_of(list)) # type: List[Tuple[UnblindedToken, Pass]] + _factory = attr.ib( + validator=attr.validators.provides(IPassFactory) + ) # type: IPassFactory + _tokens = attr.ib( + validator=attr.validators.instance_of(list) + ) # type: List[Tuple[UnblindedToken, Pass]] @property def passes(self): @@ -172,7 +177,7 @@ class SpendingController(object): invalidate_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None reset_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None - tokens_to_passes = attr.ib() # type: (bytes, [UnblindedToken]) -> [Pass] + tokens_to_passes = attr.ib() # type: (bytes, [UnblindedToken]) -> [Pass] @classmethod def for_store(cls, tokens_to_passes, store): diff --git a/src/_zkapauthorizer/tests/storage_common.py b/src/_zkapauthorizer/tests/storage_common.py index 35d52fa0..62bfae30 100644 --- a/src/_zkapauthorizer/tests/storage_common.py +++ b/src/_zkapauthorizer/tests/storage_common.py @@ -20,7 +20,7 @@ from itertools import islice from os import SEEK_CUR from struct import pack -from typing import List, Set, Dict +from typing import Dict, List, Set import attr from challenge_bypass_ristretto import RandomToken diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index 977362a0..d8b48e24 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -18,6 +18,7 @@ from base64 import b64encode, urlsafe_b64encode from datetime import datetime, timedelta +from urllib.parse import quote import attr from allmydata.client import config_from_string @@ -40,7 +41,6 @@ text, tuples, ) -from urllib.parse import quote from twisted.internet.defer import succeed from twisted.internet.task import Clock from twisted.web.test.requesthelper import DummyRequest @@ -728,6 +728,7 @@ def request_paths(): :see: ``requests`` """ + def quote_segment(seg): return quote(seg, safe="").encode("utf-8") diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index f206bddc..e72c0b92 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -19,9 +19,8 @@ from datetime import datetime from io import BytesIO +from typing import Optional, Set from urllib.parse import quote -from six import ensure_text -from typing import Set, TypeVar, Optional import attr from allmydata.client import config_from_string @@ -29,6 +28,7 @@ from fixtures import TempDir from hypothesis import given, note from hypothesis.strategies import ( + SearchStrategy, binary, builds, datetimes, @@ -70,9 +70,9 @@ from twisted.web.http_headers import Headers from twisted.web.resource import IResource, getChildForRequest -from .. _json import dumps_utf8 from .. import __version__ as zkapauthorizer_version from .._base64 import urlsafe_b64decode +from .._json import dumps_utf8 from ..configutil import config_string_from_sections from ..model import ( DoubleSpend, @@ -110,8 +110,6 @@ vouchers, ) -Strategy = TypeVar() - TRANSIENT_ERROR = "something went wrong, who knows what" # Helper to work-around https://github.com/twisted/treq/issues/161 @@ -142,9 +140,11 @@ def not_vouchers(): Builds byte strings which are not legal vouchers. """ return one_of( - text().filter( + text() + .filter( lambda t: (not is_urlsafe_base64(t)), - ).map(lambda t: t.encode("utf-8")), + ) + .map(lambda t: t.encode("utf-8")), vouchers().map( # Turn a valid voucher into a voucher that is invalid only by # containing a character from the base64 alphabet in place of one @@ -176,7 +176,7 @@ def invalid_bodies(): # The wrong key but the right kind of value. fixed_dictionaries( { - "some-key": vouchers().map(ensure_text), + "some-key": vouchers().map(lambda v: v.decode("utf-8")), } ).map(dumps_utf8), # The right key but the wrong kind of value. @@ -184,7 +184,7 @@ def invalid_bodies(): { "voucher": one_of( integers(), - not_vouchers().map(ensure_text), + not_vouchers().map(lambda v: v.decode("utf-8")), ), } ).map(dumps_utf8), @@ -456,7 +456,7 @@ def maybe_extra_tokens(): return one_of( just(None), # If we do, we can't have fewer than the number of redemption groups - # which we don't know until we're further inside the test. So supply + # which we don't know until we're further inside the test. oSo supply # an amount to add to that, in the case where we have tokens at all. integers(min_value=0, max_value=100), ) @@ -520,7 +520,11 @@ def test_post(self, get_config, api_auth_token, voucher, unblinded_tokens): self.assertThat( stored_tokens, - Equals(list(token.unblinded_token.decode("ascii") for token in unblinded_tokens)), + Equals( + list( + token.unblinded_token.decode("ascii") for token in unblinded_tokens + ) + ), ) @given( @@ -1401,7 +1405,8 @@ def _test_list_vouchers( ), ) -def mime_types(blacklist: Optional[Set[str]] = None) -> Strategy[str]: + +def mime_types(blacklist: Optional[Set[str]] = None) -> SearchStrategy[str]: """ Build MIME types as b"major/minor" byte strings. diff --git a/src/_zkapauthorizer/tests/test_controller.py b/src/_zkapauthorizer/tests/test_controller.py index b7efbe82..f48b8e11 100644 --- a/src/_zkapauthorizer/tests/test_controller.py +++ b/src/_zkapauthorizer/tests/test_controller.py @@ -21,7 +21,6 @@ from json import loads import attr -from six import ensure_text from challenge_bypass_ristretto import ( BatchDLEQProof, BlindedToken, @@ -794,7 +793,9 @@ def test_good_ristretto_redemption(self, voucher, counter, num_tokens): HasLength(num_tokens), ), public_key=Equals( - ensure_text(PublicKey.from_signing_key(signing_key).encode_base64()), + PublicKey.from_signing_key(signing_key) + .encode_base64() + .decode("utf-8"), ), ), ), @@ -1156,9 +1157,9 @@ def render_POST(self, request): return dumps_utf8( { u"success": True, - u"public-key": ensure_text(self.public_key.encode_base64()), - u"signatures": list(ensure_text(t) for t in marshaled_signed_tokens), - u"proof": ensure_text(marshaled_proof), + u"public-key": self.public_key.encode_base64().decode("utf-8"), + u"signatures": list(t.decode("utf-8") for t in marshaled_signed_tokens), + u"proof": marshaled_proof.decode("utf-8"), } ) @@ -1297,6 +1298,7 @@ class _BracketTestMixin: """ Tests for ``bracket``. """ + def wrap_success(self, result): raise NotImplementedError() @@ -1370,6 +1372,7 @@ class SomeException(Exception): actions = [] first = partial(actions.append, "first") + def between(): actions.append("between") return self.wrap_success(None) @@ -1461,14 +1464,18 @@ def first(): Equals(["first"]), ) + class BracketTests(_BracketTestMixin, TestCase): def wrap_success(self, result): return result + def wrap_failure(self, exception): raise exception + class SynchronousDeferredBracketTests(_BracketTestMixin, TestCase): def wrap_success(self, result): return succeed(result) + def wrap_failure(self, exception): return fail(exception) diff --git a/src/_zkapauthorizer/tests/test_lease_maintenance.py b/src/_zkapauthorizer/tests/test_lease_maintenance.py index 2f20fcdf..703305b1 100644 --- a/src/_zkapauthorizer/tests/test_lease_maintenance.py +++ b/src/_zkapauthorizer/tests/test_lease_maintenance.py @@ -17,6 +17,7 @@ """ from datetime import datetime, timedelta +from typing import Dict, List import attr from allmydata.client import SecretHolder @@ -35,7 +36,6 @@ randoms, sets, ) -from six import ensure_binary from testtools import TestCase from testtools.matchers import ( AfterPreprocessing, @@ -77,12 +77,6 @@ storage_indexes, ) -try: - from typing import Dict, List -except ImportError: - pass - - default_lease_maint_config = lease_maintenance_from_tahoe_config(empty_config) @@ -312,7 +306,7 @@ def test_initial_interval_with_last_run(self, random, clock, mean, since_last_ru # Figure out the absolute last run time. last_run = datetime_now - since_last_run last_run_path = FilePath(self.useFixture(TempDir()).join("last-run")) - last_run_path.setContent(ensure_binary(last_run.isoformat())) + last_run_path.setContent(last_run.isoformat().encode("utf-8")) service = lease_maintenance_service( dummy_maintain_leases, diff --git a/src/_zkapauthorizer/tests/test_model.py b/src/_zkapauthorizer/tests/test_model.py index 914bb9ab..f5d195f2 100644 --- a/src/_zkapauthorizer/tests/test_model.py +++ b/src/_zkapauthorizer/tests/test_model.py @@ -382,7 +382,8 @@ def _spend_order_test(self, get_config, voucher_value, public_key, now, data): while tokens_remaining > 0: to_spend = data.draw(integers(min_value=1, max_value=tokens_remaining)) extracted_tokens.extend( - token.unblinded_token.decode("ascii") for token in store.get_unblinded_tokens(to_spend) + token.unblinded_token.decode("ascii") + for token in store.get_unblinded_tokens(to_spend) ) tokens_remaining -= to_spend diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 80d98139..10cdaf4a 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -16,10 +16,11 @@ Tests for the Tahoe-LAFS plugin. """ +import os.path from datetime import timedelta from functools import partial +from io import StringIO from os import makedirs -import os.path from allmydata.client import config_from_string, create_client_from_config from allmydata.interfaces import ( @@ -38,7 +39,6 @@ from hypothesis.strategies import datetimes, just, sampled_from, timedeltas from prometheus_client import Gauge from prometheus_client.parser import text_string_to_metric_families -from io import StringIO from testtools import TestCase from testtools.content import text_content from testtools.matchers import ( diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index a231c66b..3a218042 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -1084,7 +1084,9 @@ def write(): data_vector=slot_data_vectors(), replacement_data_vector=slot_data_vectors(), ) - def test_test_vectors_match(self, storage_index, secrets, sharenum, data_vector, replacement_data_vector): + def test_test_vectors_match( + self, storage_index, secrets, sharenum, data_vector, replacement_data_vector + ): """ If test vectors are given then the write is allowed if they match the existing data. @@ -1105,28 +1107,37 @@ def read(sharenum, readv): return d def equal_test_vector(data_vector): - return list( - (offset, len(data), data) - for (offset, data) - in data_vector - ) + return list((offset, len(data), data) for (offset, data) in data_vector) # Create the share - d = write({ - sharenum: (empty_test_vector, data_vector, None), - }) + d = write( + { + sharenum: (empty_test_vector, data_vector, None), + } + ) self.assertThat(d, is_successful_write()) # Write some new data to with a correct test vector. We can only be # sure we know data from the last element of the test vector since # earlier elements may have been overwritten. - d = write({ - sharenum: (equal_test_vector(data_vector)[-1:], replacement_data_vector, None), - }) + d = write( + { + sharenum: ( + equal_test_vector(data_vector)[-1:], + replacement_data_vector, + None, + ), + } + ) self.assertThat(d, is_successful_write()) # Check that the new data is present - assert_read_back_data(self, storage_index, secrets, {sharenum: TestAndWriteVectors(None, replacement_data_vector, None)}) + assert_read_back_data( + self, + storage_index, + secrets, + {sharenum: TestAndWriteVectors(None, replacement_data_vector, None)}, + ) def assert_read_back_data( From 0e489fab6e60c375aef22dede07d1412f360668f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:21:51 -0500 Subject: [PATCH 28/59] update packaging metadata --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 65fd066c..f3b1df62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,8 +10,8 @@ keywords = tahoe-lafs, storage, privacy, cryptography license = Apache 2.0 classifiers = Framework :: Twisted - Programming Language :: Python :: 2 - Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.9 author = PrivateStorage.io, LLC maintainer = PrivateStorage.io, LLC home-page = https://privatestorage.io/ From aab21d0e59e853a71ebb54755e8cbea27789607c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:28:25 -0500 Subject: [PATCH 29/59] drop py27 support from macos ci --- .circleci/config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59b22d6f..5158c936 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,14 +71,7 @@ jobs: - run: name: "Get Pip" command: | - # The CircleCI macOS environment has curl and Python but does not - # have pip. So, for starters, use curl and Python to get pip. - if [ "<< parameters.py-version >>" == "2.7" ]; then - - curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py - else - curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py - fi + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py python<< parameters.py-version >> get-pip.py - run: From a9906350bf3ae775855c5dfcea41c3fbba63dd28 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:28:31 -0500 Subject: [PATCH 30/59] parameterize python version in nixos ci --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5158c936..d8e51b54 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,6 +129,8 @@ jobs: linux-tests: &LINUX_TESTS parameters: + py-version: + type: "string" tahoe-lafs-source: # The name of a niv source in nix/sources.json which corresponds to # a Tahoe-LAFS version. This is the version that will be declared as a @@ -237,6 +239,8 @@ workflows: - "linux-tests": matrix: parameters: + py-version: + - "3.9" tahoe-lafs-source: - "tahoe-lafs" From 11b4cc2782e2c0c849b179e4b9154e5f31e4d872 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:35:02 -0500 Subject: [PATCH 31/59] drop our copy of some eliot functions that we can now use from upstream --- setup.cfg | 2 +- src/_zkapauthorizer/tests/eliot.py | 76 ------------------------ src/_zkapauthorizer/tests/test_plugin.py | 3 +- 3 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 src/_zkapauthorizer/tests/eliot.py diff --git a/setup.cfg b/setup.cfg index 065d28f7..8fd1f296 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ packages = install_requires = attrs zope.interface - eliot + eliot >= 1.11,<2 aniso8601 python-challenge-bypass-ristretto # The pip resolver sometimes finds treq's dependencies first and these are diff --git a/src/_zkapauthorizer/tests/eliot.py b/src/_zkapauthorizer/tests/eliot.py deleted file mode 100644 index 094b64dd..00000000 --- a/src/_zkapauthorizer/tests/eliot.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2019 PrivateStorage.io, LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -Eliot testing helpers. -""" - -from functools import wraps -from unittest import SkipTest - -from eliot import MemoryLogger -from eliot.testing import check_for_errors, swap_logger - - -# validate_logging and capture_logging copied from Eliot around 1.11. We -# can't upgrade past 1.7 because we're not Python 3 compatible. -def validate_logging(assertion, *assertionArgs, **assertionKwargs): - def decorator(function): - @wraps(function) - def wrapper(self, *args, **kwargs): - skipped = False - - kwargs["logger"] = logger = MemoryLogger() - self.addCleanup(check_for_errors, logger) - # TestCase runs cleanups in reverse order, and we want this to - # run *before* tracebacks are checked: - if assertion is not None: - self.addCleanup( - lambda: skipped - or assertion(self, logger, *assertionArgs, **assertionKwargs) - ) - try: - return function(self, *args, **kwargs) - except SkipTest: - skipped = True - raise - - return wrapper - - return decorator - - -def capture_logging(assertion, *assertionArgs, **assertionKwargs): - """ - Capture and validate all logging that doesn't specify a L{Logger}. - - See L{validate_logging} for details on the rest of its behavior. - """ - - def decorator(function): - @validate_logging(assertion, *assertionArgs, **assertionKwargs) - @wraps(function) - def wrapper(self, *args, **kwargs): - logger = kwargs["logger"] - previous_logger = swap_logger(logger) - - def cleanup(): - swap_logger(previous_logger) - - self.addCleanup(cleanup) - return function(self, *args, **kwargs) - - return wrapper - - return decorator diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 10cdaf4a..ce0c16ba 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -30,7 +30,7 @@ RIStorageServer, ) from challenge_bypass_ristretto import SigningKey -from eliot.testing import LoggedMessage +from eliot.testing import LoggedMessage, capture_logging from fixtures import TempDir from foolscap.broker import Broker from foolscap.ipb import IReferenceable, IRemotelyCallable @@ -75,7 +75,6 @@ from ..model import NotEnoughTokens, VoucherStore from ..spending import GET_PASSES from .common import skipIf -from .eliot import capture_logging from .foolscap import DummyReferenceable, LocalRemote, get_anonymous_storage_server from .matchers import Provides, raises from .strategies import ( From ec1bf28d83be3c0786f8904f81d2e2867f9a7fb2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:36:17 -0500 Subject: [PATCH 32/59] switch GitHub Actions to Python 3.9 as well --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8ad846a8..28c3479b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: python-version: - - "2.7" + - "3.9" steps: # Avoid letting Windows newlines confusing milksnake. From ba620311b86081431941b7330d79d34e7962cd65 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 13:43:05 -0500 Subject: [PATCH 33/59] incompatible version formats ... --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8e51b54..67c8ded0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -240,7 +240,7 @@ workflows: matrix: parameters: py-version: - - "3.9" + - "39" tahoe-lafs-source: - "tahoe-lafs" From adf0b03dc24bf0a44e61a698f0eaf0ba71dcaa5b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 14:03:43 -0500 Subject: [PATCH 34/59] maybe this runs the configs I want --- .circleci/config.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 67c8ded0..3936fe14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -244,13 +244,11 @@ workflows: tahoe-lafs-source: - "tahoe-lafs" - - "macos-tests": - matrix: - parameters: - py-version: - - "3.9" - - xcode-version: - # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions - - "12.3.0" - - "11.7.0" + # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions + - "macos-tests-py3_8-xcode11_7_0": + py-version: "3.8" + xcode-version: "11.7.0" + + - "macos-tests-py3_9-xcode12_3_0": + py-version: "3.9" + xcode-version: "12.3.0" From a24eeee612c8bed73937c9144ef8250077f1dcca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 14:06:01 -0500 Subject: [PATCH 35/59] nope, how about this --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3936fe14..787fc369 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -245,10 +245,12 @@ workflows: - "tahoe-lafs" # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions - - "macos-tests-py3_8-xcode11_7_0": + - "macos-tests": + name: "macOS tests python 3.8 xcode 11.7.0": py-version: "3.8" xcode-version: "11.7.0" - - "macos-tests-py3_9-xcode12_3_0": + - "macos-tests": + name: "macOS tests python 3.9 xcode 12.3.0": py-version: "3.9" xcode-version: "12.3.0" From 9ce11d036a8da352ef6664d5738f84e4155c7f8f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 14:06:27 -0500 Subject: [PATCH 36/59] spurious mapping --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 787fc369..3d4ff394 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -246,11 +246,11 @@ workflows: # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions - "macos-tests": - name: "macOS tests python 3.8 xcode 11.7.0": + name: "macOS tests python 3.8 xcode 11.7.0" py-version: "3.8" xcode-version: "11.7.0" - "macos-tests": - name: "macOS tests python 3.9 xcode 12.3.0": + name: "macOS tests python 3.9 xcode 12.3.0" py-version: "3.9" xcode-version: "12.3.0" From 9936c1fd70ce9344d956cc92b33fecabf78e59e5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 14:07:55 -0500 Subject: [PATCH 37/59] normalize the CircleCI job names --- .circleci/config.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d4ff394..bc94d6fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,12 +237,9 @@ workflows: jobs: - "documentation" - "linux-tests": - matrix: - parameters: - py-version: - - "39" - tahoe-lafs-source: - - "tahoe-lafs" + name: "Linux tests python 3.9" + py-version: "39" + tahoe-lafs-source: "tahoe-lafs" # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions - "macos-tests": From cc70a9cd80b645674e5b2c5fe9f9e98c635362fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 14:27:33 -0500 Subject: [PATCH 38/59] Drop the workaround for the problem with older prometheus_client --- src/_zkapauthorizer/_storage_server.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index 73e4d55a..b2e45e2a 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -68,20 +68,6 @@ slot_testv_and_readv_and_writev_message, ) - -# The last Python 2-supporting prometheus_client nevertheless tries to use -# FileNotFoundError, an exception type from Python 3. Since that release, -# prometheus_client has dropped Python 2 support entirely so there is little -# hope of ever having this fixed upstream. When ZKAPAuthorizer is ported to -# Python 3, this should no longer be necessary. -def _prometheus_client_fix(): - import prometheus_client.exposition - - prometheus_client.exposition.FileNotFoundError = IOError - - -_prometheus_client_fix() - # See allmydata/storage/mutable.py SLOT_HEADER_SIZE = 468 LEASE_TRAILER_SIZE = 4 From a8225b0e7dc130360c873274196cbe4cc3514acd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 15:15:11 -0500 Subject: [PATCH 39/59] let setrlimit fail :/ --- src/_zkapauthorizer/_stack.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/_zkapauthorizer/_stack.py b/src/_zkapauthorizer/_stack.py index c6d55337..abb725dc 100644 --- a/src/_zkapauthorizer/_stack.py +++ b/src/_zkapauthorizer/_stack.py @@ -36,7 +36,12 @@ def less_limited_stack(): More precisely, the soft stack limit is raised to the hard limit. """ soft, hard = getrlimit(RLIMIT_STACK) - # We can raise the soft limit to the hard limit and no higher. - setrlimit(RLIMIT_STACK, (hard, hard)) - yield - setrlimit(RLIMIT_STACK, (soft, hard)) + try: + # We can raise the soft limit to the hard limit and no higher. + setrlimit(RLIMIT_STACK, (hard, hard)) + except ValueError: + # Well, not on macOS: https://bugs.python.org/issue34602 + yield + else: + yield + setrlimit(RLIMIT_STACK, (soft, hard)) From 6d098bc5f473ed05819793fbb20709dde41d8a2d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 5 Jan 2022 16:42:20 -0500 Subject: [PATCH 40/59] Add a test for `get_root_nodes` and fix the implementation --- src/_zkapauthorizer/_plugin.py | 13 +++++---- src/_zkapauthorizer/tests/test_plugin.py | 36 ++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index ba941158..4777084b 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -20,12 +20,12 @@ import random from datetime import datetime from functools import partial -from typing import Callable +from typing import Callable, List from weakref import WeakValueDictionary import attr from allmydata.client import _Client -from allmydata.interfaces import IAnnounceableStorageServer, IFoolscapStoragePlugin +from allmydata.interfaces import IAnnounceableStorageServer, IFoolscapStoragePlugin, IFilesystemNode from allmydata.node import MissingConfigEntry from challenge_bypass_ristretto import PublicKey, SigningKey from eliot import start_action @@ -297,13 +297,16 @@ def get_now(): ) -def get_root_nodes(client_node, node_config): +def get_root_nodes(client_node, node_config) -> List[IFilesystemNode]: + """ + Get the configured starting points for lease maintenance traversal. + """ try: - rootcap = node_config.get_private_config(b"rootcap") + rootcap = node_config.get_private_config("rootcap") except MissingConfigEntry: return [] else: - return [client_node.create_node_from_uri(rootcap)] + return [client_node.create_node_from_uri(rootcap.encode("utf-8"))] def load_signing_key(path): diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index ce0c16ba..3b109f8c 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -28,6 +28,7 @@ IFoolscapStoragePlugin, IStorageServer, RIStorageServer, + IFilesystemNode, ) from challenge_bypass_ristretto import SigningKey from eliot.testing import LoggedMessage, capture_logging @@ -55,6 +56,7 @@ MatchesAll, MatchesListwise, MatchesStructure, + Not, ) from testtools.twistedsupport import succeeded from testtools.twistedsupport._deferred import extract_result @@ -67,7 +69,7 @@ from twisted.plugins.zkapauthorizer import storage_server -from .._plugin import load_signing_key +from .._plugin import load_signing_key, get_root_nodes from .._storage_client import IncorrectStorageServerReference from ..controller import DummyRedeemer, IssuerConfigurationMismatch, PaymentController from ..foolscap import RIPrivacyPassAuthorizedStorageServer @@ -351,7 +353,6 @@ class ClientPluginTests(TestCase): Tests for the plugin's implementation of ``IFoolscapStoragePlugin.get_storage_client``. """ - @given(tahoe_configs(), announcements()) def test_interface(self, get_config, announcement): """ @@ -640,6 +641,37 @@ def write_private_config(name, value): return create_client_from_config(config) + @given(tahoe_configs()) + def test_get_root_nodes_rootcap_present(self, get_config): + """ + ``get_root_nodes`` returns a ``list`` of one ``IFilesystemNode`` provider + derived from the contents of the *rootcap* private configuration. + """ + d = self._create(get_config, servers_yaml=None, rootcap=True) + client_node = extract_result(d) + roots = get_root_nodes(client_node, client_node.config) + self.assertThat( + roots, + MatchesAll( + Not(HasLength(0)), + AllMatch(Provides([IFilesystemNode])), + ), + ) + + @given(tahoe_configs()) + def test_get_root_nodes_rootcap_missing(self, get_config): + """ + ``get_root_nodes`` returns an empty ``list`` if there is no private + *rootcap* configuration. + """ + d = self._create(get_config, servers_yaml=None, rootcap=False) + client_node = extract_result(d) + roots = get_root_nodes(client_node, client_node.config) + self.assertThat( + roots, + Equals([]), + ) + @settings( deadline=None, ) From 1f134f956c27ae19f0172eb98ed08b0b53793a5a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:20:39 -0500 Subject: [PATCH 41/59] match the exact expected length --- src/_zkapauthorizer/tests/test_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 3b109f8c..0136da11 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -56,7 +56,6 @@ MatchesAll, MatchesListwise, MatchesStructure, - Not, ) from testtools.twistedsupport import succeeded from testtools.twistedsupport._deferred import extract_result @@ -353,6 +352,7 @@ class ClientPluginTests(TestCase): Tests for the plugin's implementation of ``IFoolscapStoragePlugin.get_storage_client``. """ + @given(tahoe_configs(), announcements()) def test_interface(self, get_config, announcement): """ @@ -653,7 +653,7 @@ def test_get_root_nodes_rootcap_present(self, get_config): self.assertThat( roots, MatchesAll( - Not(HasLength(0)), + HasLength(1), AllMatch(Provides([IFilesystemNode])), ), ) From 5d2670ea4c0524efe39b86ae1517c5573d100242 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:24:08 -0500 Subject: [PATCH 42/59] more general reasoning about special cases for provider choices --- default.nix | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/default.nix b/default.nix index 48bdab38..47247993 100644 --- a/default.nix +++ b/default.nix @@ -41,16 +41,12 @@ in # this another way, dependencies have undetected dependencies, easier # to just use the wheel. collections-extended = "wheel"; - # same as collections-extended isort = "wheel"; - # From nixpkgs or sdist, fails with - # cp: cannot stat 'benchmark/': No such file or directory - # cp: cannot stat 'tests/': No such file or directory + # The sdists for these packages have a different source/directory layout + # than the github archive the nixpkgs derivations expects to operate on + # so the sdists provider fails to build them. tomli = "wheel"; - - # repo re-org or something? - # find: ‘hypothesis-6.32.1/hypothesis-python’: No such file or directory hypothesis = "wheel"; }; in From 46fa124a5bbcea6967b4ecefef9a6fff0e787d43 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:33:50 -0500 Subject: [PATCH 43/59] replace typing comments with syntax --- src/_zkapauthorizer/_plugin.py | 6 ++- src/_zkapauthorizer/controller.py | 4 +- src/_zkapauthorizer/spending.py | 43 ++++++++------------- src/_zkapauthorizer/tests/storage_common.py | 25 ++++++------ src/_zkapauthorizer/tests/test_plugin.py | 4 +- 5 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/_zkapauthorizer/_plugin.py b/src/_zkapauthorizer/_plugin.py index 4777084b..95a45a43 100644 --- a/src/_zkapauthorizer/_plugin.py +++ b/src/_zkapauthorizer/_plugin.py @@ -25,7 +25,11 @@ import attr from allmydata.client import _Client -from allmydata.interfaces import IAnnounceableStorageServer, IFoolscapStoragePlugin, IFilesystemNode +from allmydata.interfaces import ( + IAnnounceableStorageServer, + IFilesystemNode, + IFoolscapStoragePlugin, +) from allmydata.node import MissingConfigEntry from challenge_bypass_ristretto import PublicKey, SigningKey from eliot import start_action diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index 70781129..de456a9e 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -99,10 +99,10 @@ class RedemptionResult(object): the redemption process. """ - unblinded_tokens = attr.ib( # type: List[UnblindedToken] + unblinded_tokens: List[UnblindedToken] = attr.ib( validator=attr.validators.instance_of(list), ) - public_key = attr.ib( # type: str + public_key: str = attr.ib( validator=attr.validators.instance_of(str), ) diff --git a/src/_zkapauthorizer/spending.py b/src/_zkapauthorizer/spending.py index 89625b56..e3af5af9 100644 --- a/src/_zkapauthorizer/spending.py +++ b/src/_zkapauthorizer/spending.py @@ -16,7 +16,7 @@ A module for logic controlling the manner in which ZKAPs are spent. """ -from typing import List, Tuple +from typing import Callable, List, Tuple import attr from zope.interface import Attribute, Interface, implementer @@ -112,26 +112,21 @@ class PassGroup(object): :ivar list[Pass] passes: The passes of which this group consists. """ - _message = attr.ib(validator=attr.validators.instance_of(bytes)) # type: bytes - _factory = attr.ib( - validator=attr.validators.provides(IPassFactory) - ) # type: IPassFactory - _tokens = attr.ib( + _message: bytes = attr.ib(validator=attr.validators.instance_of(bytes)) + _factory: IPassFactory = attr.ib(validator=attr.validators.provides(IPassFactory)) + _tokens: List[Tuple[UnblindedToken, Pass]] = attr.ib( validator=attr.validators.instance_of(list) - ) # type: List[Tuple[UnblindedToken, Pass]] + ) @property - def passes(self): - # type: () -> List[Pass] + def passes(self) -> List[Pass]: return list(pass_ for (unblinded_token, pass_) in self._tokens) @property - def unblinded_tokens(self): - # type: () -> List[UnblindedToken] + def unblinded_tokens(self) -> List[UnblindedToken]: return list(unblinded_token for (unblinded_token, pass_) in self._tokens) - def split(self, select_indices): - # type: (List[int]) -> (PassGroup, PassGroup) + def split(self, select_indices: List[int]) -> ("PassGroup", "PassGroup"): selected = [] unselected = [] for idx, t in enumerate(self._tokens): @@ -144,23 +139,19 @@ def split(self, select_indices): attr.evolve(self, tokens=unselected), ) - def expand(self, by_amount): - # type: (int) -> PassGroup + def expand(self, by_amount: int) -> "PassGroup": return attr.evolve( self, tokens=self._tokens + self._factory.get(self._message, by_amount)._tokens, ) - def mark_spent(self): - # type: () -> None + def mark_spent(self) -> None: self._factory._mark_spent(self.unblinded_tokens) - def mark_invalid(self, reason): - # type: () -> None + def mark_invalid(self, reason) -> None: self._factory._mark_invalid(reason, self.unblinded_tokens) - def reset(self): - # tye: () -> None + def reset(self) -> None: self._factory._reset(self.unblinded_tokens) @@ -172,12 +163,12 @@ class SpendingController(object): attempts when necessary. """ - get_unblinded_tokens = attr.ib() # type: (int) -> [UnblindedToken] - discard_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None - invalidate_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None - reset_unblinded_tokens = attr.ib() # type: ([UnblindedToken]) -> None + get_unblinded_tokens: Callable[[int], List[UnblindedToken]] = attr.ib() + discard_unblinded_tokens: Callable[[List[UnblindedToken]], None] = attr.ib() + invalidate_unblinded_tokens: Callable[[List[UnblindedToken]], None] = attr.ib() + reset_unblinded_tokens: Callable[[List[UnblindedToken]], None] = attr.ib() - tokens_to_passes = attr.ib() # type: (bytes, [UnblindedToken]) -> [Pass] + tokens_to_passes: Callable[[bytes, List[UnblindedToken]], List[Pass]] = attr.ib() @classmethod def for_store(cls, tokens_to_passes, store): diff --git a/src/_zkapauthorizer/tests/storage_common.py b/src/_zkapauthorizer/tests/storage_common.py index 62bfae30..a168395c 100644 --- a/src/_zkapauthorizer/tests/storage_common.py +++ b/src/_zkapauthorizer/tests/storage_common.py @@ -20,10 +20,10 @@ from itertools import islice from os import SEEK_CUR from struct import pack -from typing import Dict, List, Set +from typing import Callable, Dict, List, Set import attr -from challenge_bypass_ristretto import RandomToken +from challenge_bypass_ristretto import RandomToken, SigningKey from twisted.python.filepath import FilePath from zope.interface import implementer @@ -135,7 +135,7 @@ def whitebox_write_sparse_share(sharepath, version, size, leases, now): ) -def integer_passes(limit): +def integer_passes(limit: int) -> Callable[[bytes, int], List[int]]: """ :return: A function which can be used to get a number of passes. The function accepts a unicode request-binding message and an integer @@ -153,7 +153,9 @@ def get_passes(message, num_passes): return get_passes -def get_passes(message, count, signing_key): +def get_passes( + message: bytes, count: int, signing_key: SigningKey +) -> List[RandomToken]: """ :param bytes message: Request-binding message for PrivacyPass. @@ -219,16 +221,15 @@ class _PassFactory(object): via ``IPassGroup.reset``. """ - _get_passes = attr.ib() # type: (bytes, int) -> List[bytes] + _get_passes: Callable[[bytes, int], List[bytes]] = attr.ib() - returned = attr.ib(default=attr.Factory(list), init=False) # type: List[int] - in_use = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] - invalid = attr.ib(default=attr.Factory(dict), init=False) # type: Dict[int, str] - spent = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] - issued = attr.ib(default=attr.Factory(set), init=False) # type: Set[int] + returned: List[int] = attr.ib(default=attr.Factory(list), init=False) + in_use: Set[int] = attr.ib(default=attr.Factory(set), init=False) + invalid: Dict[int, str] = attr.ib(default=attr.Factory(dict), init=False) + spent: Set[int] = attr.ib(default=attr.Factory(set), init=False) + issued: Set[int] = attr.ib(default=attr.Factory(set), init=False) - def get(self, message, num_passes): - # type: (bytes, int) -> PassGroup + def get(self, message: bytes, num_passes: int) -> PassGroup: passes = [] if self.returned: passes.extend(self.returned[:num_passes]) diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 0136da11..01b56e4b 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -25,10 +25,10 @@ from allmydata.client import config_from_string, create_client_from_config from allmydata.interfaces import ( IAnnounceableStorageServer, + IFilesystemNode, IFoolscapStoragePlugin, IStorageServer, RIStorageServer, - IFilesystemNode, ) from challenge_bypass_ristretto import SigningKey from eliot.testing import LoggedMessage, capture_logging @@ -68,7 +68,7 @@ from twisted.plugins.zkapauthorizer import storage_server -from .._plugin import load_signing_key, get_root_nodes +from .._plugin import get_root_nodes, load_signing_key from .._storage_client import IncorrectStorageServerReference from ..controller import DummyRedeemer, IssuerConfigurationMismatch, PaymentController from ..foolscap import RIPrivacyPassAuthorizedStorageServer From f6591cb91e991a274c3e5eaecb3e2a0ab899b909 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:35:41 -0500 Subject: [PATCH 44/59] base64 encoded tokens are always ascii --- src/_zkapauthorizer/controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index de456a9e..663f65fa 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -506,7 +506,7 @@ def redeemWithCounter(self, voucher, counter, encoded_random_tokens): "redeemVoucher": voucher.number.decode("ascii"), "redeemCounter": counter, "redeemTokens": list( - token.encode_base64().decode("utf-8") + token.encode_base64().decode("ascii") for token in blinded_tokens ), } From 92ba74cea8c6f0ad3a50fe6ddd8be85db3ed1536 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:39:20 -0500 Subject: [PATCH 45/59] required_passes is content with dict_values now --- src/_zkapauthorizer/_storage_client.py | 2 +- src/_zkapauthorizer/storage_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index 6ebd3cce..1add7458 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -387,7 +387,7 @@ def add_lease( None, ) ).values() - num_passes = required_passes(self._pass_value, list(share_sizes)) + num_passes = required_passes(self._pass_value, share_sizes) result = yield call_with_passes( lambda passes: rref.callRemote( diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index 5313b315..34360810 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -254,7 +254,7 @@ def get_required_new_passes_for_mutable_write(pass_value, current_sizes, tw_vect """ current_passes = required_passes( pass_value, - list(current_sizes.values()), + current_sizes.values(), ) new_sizes = current_sizes.copy() From c621b053373f52a5d4333e999c0506e5c7189b64 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:43:15 -0500 Subject: [PATCH 46/59] use f"" to simplify some string construction --- src/_zkapauthorizer/tests/strategies.py | 4 ++-- src/_zkapauthorizer/tests/test_client_resource.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/_zkapauthorizer/tests/strategies.py b/src/_zkapauthorizer/tests/strategies.py index d8b48e24..d468030d 100644 --- a/src/_zkapauthorizer/tests/strategies.py +++ b/src/_zkapauthorizer/tests/strategies.py @@ -142,7 +142,7 @@ def tahoe_config_texts(storage_client_plugins, shares): def merge_shares(shares, the_rest): for (k, v) in zip(("needed", "happy", "total"), shares): if v is not None: - the_rest["shares." + k] = "{}".format(v) + the_rest["shares." + k] = f"{v}" return the_rest client_section = builds( @@ -249,7 +249,7 @@ def server_configurations(signing_key_path): { "pass-value": # The configuration is ini so everything is always a byte string! - integers(min_value=1).map(lambda v: "{}".format(v).encode("ascii")), + integers(min_value=1).map(lambda v: f"{v}".encode("ascii")), } ), just({}), diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index e72c0b92..5cedaf48 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -331,7 +331,7 @@ def test_get_token_count(self, token_count): token_config = {} else: expected_count = token_count - token_config = {"default-token-count": "{}".format(expected_count)} + token_config = {"default-token-count": f"{expected_count}"} config_text = config_string_from_sections( [ @@ -567,7 +567,7 @@ def test_get(self, get_config, api_auth_token, voucher, extra_tokens): ) self.addDetail( "requesting result", - text_content("{}".format(vars(requesting.result))), + text_content(f"{vars(requesting.result)}"), ) self.assertThat( requesting, @@ -615,7 +615,7 @@ def test_get_limit(self, get_config, api_auth_token, voucher, extra_tokens, limi ) self.addDetail( "requesting result", - text_content("{}".format(vars(requesting.result))), + text_content(f"{vars(requesting.result)}"), ) self.assertThat( requesting, @@ -670,7 +670,7 @@ def test_get_position( ) self.addDetail( "requesting result", - text_content("{}".format(vars(requesting.result))), + text_content(f"{vars(requesting.result)}"), ) self.assertThat( requesting, @@ -958,7 +958,7 @@ def test_put_voucher(self, get_config, api_auth_token, voucher): ) self.addDetail( "requesting result", - text_content("{}".format(vars(requesting.result))), + text_content(f"{vars(requesting.result)}"), ) self.assertThat( requesting, @@ -990,7 +990,7 @@ def test_put_invalid_body(self, get_config, api_auth_token, body): ) self.addDetail( "requesting result", - text_content("{}".format(vars(requesting.result))), + text_content(f"{vars(requesting.result)}"), ) self.assertThat( requesting, From ea07a480a608ad2ba922e70079e4225ee6e815f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:43:42 -0500 Subject: [PATCH 47/59] cleanup stray keystroke --- src/_zkapauthorizer/tests/test_client_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_zkapauthorizer/tests/test_client_resource.py b/src/_zkapauthorizer/tests/test_client_resource.py index 5cedaf48..c8af0774 100644 --- a/src/_zkapauthorizer/tests/test_client_resource.py +++ b/src/_zkapauthorizer/tests/test_client_resource.py @@ -456,7 +456,7 @@ def maybe_extra_tokens(): return one_of( just(None), # If we do, we can't have fewer than the number of redemption groups - # which we don't know until we're further inside the test. oSo supply + # which we don't know until we're further inside the test. So supply # an amount to add to that, in the case where we have tokens at all. integers(min_value=0, max_value=100), ) From f7f0c933b341ecf5fbb0d08e8306f235df0781d4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:45:46 -0500 Subject: [PATCH 48/59] Use FilePath instead of os.path --- src/_zkapauthorizer/tests/test_plugin.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/_zkapauthorizer/tests/test_plugin.py b/src/_zkapauthorizer/tests/test_plugin.py index 01b56e4b..27e2a5a6 100644 --- a/src/_zkapauthorizer/tests/test_plugin.py +++ b/src/_zkapauthorizer/tests/test_plugin.py @@ -16,7 +16,6 @@ Tests for the Tahoe-LAFS plugin. """ -import os.path from datetime import timedelta from functools import partial from io import StringIO @@ -623,9 +622,8 @@ def _create(self, get_config, servers_yaml, rootcap): # and unicode in an os.path.join() call that always fails with a # TypeError. def write_private_config(name, value): - privname = os.path.join(config._basedir, u"private", name) - with open(privname, "wb") as f: - f.write(value) + privpath = FilePath(config._basedir).descendant([u"private", name]) + privpath.setContent(value) if servers_yaml is not None: # Provide it a statically configured server to connect to. From 14956bf664521e13ac5edd62406485f83182158f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:53:12 -0500 Subject: [PATCH 49/59] get rid of past.builtins.long usage --- src/_zkapauthorizer/model.py | 7 +++---- src/_zkapauthorizer/resource.py | 3 +-- src/_zkapauthorizer/tests/test_pricecalculator.py | 3 +-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index df00eb63..fb67ec74 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -25,7 +25,6 @@ import attr from aniso8601 import parse_datetime -from past.builtins import long from twisted.logger import Logger from twisted.python.filepath import FilePath from zope.interface import Interface, implementer @@ -891,7 +890,7 @@ class RandomToken(object): def _counter_attribute(): return attr.ib( validator=attr.validators.and_( - attr.validators.instance_of((int, long)), + attr.validators.instance_of(int), greater_than(-1), ), ) @@ -952,7 +951,7 @@ class Redeemed(object): """ finished = attr.ib(validator=attr.validators.instance_of(datetime)) - token_count = attr.ib(validator=attr.validators.instance_of((int, long))) + token_count = attr.ib(validator=attr.validators.instance_of(int)) def should_start_redemption(self): return False @@ -1051,7 +1050,7 @@ class Voucher(object): expected_tokens = attr.ib( validator=attr.validators.optional( attr.validators.and_( - attr.validators.instance_of((int, long)), + attr.validators.instance_of(int), greater_than(0), ), ), diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 93993921..62c3f3ca 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -25,7 +25,6 @@ from json import load, loads from sys import maxsize -from past.builtins import long from twisted.logger import Logger from twisted.web.http import BAD_REQUEST from twisted.web.resource import ErrorPage, IResource, NoResource, Resource @@ -256,7 +255,7 @@ def render_POST(self, request): ) if not isinstance(sizes, list) or not all( - isinstance(size, (int, long)) and size >= 0 for size in sizes + isinstance(size, int) and size >= 0 for size in sizes ): request.setResponseCode(BAD_REQUEST) return dumps_utf8( diff --git a/src/_zkapauthorizer/tests/test_pricecalculator.py b/src/_zkapauthorizer/tests/test_pricecalculator.py index 2d447e3f..c1652f2b 100644 --- a/src/_zkapauthorizer/tests/test_pricecalculator.py +++ b/src/_zkapauthorizer/tests/test_pricecalculator.py @@ -21,7 +21,6 @@ from hypothesis import given from hypothesis.strategies import integers, lists, tuples -from past.builtins import long from testtools import TestCase from testtools.matchers import Equals, GreaterThan, IsInstance, MatchesAll @@ -156,7 +155,7 @@ def test_positive_integer_price(self, pass_value, parameters, file_sizes): self.assertThat( price, MatchesAll( - IsInstance((int, long)), + IsInstance(int), GreaterThan(0), ), ) From 34e05c20e39e7cd694fa9287fc9376466df4da1d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:57:52 -0500 Subject: [PATCH 50/59] drop the silly decode/encode dance for all the binding messages --- src/_zkapauthorizer/_storage_client.py | 6 +++--- src/_zkapauthorizer/_storage_server.py | 6 +++--- src/_zkapauthorizer/storage_common.py | 7 ++++--- .../tests/test_storage_protocol.py | 4 ++-- .../tests/test_storage_server.py | 20 +++++++++---------- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/_zkapauthorizer/_storage_client.py b/src/_zkapauthorizer/_storage_client.py index 1add7458..e8fe8c46 100644 --- a/src/_zkapauthorizer/_storage_client.py +++ b/src/_zkapauthorizer/_storage_client.py @@ -355,7 +355,7 @@ def allocate_buckets( num_passes, partial( self._get_passes, - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), ), partial(self._spend_for_allocate_buckets, allocated_size), ) @@ -398,7 +398,7 @@ def add_lease( cancel_secret, ), num_passes, - partial(self._get_passes, add_lease_message(storage_index).encode("utf-8")), + partial(self._get_passes, add_lease_message(storage_index)), ) returnValue(result) @@ -503,7 +503,7 @@ def slot_testv_and_readv_and_writev( num_passes, partial( self._get_passes, - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), ), ) returnValue(result) diff --git a/src/_zkapauthorizer/_storage_server.py b/src/_zkapauthorizer/_storage_server.py index b2e45e2a..aa11097a 100644 --- a/src/_zkapauthorizer/_storage_server.py +++ b/src/_zkapauthorizer/_storage_server.py @@ -270,7 +270,7 @@ def remote_allocate_buckets( storage for immutable shares if they present valid passes. """ validation = _ValidationResult.validate_passes( - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), passes, self._signing_key, ) @@ -353,7 +353,7 @@ def remote_add_lease(self, passes, storage_index, *a, **kw): duration of share storage if they present valid passes. """ validation = _ValidationResult.validate_passes( - add_lease_message(storage_index).encode("utf-8"), + add_lease_message(storage_index), passes, self._signing_key, ) @@ -461,7 +461,7 @@ def _slot_testv_and_readv_and_writev( # Check passes for cryptographic validity. validation = _ValidationResult.validate_passes( - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), passes, self._signing_key, ) diff --git a/src/_zkapauthorizer/storage_common.py b/src/_zkapauthorizer/storage_common.py index 34360810..db04c1f6 100644 --- a/src/_zkapauthorizer/storage_common.py +++ b/src/_zkapauthorizer/storage_common.py @@ -17,6 +17,7 @@ """ from base64 import b64encode +from typing import Callable import attr from pyutil.mathutil import div_ceil @@ -46,12 +47,12 @@ class MorePassesRequired(Exception): signature_check_failed = attr.ib(converter=frozenset) -def _message_maker(label): +def _message_maker(label: str) -> Callable[[str], bytes]: def make_message(storage_index): return "{label} {storage_index}".format( label=label, - storage_index=b64encode(storage_index).decode("utf-8"), - ) + storage_index=b64encode(storage_index).decode("ascii"), + ).encode("ascii") return make_message diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index 3a218042..c225487c 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -235,14 +235,14 @@ def test_rejected_passes_reported( # Make some passes with a key untrusted by the server. bad_passes = get_passes( - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), len(bad_pass_indexes), random_signing_key(), ) # Make some passes with a key trusted by the server. good_passes = get_passes( - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), num_passes - len(bad_passes), self.signing_key, ) diff --git a/src/_zkapauthorizer/tests/test_storage_server.py b/src/_zkapauthorizer/tests/test_storage_server.py index d4829914..73a0001f 100644 --- a/src/_zkapauthorizer/tests/test_storage_server.py +++ b/src/_zkapauthorizer/tests/test_storage_server.py @@ -241,7 +241,7 @@ def test_allocate_buckets_fails_without_enough_passes(self): renew_secret = b"x" * 32 cancel_secret = b"y" * 32 valid_passes = get_passes( - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), required_passes - 1, self.signing_key, ) @@ -343,7 +343,7 @@ def _test_extend_mutable_fails_without_passes( ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), required_pass_count, self.signing_key, ) @@ -479,7 +479,7 @@ def test_mutable_new_length_rejected( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), required_pass_count, self.signing_key, ) @@ -548,7 +548,7 @@ def test_add_lease_fails_without_passes( # Attempt the lease operation with one fewer pass than is required. passes = get_passes( - add_lease_message(storage_index).encode("utf-8"), + add_lease_message(storage_index), required_count - 1, self.signing_key, ) @@ -612,7 +612,7 @@ def test_mutable_share_sizes( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(slot).encode("utf-8"), + slot_testv_and_readv_and_writev_message(slot), required_pass_count, self.signing_key, ) @@ -676,7 +676,7 @@ def test_mutable_spending_metrics( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), num_passes, self.signing_key, ) @@ -737,7 +737,7 @@ def test_mutable_failure_spending_metrics( tw_vectors, ) valid_passes = get_passes( - slot_testv_and_readv_and_writev_message(storage_index).encode("utf-8"), + slot_testv_and_readv_and_writev_message(storage_index), num_passes, self.signing_key, ) @@ -818,7 +818,7 @@ def test_immutable_spending_metrics( [size] * len(new_sharenums - existing_sharenums), ) valid_passes = get_passes( - allocate_buckets_message(storage_index).encode("utf-8"), + allocate_buckets_message(storage_index), num_passes, self.signing_key, ) @@ -885,7 +885,7 @@ def test_add_lease_metrics( self.storage_server._pass_value, [allocated_size] * len(sharenums) ) valid_passes = get_passes( - add_lease_message(storage_index).encode("utf-8"), + add_lease_message(storage_index), num_passes, self.signing_key, ) @@ -946,7 +946,7 @@ def test_add_lease_metrics_on_failure( self.storage_server._pass_value, [allocated_size] * len(sharenums) ) valid_passes = get_passes( - add_lease_message(storage_index).encode("utf-8"), + add_lease_message(storage_index), num_passes, self.signing_key, ) From 22b0014df108becda544dfc8cf0e6bdde608678a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 08:59:14 -0500 Subject: [PATCH 51/59] fixup comment in new test --- src/_zkapauthorizer/tests/test_storage_protocol.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/_zkapauthorizer/tests/test_storage_protocol.py b/src/_zkapauthorizer/tests/test_storage_protocol.py index c225487c..ceb34deb 100644 --- a/src/_zkapauthorizer/tests/test_storage_protocol.py +++ b/src/_zkapauthorizer/tests/test_storage_protocol.py @@ -1117,9 +1117,10 @@ def equal_test_vector(data_vector): ) self.assertThat(d, is_successful_write()) - # Write some new data to with a correct test vector. We can only be - # sure we know data from the last element of the test vector since - # earlier elements may have been overwritten. + # Write some new data with a correct test vector. We can only be sure + # we know data from the last element of the test vector since earlier + # elements may have been overwritten so only use that last element in + # our test vector. d = write( { sharenum: ( From 1ebff29fa0b78ef4e827432021cdb04e77e0cdda Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 10:59:41 -0500 Subject: [PATCH 52/59] Use auto_attribus on a couple exception types However, we still cannot be frozen (because of Failure.cleanFailure) and cannot use attrs exception support (because we want value-based comparison). --- src/_zkapauthorizer/controller.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index 663f65fa..88234060 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -52,14 +52,14 @@ RETRY_INTERVAL = timedelta(milliseconds=1000) -@attr.s +@attr.s(auto_attribs=True) class UnexpectedResponse(Exception): """ The issuer responded in an unexpected and unhandled way. """ - code = attr.ib() - body = attr.ib() + code: int = attr.ib() + body: bytes = attr.ib() class AlreadySpent(Exception): @@ -77,7 +77,7 @@ class Unpaid(Exception): """ -@attr.s +@attr.s(auto_attribs=True) class UnrecognizedFailureReason(Exception): """ An attempt was made to redeem a voucher and the response contained an unknown reason. @@ -85,7 +85,7 @@ class UnrecognizedFailureReason(Exception): The redemption attempt may be automatically retried at some point. """ - response = attr.ib() + response: dict = attr.ib() @attr.s From 67072baa3b62a4bed2c87f84cff60019b6e9b8d3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 11:35:10 -0500 Subject: [PATCH 53/59] Compare dict keys directly instead of via a set conversion Co-authored-by: Tom Prince --- src/_zkapauthorizer/resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_zkapauthorizer/resource.py b/src/_zkapauthorizer/resource.py index 62c3f3ca..2a372492 100644 --- a/src/_zkapauthorizer/resource.py +++ b/src/_zkapauthorizer/resource.py @@ -403,7 +403,7 @@ def render_PUT(self, request): payload = loads(request.content.read()) except Exception: return bad_request("json request body required").render(request) - if set(payload) != {"voucher"}: + if payload.keys() != {"voucher"}: return bad_request( "request object must have exactly one key: 'voucher'" ).render(request) From 39985b168b171f6e251824121a476074e7e2085e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 11:49:25 -0500 Subject: [PATCH 54/59] It's already a text-mode FilePath, right? TempDir.join(unicode) returns unicode TempDir.join(bytes) raises an Exception FilePath(unicode) produces a text-mode FilePath StorageServer.__init__ raises an exception if passed bytes So we can just rely on self.tempdir.path being the correct type already and even if it weren't StorageServer would immediately tell us. --- src/_zkapauthorizer/tests/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_zkapauthorizer/tests/fixtures.py b/src/_zkapauthorizer/tests/fixtures.py index 9ed90e1f..54d27b96 100644 --- a/src/_zkapauthorizer/tests/fixtures.py +++ b/src/_zkapauthorizer/tests/fixtures.py @@ -49,7 +49,7 @@ class AnonymousStorageServer(Fixture): def _setUp(self): self.tempdir = FilePath(self.useFixture(TempDir()).join(u"storage")) self.storage_server = StorageServer( - self.tempdir.asTextMode().path, + self.tempdir.path, b"x" * 20, clock=self.clock, ) From 4ddd84c1542e92e511f8395ca617d207c191deb2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 18:33:55 -0500 Subject: [PATCH 55/59] Pin Tahoe-LAFS exactly because we use an unofficial/unstable interface --- setup.cfg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8fd1f296..5f7b828e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,10 @@ install_requires = # incompatible with Tahoe-LAFS'. So duplicate them here (the ones that # have been observed to cause problems). Twisted[tls,conch] >= 19.10.0 - tahoe-lafs >=1.17,<1.18 + + # Tahoe has no stable Python API but we use its Python API so there's + # basically no wiggle room here. + tahoe-lafs == 1.17.0 treq pyutil prometheus-client From 4f59ff8c639eb8bc2c0ff1229563e426a8f3b7a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jan 2022 19:39:25 -0500 Subject: [PATCH 56/59] widen the required range to allow the Nix package --- setup.cfg | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5f7b828e..768163ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -43,8 +43,9 @@ install_requires = Twisted[tls,conch] >= 19.10.0 # Tahoe has no stable Python API but we use its Python API so there's - # basically no wiggle room here. - tahoe-lafs == 1.17.0 + # basically no wiggle room here. We still use a (really tiny) range + # because our Nix packaging provides a Tahoe-LAFS with a .postNNN version. + tahoe-lafs >=1.17.0,<1.17.1 treq pyutil prometheus-client From 209c83c4f60447351e1343d98652459f2a3d0d45 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Jan 2022 21:14:53 -0500 Subject: [PATCH 57/59] link to twisted tickets related to this --- src/_zkapauthorizer/controller.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/_zkapauthorizer/controller.py b/src/_zkapauthorizer/controller.py index 88234060..f58192bb 100644 --- a/src/_zkapauthorizer/controller.py +++ b/src/_zkapauthorizer/controller.py @@ -52,6 +52,10 @@ RETRY_INTERVAL = timedelta(milliseconds=1000) +# It would be nice to have frozen exception types but Failure.cleanFailure +# interacts poorly with these. +# https://twistedmatrix.com/trac/ticket/9641 +# https://twistedmatrix.com/trac/ticket/9771 @attr.s(auto_attribs=True) class UnexpectedResponse(Exception): """ From 75811491b864acb3ec5a9457203b43bb9bf12a44 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Jan 2022 21:31:24 -0500 Subject: [PATCH 58/59] Use __future__.annotations --- src/_zkapauthorizer/spending.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/_zkapauthorizer/spending.py b/src/_zkapauthorizer/spending.py index e3af5af9..f718ac00 100644 --- a/src/_zkapauthorizer/spending.py +++ b/src/_zkapauthorizer/spending.py @@ -16,6 +16,8 @@ A module for logic controlling the manner in which ZKAPs are spent. """ +from __future__ import annotations + from typing import Callable, List, Tuple import attr @@ -126,7 +128,7 @@ def passes(self) -> List[Pass]: def unblinded_tokens(self) -> List[UnblindedToken]: return list(unblinded_token for (unblinded_token, pass_) in self._tokens) - def split(self, select_indices: List[int]) -> ("PassGroup", "PassGroup"): + def split(self, select_indices: List[int]) -> (PassGroup, PassGroup): selected = [] unselected = [] for idx, t in enumerate(self._tokens): @@ -139,7 +141,7 @@ def split(self, select_indices: List[int]) -> ("PassGroup", "PassGroup"): attr.evolve(self, tokens=unselected), ) - def expand(self, by_amount: int) -> "PassGroup": + def expand(self, by_amount: int) -> PassGroup: return attr.evolve( self, tokens=self._tokens + self._factory.get(self._message, by_amount)._tokens, From cb771a94f5fee441caa216a39dca2a77cbb50b2c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Jan 2022 21:32:55 -0500 Subject: [PATCH 59/59] remove one more unnecessary asBytesMode call --- src/_zkapauthorizer/model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/_zkapauthorizer/model.py b/src/_zkapauthorizer/model.py index fb67ec74..726be236 100644 --- a/src/_zkapauthorizer/model.py +++ b/src/_zkapauthorizer/model.py @@ -96,10 +96,9 @@ def open_and_initialize(path, connect=None): except OSError as e: raise StoreOpenError(e) - dbfile = path.asBytesMode().path try: conn = connect( - dbfile, + path.path, isolation_level="IMMEDIATE", ) except OperationalError as e: