Port to Python 3

.circleci/config.yml

@@ -71,14 +71,7 @@ jobs:
       - run:
           name: "Get Pip"
           command: |
-            # The CircleCI macOS environment has curl and Python but does not
-            # have pip.  So, for starters, use curl and Python to get pip.
-            if [ "<< parameters.py-version >>" == "2.7" ]; then
-
-              curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py
-            else
-              curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-            fi
+            curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
             python<< parameters.py-version >> get-pip.py
 
       - run:
@@ -136,6 +129,8 @@ jobs:
 
   linux-tests: &LINUX_TESTS
     parameters:
+      py-version:
+        type: "string"
       tahoe-lafs-source:
         # The name of a niv source in nix/sources.json which corresponds to
         # a Tahoe-LAFS version. This is the version that will be declared as a
@@ -209,7 +204,8 @@ jobs:
             nix-build tests.nix \
               --argstr hypothesisProfile ci \
               --arg collectCoverage true \
-              --argstr tahoe-lafs-source << parameters.tahoe-lafs-source >>
+              --argstr tahoe-lafs-source << parameters.tahoe-lafs-source >> \
+              --argstr python python<< parameters.py-version >>
 
       - run:
           name: "Push to Cachix"
@@ -241,18 +237,17 @@ workflows:
     jobs:
     - "documentation"
     - "linux-tests":
-        matrix:
-          parameters:
-            tahoe-lafs-source:
-            - "tahoe-lafs"
+        name: "Linux tests python 3.9"
+        py-version: "39"
+        tahoe-lafs-source: "tahoe-lafs"
+
+    # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
+    - "macos-tests":
+        name: "macOS tests python 3.8 xcode 11.7.0"
+        py-version: "3.8"
+        xcode-version: "11.7.0"
 
     - "macos-tests":
-        matrix:
-          parameters:
-            py-version:
-            - "2.7"
-
-            xcode-version:
-            # https://circleci.com/docs/2.0/testing-ios/#supported-xcode-versions
-            - "12.3.0"
-            - "11.7.0"
+        name: "macOS tests python 3.9 xcode 12.3.0"
+        py-version: "3.9"
+        xcode-version: "12.3.0"

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "2.7"
+          - "3.9"
 
     steps:
     # Avoid letting Windows newlines confusing milksnake.

default.nix

@@ -1,17 +1,19 @@
 let
   sources = import nix/sources.nix;
 in
-{ pkgs ? import sources.release2105 {}
+{ pkgs ? import sources.release2111 { }
 , pypiData ? sources.pypi-deps-db
-, mach-nix ? import sources.mach-nix { inherit pkgs pypiData; }
+, python ? "python39"
+, mach-nix ? import sources.mach-nix { inherit pkgs pypiData python; }
 , tahoe-lafs-source ? "tahoe-lafs"
 , tahoe-lafs-repo ? sources.${tahoe-lafs-source}
+, ...
 }:
   let
     lib = pkgs.lib;
-    python = "python27";
     providers = {
       _default = "sdist,nixpkgs,wheel";
+
       # mach-nix doesn't provide a good way to depend on mach-nix packages,
       # so we get it as a nixpkgs dependency from an overlay. See below for
       # details.
@@ -34,6 +36,22 @@ in
       # The version of Klein we get doesn't need / can't have the patch that
       # comes from the nixpkgs derivation mach-nix picks up from 21.05.
       klein = "wheel";
+
+      # - has an undetected poetry dependency and when trying to work around
+      #   this another way, dependencies have undetected dependencies, easier
+      #   to just use the wheel.
+      collections-extended = "wheel";
+      # same as collections-extended
+      isort = "wheel";
+
+      # From nixpkgs or sdist, fails with
+      # cp: cannot stat 'benchmark/': No such file or directory
+      # cp: cannot stat 'tests/': No such file or directory
+      tomli = "wheel";
+
+      # repo re-org or something?
+      # find: ‘hypothesis-6.32.1/hypothesis-python’: No such file or directory
+      hypothesis = "wheel";
     };
   in
     rec {
@@ -52,12 +70,21 @@ in
         # going on and discover the real version specified by `src` below.
         version = "1.17.0.post999";
         # See https://github.com/DavHau/mach-nix/issues/190
-        requirementsExtra = ''
+        requirementsExtra =
+          ''
+          # See https://github.com/DavHau/mach-nix/issues/190
           pyrsistent < 0.17
-          foolscap == 0.13.1
           configparser
           eliot
-        '';
+          foolscap >= 21.7.0
+
+          # undetected cryptography build dependency
+          # https://github.com/DavHau/mach-nix/issues/305
+          setuptools_rust
+          # undetected tomli build dependency
+          # probably same underlying cause as cryptography issue
+          flit_core
+          '';
         postPatch = ''
           cat > src/allmydata/_version.py <<EOF
           # This _version.py is generated by nix.

nix/sources.json

@@ -29,10 +29,10 @@
         "homepage": "",
         "owner": "DavHau",
         "repo": "pypi-deps-db",
-        "rev": "96d01556b4597c022647acbf8c3b58d2a99bc963",
-        "sha256": "0s6ll2hi40gj6mp2zdg7w3dq17g381gnfkm390mqgp574lmbq6yw",
+        "rev": "856d67ab093a68425c3896ec2961cea3b95ae93f",
+        "sha256": "0a7avm4xvm454gxy4dq0fc19j3f9ik8gf3kjpsqhszfkamrn9y0p",
         "type": "tarball",
-        "url": "https://github.com/DavHau/pypi-deps-db/archive/96d01556b4597c022647acbf8c3b58d2a99bc963.tar.gz",
+        "url": "https://github.com/DavHau/pypi-deps-db/archive/856d67ab093a68425c3896ec2961cea3b95ae93f.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "release2105": {
@@ -41,6 +41,18 @@
         "url": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz",
         "url_template": "https://releases.nixos.org/nixos/21.05/nixos-21.05.3740.ce7a1190a0f/nixexprs.tar.xz"
     },
+    "release2111": {
+        "branch": "release-21.11",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d887ac7aee92e8fc54dde9060d60d927afae9d69",
+        "sha256": "1bpgfv45b1yvrgpwdgc4fm4a6sav198yd41bsrvlmm3jn2wi6qx5",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/d887ac7aee92e8fc54dde9060d60d927afae9d69.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
     "tahoe-lafs": {
         "branch": "master",
         "description": "The Tahoe-LAFS decentralized secure filesystem.",

setup.cfg

@@ -10,8 +10,8 @@ keywords = tahoe-lafs, storage, privacy, cryptography
 license = Apache 2.0
 classifiers =
     Framework :: Twisted
-    Programming Language :: Python :: 2
-    Programming Language :: Python :: 2.7
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.9
 author = PrivateStorage.io, LLC
 maintainer = PrivateStorage.io, LLC
 home-page = https://privatestorage.io/
@@ -34,7 +34,7 @@ packages =
 install_requires =
     attrs
     zope.interface
-    eliot
+    eliot >= 1.11,<2
     aniso8601
     python-challenge-bypass-ristretto
     # The pip resolver sometimes finds treq's dependencies first and these are
@@ -55,3 +55,10 @@ install_requires =
 
 [options.extras_require]
 test = coverage; fixtures; testtools; hypothesis
+
+[flake8]
+# Enforce all pyflakes constraints, and also prohibit tabs for indentation.
+# Reference:
+#   https://flake8.pycqa.org/en/latest/user/error-codes.html
+#   https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
+select = F, W191

shell.nix

@@ -3,12 +3,34 @@
 { ... }@args:
 let
   tests = import ./tests.nix args;
-  inherit (tests) pkgs;
+  inherit (tests) privatestorage lint-python;
+  inherit (privatestorage) pkgs mach-nix tahoe-lafs zkapauthorizer;
+
+  python-env = mach-nix.mkPython {
+    inherit (zkapauthorizer.meta.mach-nix) python providers;
+    overridesPre = [
+      (
+        self: super: {
+          inherit tahoe-lafs;
+        }
+      )
+    ];
+    requirements =
+      ''
+      ${builtins.readFile ./requirements/test.in}
+      ${zkapauthorizer.requirements}
+      '';
+  };
 in
 pkgs.mkShell {
+  # Avoid leaving .pyc all over the source tree when manually triggering tests
+  # runs.
+  PYTHONDONTWRITEBYTECODE = "1";
+
   buildInputs = [
-    tests.python
-    tests.lint-python
-    pkgs.niv
+    # Provide the linting tools for interactive usage.
+    lint-python
+    # Supply all of the runtime and testing dependencies.
+    python-env
   ];
 }

src/_zkapauthorizer/_base64.py

@@ -16,8 +16,6 @@
 This module implements base64 encoding-related functionality.
 """
 
-from __future__ import absolute_import
-
 from base64 import b64decode as _b64decode
 from binascii import Error
 from re import compile as _compile

src/_zkapauthorizer/_json.py

@@ -0,0 +1,23 @@
+# Copyright 2022 PrivateStorage.io, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from json import dumps as _dumps
+from typing import Any
+
+
+def dumps_utf8(o: Any) -> bytes:
+    """
+    Serialize an object to a UTF-8-encoded JSON byte string.
+    """
+    return _dumps(o).encode("utf-8")

src/_zkapauthorizer/_plugin.py

@@ -17,21 +17,15 @@
 Tahoe-LAFS.
 """
 
-from __future__ import absolute_import
-
 import random
 from datetime import datetime
 from functools import partial
+from typing import Callable, List
 from weakref import WeakValueDictionary
 
-try:
-    from typing import Callable
-except ImportError:
-    pass
-
 import attr
 from allmydata.client import _Client
-from allmydata.interfaces import IAnnounceableStorageServer, IFoolscapStoragePlugin
+from allmydata.interfaces import IAnnounceableStorageServer, IFoolscapStoragePlugin, IFilesystemNode
 from allmydata.node import MissingConfigEntry
 from challenge_bypass_ristretto import PublicKey, SigningKey
 from eliot import start_action
@@ -291,7 +285,7 @@ def get_now():
         get_now=get_now,
     )
     last_run_path = FilePath(
-        node_config.get_private_path(b"last-lease-maintenance-run")
+        node_config.get_private_path(u"last-lease-maintenance-run")
     )
     # Create the service to periodically run the lease maintenance operation.
     return lease_maintenance_service(
@@ -303,13 +297,16 @@ def get_now():
     )
 
 
-def get_root_nodes(client_node, node_config):
+def get_root_nodes(client_node, node_config) -> List[IFilesystemNode]:
+    """
+    Get the configured starting points for lease maintenance traversal.
+    """
     try:
-        rootcap = node_config.get_private_config(b"rootcap")
+        rootcap = node_config.get_private_config("rootcap")
     except MissingConfigEntry:
         return []
     else:
-        return [client_node.create_node_from_uri(rootcap)]
+        return [client_node.create_node_from_uri(rootcap.encode("utf-8"))]
 
 
 def load_signing_key(path):

src/_zkapauthorizer/_stack.py

@@ -36,7 +36,12 @@ def less_limited_stack():
     More precisely, the soft stack limit is raised to the hard limit.
     """
     soft, hard = getrlimit(RLIMIT_STACK)
-    # We can raise the soft limit to the hard limit and no higher.
-    setrlimit(RLIMIT_STACK, (hard, hard))
-    yield
-    setrlimit(RLIMIT_STACK, (soft, hard))
+    try:
+        # We can raise the soft limit to the hard limit and no higher.
+        setrlimit(RLIMIT_STACK, (hard, hard))
+    except ValueError:
+        # Well, not on macOS: https://bugs.python.org/issue34602
+        yield
+    else:
+        yield
+        setrlimit(RLIMIT_STACK, (soft, hard))