CI: test reliability

gevent worker on OmniOS py3.12/gcc14 fails with:
AttributeError: module 'threading' has no attribute 'get_native_id'

skip slow benchmark on PyPy

.github/workflows/illumos.yml

@@ -36,8 +36,9 @@ jobs:
           # /tmp/.nginx must exist because nginx will not create configured tmp
           # build-essential shall point to suitable gcc13/gcc14/..
           # TODO: replace python-MN with version-agnostic alias
+          # release: "r151048"
           prepare: |
-            pkg install pip-312 python-312 sqlite-3 nginx build-essential
+            pkg install pip-312 python-312 sqlite-3 nginx gcc14
           usesh: true
           copyback: false
           run: |

.github/workflows/tox.yml

@@ -6,6 +6,8 @@ on:
   pull_request:
     branches:
       - master
+  workflow_dispatch:
+  # allow manual trigger
 permissions:
   contents: read # to fetch code (actions/checkout)
 env:
@@ -64,7 +66,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo systemctl mask nginx.service
-          sudo apt install nginx openssl
+          sudo apt install nginx openssl wrk
       - name: Install Dependencies
         run: |
           python -m pip install --upgrade pip

tests/support_subprocess.py

@@ -63,7 +63,9 @@ def {APP_FUNC_NAME}(environ, start_response):
   default_type application/octet-stream;
   access_log /dev/stdout combined;
   upstream upstream_gunicorn {{
-    server {gunicorn_upstream} fail_timeout=0;
+    # max_fails=0 prevents nginx from assuming unavailable
+    #  .. which is nowadays (reasonably) ignored for single server
+    server {gunicorn_upstream} max_fails=0;
   }}
 
   server {{ listen {server_bind} default_server; return 400; }}
@@ -146,8 +148,11 @@ def __exit__(self, *exc):
             self.send_signal(self.EXIT_SIGNAL)
             try:
                 stdout, stderr = self.communicate(timeout=1)
-                # assert stdout[-512:] == b"", stdout
-                logger.debug(f"stdout not empty on shutdown, sample: {stdout[-512:]!r}")
+                if stdout:
+                    logger.debug(
+                        f"stdout not empty on shutdown, sample: {stdout[-512:]!r}"
+                    )
+                assert stdout[-512:] == b"", stdout
             except subprocess.TimeoutExpired:
                 pass
             # only helpful for diagnostics. we are shutting down unexpected
@@ -166,7 +171,7 @@ def read_stdio(self, *, timeout_sec, wait_for_keyword, expect=None, stderr=False
         buf = ["", ""]
         seen_keyword = 0
         unseen_keywords = list(expect or [])
-        poll_per_second = 20
+        poll_per_second = 30
         assert key in {0, 1}, key
         assert self.stdout is not None  # this helps static type checkers
         assert self.stderr is not None  # this helps static type checkers
@@ -303,7 +308,7 @@ def __init__(
 
 def generate_dummy_ssl_cert(cert_path, key_path):
     # dummy self-signed cert
-    subprocess.check_output(
+    subprocess.check_call(
         [
             CMD_OPENSSL,
             "req",
@@ -329,7 +334,11 @@ def generate_dummy_ssl_cert(cert_path, key_path):
             "-out",
             "%s" % (cert_path),
         ],
+        stdin=subprocess.DEVNULL,
+        stderr=None,
+        stdout=subprocess.DEVNULL,
         shell=False,
+        timeout=20,
     )
 
 
@@ -396,7 +405,8 @@ def __init__(
             "--disable-redirect-access-to-syslog",
             "--graceful-timeout=%d" % (GRACEFUL_TIMEOUT,),
             "--bind=%s" % server_bind,
-            "--reuse-port",
+            # untested on non-Linux
+            # "--reuse-port",
             *thread_opt,
             *ssl_opt,
             "--",
@@ -414,15 +424,25 @@ def __enter__(self):
         # type: () -> Self
         import http.client
 
-        self.conn = http.client.HTTPConnection(self._host_port, timeout=2)
+        self.conn = http.client.HTTPConnection(self._host_port, timeout=5)
         return self
 
     def __exit__(self, *exc):
         self.conn.close()
 
-    def get(self, path):
+    def get(self, path="/", test=False):
         # type: () -> http.client.HTTPResponse
-        self.conn.request("GET", path, headers={"Host": HTTP_HOST}, body="GETBODY!")
+        body = b"GETBODY!"
+        self.conn.request(
+            "GET",
+            path,
+            headers={
+                "Host": "invalid.invalid." if test else HTTP_HOST,
+                "Connection": "close",
+                "Content-Length": "%d" % (len(body),),
+            },
+            body=body,
+        )
         return self.conn.getresponse()
 
 

tests/test_nginx.py

@@ -86,9 +86,22 @@ def test_nginx_proxy(*, ssl, worker_class, dummy_ssl_cert, read_size=1024):
                 },
             )
 
+            path = "/dummy"
+            try:
+                response = client.get(path, test=True)
+            except TimeoutError as exc:
+                raise AssertionError(f"failed to query proxy: {exc!r}") from exc
+            assert response.status == 400
+            test_body = response.read()
+            assert b"nginx" in test_body
+            proxy.read_stdio(timeout_sec=4, wait_for_keyword="GET %s HTTP/1.1" % path)
+
             for num_request in range(5):
                 path = "/pytest/%d" % (num_request)
-                response = client.get(path)
+                try:
+                    response = client.get(path)
+                except TimeoutError as exc:
+                    raise AssertionError(f"failed to fetch {path!r}: {exc!r}") from exc
                 assert response.status == 200
                 assert response.read() == b"response body from app"
 

tests/test_wrk.py

@@ -7,6 +7,7 @@
 #   --override-ini=addopts=--strict-markers --exitfirst \
 #   -- tests/test_nginx.py
 
+import platform
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -22,6 +23,9 @@
 
 # @pytest.mark.parametrize("read_size", [50+secrets.randbelow(2048)])
 @WrkClient.pytest_supported()
+@pytest.mark.skipif(
+    platform.python_implementation() == "PyPy", reason="slow on Github CI"
+)
 @pytest.mark.parametrize("ssl", [False, True], ids=["plain", "ssl"])
 @pytest.mark.parametrize("worker_class", WORKER_PYTEST_LIST)
 def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024):
@@ -65,10 +69,13 @@ def test_wrk(*, ssl, worker_class, dummy_ssl_cert, read_size=1024):
             extract = WrkClient.RE_RATE.search(out)
             assert extract is not None, out
             rate = float(extract.groups()[0])
+            expected = 50
             if worker_class == "sync":
-                assert rate > 5
-            else:
-                assert rate > 50
+                expected = 5
+            # test way too short to make slow GitHub runners fast on PyPy
+            if platform.python_implementation() == "PyPy":
+                expected //= 5
+            assert rate > expected, (rate, expected)
 
             server.read_stdio(timeout_sec=2, wait_for_keyword="GET %s HTTP/1.1" % path)
             if ssl: