diff --git a/README-PyPI.md b/README-PyPI.md index 418f762..d25d742 100644 --- a/README-PyPI.md +++ b/README-PyPI.md @@ -1,6 +1,6 @@ # FediTest -Testing distributed, heterogeneous systems... +Test framework to test distributed, heterogeneous systems... ...with complex protocols diff --git a/RELEASE-HOWTO.md b/RELEASE-HOWTO.md index 096dac2..1a41f76 100644 --- a/RELEASE-HOWTO.md +++ b/RELEASE-HOWTO.md @@ -1,69 +1,89 @@ # Release How-To +## Merge + 1. Repo `feditest`: merge `develop` into `main` 1. Repo `feditest-tests-fediverse`: merge `develop` into `main` 1. Repo `feditest-tests-sandbox`: merge `develop` into `main` +## Smoke test and test with sandbox + 1. On the Mac: - 1. Repo `feditest`: `git checkout main` + 1. Repo `feditest`: `git checkout develop` 1. In `pyproject.toml`, change `project` / `version` to the new version `VERSION` (needed so the generated files have the right version in them before check-in) 1. Clean rebuild: 1. `rm -rf venv.*` - 1. `make venv PYTHON=python3.11` + 1. `make venv` 1. `make lint`: ruff and mypy show no errors - 1. `make` - 1. Repo `feditest-tests-sandbox`: `git checkout main` + 1. `make tests`: unit tests show no errors (smoke tests don't run on macOS) + 1. Repo `feditest-tests-sandbox`: `git checkout develop` 1. Clean re-run and report generation of the sandbox tests: - 1. `make -f Makefile.generate clean FEDITEST=../feditest/venv.darwin.main/bin/feditest` - 1. `make -f Makefile.run clean FEDITEST=../feditest/venv.darwin.main/bin/feditest` - 1. `make -f Makefile.generate examples FEDITEST=../feditest/venv.darwin.main/bin/feditest` - 1. `make -f Makefile.run sandbox FEDITEST=../feditest/venv.darwin.main/bin/feditest` + 1. `make -f Makefile.create clean FEDITEST=../feditest/venv.darwin.default/bin/feditest` + 1. `make -f Makefile.run clean FEDITEST=../feditest/venv.darwin.default/bin/feditest` + 1. `make -f Makefile.create examples FEDITEST=../feditest/venv.darwin.default/bin/feditest` + 1. `make -f Makefile.run sandbox FEDITEST=../feditest/venv.darwin.default/bin/feditest` 1. `open examples/testresults/*.html` and check for plausbility of reports +## Smoke test and test quickstart + 1. On UBOS: - 1. Repo `feditest`: `git checkout main` + 1. Repo `feditest`: `git checkout develop` 1. Clean rebuild: - 1. `rm -rf venv.*` - 1. `make` - 1. Repo `feditest-tests-fediverse`: `git checkout main` + 1. `rm -rf venv.*` + 1. `make venv` + 1. `make lint`: ruff and mypy show no errors + 1. `make tests`: unit tests and smoke tests show no errors + 1. Repo `feditest-tests-fediverse`: `git checkout develop` 1. Clean re-run and report generation of the sandbox tests: - 1. `make -f Makefile.generate clean FEDITEST=../feditest/venv.linux.main/bin/feditest` + 1. `make -f Makefile.create clean FEDITEST=../feditest/venv.linux.main/bin/feditest` 1. `make -f Makefile.run clean FEDITEST=../feditest/venv.linux.main/bin/feditest` - 1. `make -f Makefile.generate examples FEDITEST=../feditest/venv.linux.main/bin/feditest` + 1. `make -f Makefile.create examples FEDITEST=../feditest/venv.linux.main/bin/feditest` 1. `make -f Makefile.run examples FEDITEST=../feditest/venv.linux.main/bin/feditest` 1. `xdg-open examples/testresults/*.html` and check for plausbility of reports +## Tag versions + 1. On the Mac: - 1. Update repo `feditest-tests-fediverse` - 1. `git commit` to `main` + 1. Update repo `feditest-tests-fediverse`, branch `develop`: 1. `git tag -a vVERSION -m vVERSION` 1. `git push` 1. `git push --tags` - 1. Update repo `feditest-tests-sandbox` - 1. `git commit` to `main` + 1. Update repo `feditest-tests-sandbox`, branch `develop`: 1. `git tag -a vVERSION -m vVERSION` 1. `git push` 1. `git push --tags` - 1. Update repo `feditest`: - 1. Change the `python-version` value in `pyproject.toml` to the "production value" that permits Python 3.11 and greater + 1. Update repo `feditest`, branch `develop`: 1. `git commit` to `main` 1. `git tag -a vVERSION -m vVERSION` 1. `git push` 1. `git push --tags` - 1. Release to PyPi - 1. `make release PYTHON=python3.11` - 1. `venv.release/bin/twine upload dist/*` - 1. `pip install --upgrade feditest` - 1. `feditest version` now shows `VERSION` - 1. Return `python-version` value in `pyproject.toml` to the "development value" that only permits Python 3.11 + +## Merge into main + +1. Repo `feditest-tests-fediverse`: pull request `develop` into `main` +1. Repo `feditest-tests-sandbox`: pull request `develop` into `main` +1. Repo `feditest`: pull request `develop` into `main` +1. Approve all three pull requests + +## Publish to PyPi + +1. On the Mac: + 1. `make release` + 1. `venv.release/bin/twine upload dist/*` + 1. `pip install --upgrade feditest` + 1. `feditest version` now shows `VERSION` + +## Publish to UBOS repos 1. On UBOS: 1. Build `feditest` for the UBOS package repos so it can be installed with `pacman -S feditest` -1. Release notes: - 1. Repo: `feditest.org`: create release notes - 1. `git push` +## Create release notes + +1. Repo: `feditest.org`: create release notes +1. `git push` + +## Announce -1. Announce: - 1. `@feditest@mastodon.social`: post link to release notes - 1. `https://matrix.to/#/#fediverse-testing:matrix.org`: post link to release notes +1. `https://matrix.to/#/#fediverse-testing:matrix.org`: post link to release notes +1. `@feditest@mastodon.social`: post link to release notes diff --git a/pyproject.toml b/pyproject.toml index cc63003..7ea632b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "feditest" -version = "0.3" +version = "0.4" authors = [ { name="Johannes Ernst", email="git@j12t.org" }, { name="Steve Bate", email="svc-github@stevebate.net" } @@ -27,7 +27,7 @@ dependencies = [ "types-requests", "pre-commit" ] -description = "Testing federated protocols" +description = "Test framework to test distributed, heterogeneous systems with complex protocols such as the Fediverse" readme = "README-PyPI.md" # We develop on 3.11, so we can support debian 12 (including Raspberry PI OS) systems, diff --git a/src/feditest/__init__.py b/src/feditest/__init__.py index 88e7262..7175823 100644 --- a/src/feditest/__init__.py +++ b/src/feditest/__init__.py @@ -242,6 +242,7 @@ class SpecLevel(Enum): UNSPECIFIED = 4 + @property def formatted_name(self): return self.name.capitalize() @@ -253,6 +254,7 @@ class InteropLevel(Enum): UNKNOWN = 4 + @property def formatted_name(self): return self.name.capitalize() @@ -268,7 +270,7 @@ def __init__(self, spec_level: SpecLevel, interop_level: InteropLevel, msg: Any) def __str__(self): - return str(self.msg) + return f'AssertionFailure ({ self.spec_level.formatted_name }, { self.interop_level.formatted_name }): { self.msg }' def _assert_match( diff --git a/src/feditest/cli/__init__.py b/src/feditest/cli/__init__.py index 06c265b..775e4b3 100644 --- a/src/feditest/cli/__init__.py +++ b/src/feditest/cli/__init__.py @@ -34,7 +34,7 @@ def main() -> None: set_reporting_level(args.verbose) - if sys.version_info.major != 3 or sys.version_info != 11: + if sys.version_info.major != 3 or sys.version_info.minor != 11: warning(f"feditest currently requires Python 3.11. You are using { sys.version }" + " and may get unpredictable results. We'll get to other versions in the future.") diff --git a/src/feditest/cli/commands/version.py b/src/feditest/cli/commands/version.py index fbab7bf..6e58b5a 100644 --- a/src/feditest/cli/commands/version.py +++ b/src/feditest/cli/commands/version.py @@ -14,7 +14,7 @@ def run(parser: ArgumentParser, args: Namespace, remaining: list[str]) -> int: parser.print_help() return 0 - print(f'feditest version { FEDITEST_VERSION }') + print(FEDITEST_VERSION) return 0 diff --git a/src/feditest/nodedrivers/imp/__init__.py b/src/feditest/nodedrivers/imp/__init__.py index 2e0f76e..66813a6 100644 --- a/src/feditest/nodedrivers/imp/__init__.py +++ b/src/feditest/nodedrivers/imp/__init__.py @@ -19,7 +19,7 @@ _HEADERS = { "User-Agent": f"feditest/{ FEDITEST_VERSION }", - "Origin": "test.example" # to trigger CORS headers in response + "Origin": "https://test.example" # to trigger CORS headers in response } class Imp(AbstractWebFingerDiagClient): diff --git a/src/feditest/protocols/webfinger/abstract.py b/src/feditest/protocols/webfinger/abstract.py index b3495e0..b903400 100644 --- a/src/feditest/protocols/webfinger/abstract.py +++ b/src/feditest/protocols/webfinger/abstract.py @@ -6,7 +6,7 @@ from feditest.protocols.web.diag import HttpRequest, HttpRequestResponsePair, WebDiagClient from feditest.protocols.webfinger import WebFingerServer from feditest.protocols.webfinger.diag import ClaimedJrd, WebFingerDiagClient -from feditest.protocols.webfinger.utils import construct_webfinger_uri_for, WebFingerQueryResponse +from feditest.protocols.webfinger.utils import construct_webfinger_uri_for, WebFingerQueryDiagResponse from feditest.utils import ParsedUri @@ -17,7 +17,7 @@ def diag_perform_webfinger_query( resource_uri: str, rels: list[str] | None = None, server: WebFingerServer | None = None - ) -> WebFingerQueryResponse: + ) -> WebFingerQueryDiagResponse: query_url = construct_webfinger_uri_for(resource_uri, rels, server.hostname() if server else None ) parsed_uri = ParsedUri.parse(query_url) if not parsed_uri: @@ -29,10 +29,10 @@ def diag_perform_webfinger_query( pair = self.http(current_request) if pair.response and pair.response.is_redirect(): if redirect_count <= 0: - return WebFingerQueryResponse(pair, None, WebDiagClient.TooManyRedirectsError(current_request)) + return WebFingerQueryDiagResponse(pair, None, [ WebDiagClient.TooManyRedirectsError(current_request) ]) parsed_location_uri = ParsedUri.parse(pair.response.location()) if not parsed_location_uri: - return WebFingerQueryResponse(pair, None, ValueError('Location header is not a valid URI:', query_url, '(from', resource_uri, ')')) + return WebFingerQueryDiagResponse(pair, None, [ ValueError('Location header is not a valid URI:', query_url, '(from', resource_uri, ')') ] ) current_request = HttpRequest(parsed_location_uri) break @@ -67,9 +67,4 @@ def diag_perform_webfinger_query( except Exception as exc: excs.append(exc) - if len(excs) > 1: - return WebFingerQueryResponse(ret_pair, jrd, ExceptionGroup('WebFinger errors', excs)) - elif len(excs) == 1: - return WebFingerQueryResponse(ret_pair, jrd, excs[0]) - else: - return WebFingerQueryResponse(ret_pair, jrd, None) + return WebFingerQueryDiagResponse(ret_pair, jrd, excs) diff --git a/src/feditest/protocols/webfinger/diag.py b/src/feditest/protocols/webfinger/diag.py index fa48f8f..fba173e 100644 --- a/src/feditest/protocols/webfinger/diag.py +++ b/src/feditest/protocols/webfinger/diag.py @@ -2,7 +2,7 @@ """ import json -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from feditest.nodedrivers import NotImplementedByNodeError @@ -471,10 +471,24 @@ def __str__(self): @dataclass -class WebFingerQueryResponse: +class WebFingerQueryDiagResponse: http_request_response_pair: HttpRequestResponsePair jrd : ClaimedJrd | None # This may be an invalid jrd - exc : Exception | None # + exceptions : list[Exception] = field(default_factory=list) # List of all things that were found to be wrong + + + def exceptions_of_type(self, filter_by: type) -> list[Exception]: + """ + Return only the subset of exceptions that are of type filter_by + """ + return [ ex for ex in self.exceptions if isinstance(ex, filter_by) ] + + + def not_exceptions_of_type(self, filter_by: tuple) -> list[Exception]: + """ + Return only the subset of exceptions that are not of any of the types in filter_by + """ + return [ ex for ex in self.exceptions if not isinstance(ex, filter_by) ] class WebFingerDiagClient(WebFingerClient, WebDiagClient): @@ -491,7 +505,7 @@ def diag_perform_webfinger_query( resource_uri: str, rels: list[str] | None = None, server: WebFingerServer | None = None - ) -> WebFingerQueryResponse: + ) -> WebFingerQueryDiagResponse: """ Make this Node perform a WebFinger query for the provided resource_uri. The resource_uri must be a valid, absolute URI, such as 'acct:foo@bar.com` or diff --git a/src/feditest/protocols/webfinger/utils.py b/src/feditest/protocols/webfinger/utils.py index 51ad829..9558ad1 100644 --- a/src/feditest/protocols/webfinger/utils.py +++ b/src/feditest/protocols/webfinger/utils.py @@ -9,7 +9,7 @@ from hamcrest.core.base_matcher import BaseMatcher from hamcrest.core.description import Description -from .diag import ClaimedJrd, WebFingerQueryResponse +from .diag import ClaimedJrd, WebFingerQueryDiagResponse class UnsupportedUriSchemeError(RuntimeError): @@ -203,23 +203,15 @@ def none_except(*allowed_excs : Type[Exception]) -> NoneExceptMatcher : return NoneExceptMatcher(list(allowed_excs)) -def wf_error(response: WebFingerQueryResponse) -> str: +def wf_error(response: WebFingerQueryDiagResponse) -> str: """ Construct an error message """ - if not response.exc: + if not response.exceptions: return 'ok' - if isinstance(response.exc, ExceptionGroup): - # Make this more compact than the default - msg = str(response.exc.args[0]).split('\n', maxsplit=1)[0] - msg += f' ({ len(response.exc.exceptions) })' - msg += f'\nAccessed URI: "{ response.http_request_response_pair.request.parsed_uri.uri }".' - for i, exc in enumerate(response.exc.exceptions): - msg += f'\n{ i }: { exc }' - - else: - msg = str(response.exc).split('\n', maxsplit=1)[0] - msg += f'\nAccessed URI: "{ response.http_request_response_pair.request.parsed_uri.uri }".' - msg += '\n'.join(str(response.exc).split('\n')[1:]) + msg = f'Accessed URI: "{ response.http_request_response_pair.request.parsed_uri.uri }":' + for i, exc in enumerate(response.exceptions): + msg += f'\n{ i }: { exc }' + return msg diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 index 7be8890..ad6de61 100644 --- a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/matrix.jinja2 @@ -2,7 +2,7 @@ {% include "partials/shared/head.jinja2" %} - {{ transcript.name }} | Feditest + {{ transcript.plan.name }} | Feditest
diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 index b897e3a..e80ef85 100644 --- a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared/summary.jinja2 @@ -54,7 +54,7 @@ {%- for spec_level in [ feditest.SpecLevel.SHOULD, feditest.SpecLevel.IMPLIED, feditest.SpecLevel.UNSPECIFIED ] %} - {{ spec_level.formatted_name() }} + {{ spec_level.formatted_name }} {%- for interop_level in feditest.InteropLevel %}
diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 index b8c6508..e5ff58a 100644 --- a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/partials/shared_session/results.jinja2 @@ -30,7 +30,7 @@

Test Results

{%- for test_index, run_test in enumerate(run_session.run_tests) %} -{%- set plan_test_spec = transcript.plan.session.tests[run_test.plan_test_index] %} +{%- set plan_test_spec = transcript.plan.session_template.tests[run_test.plan_test_index] %} {%- set test_meta = transcript.test_meta[plan_test_spec.name] %}

Test: {{ test_meta.name }}

diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 index 2ab578c..97d288a 100644 --- a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/session_single.jinja2 @@ -2,7 +2,7 @@ {% include "partials/shared/head.jinja2" %} - {{ transcript.name }} | Feditest + {{ transcript.plan.name }} | Feditest
diff --git a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css index 1907417..bc004bd 100644 --- a/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css +++ b/src/feditest/testruntranscriptserializer/templates/testplantranscript_default/static/feditest.css @@ -167,7 +167,6 @@ div.feditest.session > h3::before { } .feditest .status div { - margin: 3px 6px; padding: 5px; }