diff --git a/docs/federation_activitypub.md b/docs/federation_activitypub.md index 132fabee..f66ebcb0 100644 --- a/docs/federation_activitypub.md +++ b/docs/federation_activitypub.md @@ -2,13 +2,49 @@ - Federation of SCITT events enables near real-time communication between supply chains. - - Acceptance of claims to SCITT where payload data contains VEX, VSA, SBOM, - S2C2F alignment attestations, etc. has the side effect of enabling a - consistent pattern for notification of new vulnerability (OpenSSF Stream - 8) and other Software Supply Chain Security data. + - Acceptance of claims to SCITT where payload data contains VEX, CSAF, VSA, + SBOM, VDR, VRF, S2C2F alignment attestations, etc. has the side effect of + enabling a consistent pattern for notification of new vulnerability + and other Software Supply Chain Security data. - References - - [7.](https://www.ietf.org/archive/id/draft-ietf-scitt-architecture-02.html#name-federation) + - [SCITT Architecture: 7. Federation](https://www.ietf.org/archive/id/draft-ietf-scitt-architecture-02.html#name-federation) - https://www.w3.org/TR/activitypub/ + - [OpenSSF Stream 8](https://8112310.fs1.hubspotusercontent-na1.net/hubfs/8112310/OpenSSF/OSS%20Mobilization%20Plan.pdf): + Coordinate Industry-Wide Data Sharing to Improve the Research That Helps + Determine the Most Critical OSS Components + +```mermaid +flowchart TD + subgraph alice + subgraph aliceSCITT[SCITT] + alice_submit_claim[Submit Claim] + alice_receipt_created[Receipt Created] + + alice_submit_claim --> alice_receipt_created + end + subgraph aliceActivityPubActor[ActivityPub Actor] + alice_inbox[Inbox] + end + + alice_inbox --> alice_submit_claim + end + subgraph bob + subgraph bobSCITT[SCITT] + bob_submit_claim[Submit Claim] + bob_receipt_created[Receipt Created] + + bob_submit_claim --> bob_receipt_created + end + subgraph bobActivityPubActor[ActivityPub Actor] + bob_inbox[Inbox] + end + + bob_inbox --> bob_submit_claim + end + + alice_receipt_created --> bob_inbox + bob_receipt_created --> alice_inbox +``` ## Dependencies @@ -51,6 +87,7 @@ By the end of this tutorial you will have four terminals open. - One for submitting claims to Bob and Alice's SCITT instances and querying their ActivityPub Actors. + ### Bring up the ActivityPub Server First we install our dependencies @@ -91,7 +128,7 @@ BovineHerd(app) We'll run on port 5000 to avoid collisions with common default port choices. Keep this running for the rest of the tutorial. -> TODO Integrate Quart app launch into `SCITTFederationActivityPubBovine` +> **TODO** Integrate Quart app launch into `SCITTFederationActivityPubBovine` > initialization. ```console @@ -99,6 +136,8 @@ $ hypercorn app:app -b 0.0.0.0:5000 [2023-10-16 02:44:48 -0700] [36467] [INFO] Running on http://0.0.0.0:5000 (CTRL + C to quit) ``` +> Cleanup: `rm -f *sqlite* federation_*/config.toml` + ### Bring up Bob's SCITT Instance Populate Bob's federation config @@ -118,11 +157,19 @@ Start the server ```console $ rm -rf workspace_bob/ $ mkdir -p workspace_bob/storage/operations -$ scitt-emulator server --workspace workspace_bob/ --tree-alg CCF --port 6000 \ +$ BOVINE_DB_URL="sqlite://${PWD}/bovine.sqlite3" scitt-emulator server \ + --workspace workspace_bob/ --tree-alg CCF --port 6000 \ --federation scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ --federation-config-path federation_bob/config.json ``` +> **TODO** Figure out why the server was restarting constantly if in +> scitt-api-emulator directory (sqlite3?). +> +> ```console +> $ rm -f ~/Documents/fediverse/scitt_federation_bob/config.toml && BOVINE_DB_URL="sqlite://${HOME}/Documents/fediverse/bovine/hacking/bovine.sqlite3" scitt-emulator server --workspace workspace_bob/ --tree-alg CCF --port 6000 --federation scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine --federation-config-path ~/Documents/fediverse/scitt_federation_bob/config.json +> ``` + Create claim from allowed issuer (`.org`) and from non-allowed (`.com`). ```console @@ -157,7 +204,8 @@ Start the server ```console $ rm -rf workspace_alice/ $ mkdir -p workspace_alice/storage/operations -$ scitt-emulator server --workspace workspace_alice/ --tree-alg CCF --port 7000 \ +$ BOVINE_DB_URL="sqlite://${PWD}/bovine.sqlite3" scitt-emulator server \ + --workspace workspace_alice/ --tree-alg CCF --port 7000 \ --federation scitt_emulator.federation_activitypub_bovine:SCITTFederationActivityPubBovine \ --federation-config-path federation_alice/config.json ``` diff --git a/scitt_emulator/federation_activitypub_bovine.py b/scitt_emulator/federation_activitypub_bovine.py index 60b3be73..22ee8c80 100644 --- a/scitt_emulator/federation_activitypub_bovine.py +++ b/scitt_emulator/federation_activitypub_bovine.py @@ -1,55 +1,67 @@ -""" -#!/usr/bin/env bash -set -xeuo pipefail - -rm -rf .venv && \ -python -m venv .venv && \ -. .venv/bin/activate && \ -pip install -U pip setuptools wheel && \ -pip install \ - toml \ - bovine{-store,-process,-pubsub,-herd,-tool} \ - 'https://codeberg.org/pdxjohnny/bovine/archive/activitystreams_collection_helper_enable_multiple_iterations.tar.gz#egg=bovine&subdirectory=bovine' \ - 'https://codeberg.org/pdxjohnny/mechanical_bull/archive/event_loop_on_connect_call_handlers.tar.gz#egg=mechanical-bull' - -export HYPERCORN_PID=0 -function kill_hypercorn() { - kill "${HYPERCORN_PID}" -} -hypercorn app:app & -export HYPERCORN_PID=$! -trap kill_hypercorn EXIT -sleep 1 - -export HANDLE_NAME=alice -export BOVINE_NAME=$(python -m bovine_tool.register "${HANDLE_NAME}" --domain http://localhost:8000 | awk '{print $NF}') -python -m mechanical_bull.add_user --accept "${HANDLE_NAME}" http://localhost:8000 -python -m bovine_tool.manage "${BOVINE_NAME}" --did_key key0 $(cat config.toml | python -c 'import sys, tomllib, bovine.crypto; print(bovine.crypto.private_key_to_did_key(tomllib.load(sys.stdin.buffer)[sys.argv[-1]]["secret"]))' "${HANDLE_NAME}") - -python -c 'import sys, pathlib, toml; path = pathlib.Path(sys.argv[-3]); obj = toml.loads(path.read_text()); obj[sys.argv[-2]]["handlers"][sys.argv[-1]] = True; path.write_text(toml.dumps(obj))' config.toml "${HANDLE_NAME}" scitt_handler - -PYTHONPATH=${PYTHONPATH:-''}:$PWD timeout 5s python -m mechanical_bull.run -""" import sys import json -import pprint +import atexit +import base64 import socket import inspect import logging import asyncio import pathlib +import traceback +import contextlib import subprocess +import dataclasses +import urllib.parse from pathlib import Path from typing import Optional -import toml +import tomli +import tomli_w import bovine +import aiohttp +from bovine.activitystreams import factories_for_actor_object +from bovine.clients import lookup_uri_with_webfinger from mechanical_bull.handlers import HandlerEvent, HandlerAPIVersion from scitt_emulator.federation import SCITTFederation logger = logging.getLogger(__name__) +import pprint + + +@dataclasses.dataclass +class Follow: + id: str + domain: str = None + + +async def get_actor_url( + domain: str, + handle_name: str = None, + did_key: str = None, + session: aiohttp.ClientSession = None, +): + if did_key: + lookup = did_key + elif handle_name: + # Get domain and port without protocol + url_parse_result = urllib.parse.urlparse(domain) + actor_id = f"{handle_name}@{url_parse_result.netloc}" + lookup = f"acct:{actor_id}" + else: + raise ValueError( + f"One of the following keyword arguments must be provided: handle_name, did_key" + ) + async with contextlib.AsyncExitStack() as async_exit_stack: + # Create session if not given + if not session: + session = await async_exit_stack.enter_async_context( + aiohttp.ClientSession(trust_env=True), + ) + url, _ = await lookup_uri_with_webfinger(session, lookup, domain=domain) + return url + class SCITTFederationActivityPubBovine(SCITTFederation): def __init__( @@ -69,73 +81,96 @@ def __init__( self.domain = self.config["domain"] self.handle_name = self.config["handle_name"] - self.workspace = Path(self.config["workspace"]) + self.workspace = Path(self.config["workspace"]).expanduser() self.federate_created_entries_socket_path = self.workspace.joinpath( "federate_created_entries_socket", ) def initialize_service(self): - # read, self.write = multiprocessing.Pipe(duplex=False) - # reader_process = multiprocessing.Process(target=self.reader, args=(read,)) - - # TODO Avoid creating user if already exists - cmd = [ - sys.executable, - "-um", - "mechanical_bull.add_user", - "--accept", - self.handle_name, - domain, - ] - add_user_output = subprocess.check_output( - cmd, - cwd=self.workspace, - ) - did_key = [ - word.replace("did:key:", "") - for word in add_user_output.decode().strip().split() - if word.startswith("did:key:") - ][0] - - cmd = [ - sys.executable, - "-um", - "bovine_tool.register", - self.handle_name, - "--domain", - domain, - ] - register_output = subprocess.check_output( - cmd, - cwd=self.workspace, - ) - bovine_name = register_output.decode().strip().split()[-1] - - cmd = [ - sys.executable, - "-um", - "bovine_tool.manage", - self.handle_name, - "--did_key", - "key0", - did_key, - ] - subprocess.check_call( - cmd, - cwd=self.workspace, - ) + config_toml_path = pathlib.Path(self.workspace, "config.toml") + if not config_toml_path.exists(): + logger.info("Actor client config does not exist, creating...") + cmd = [ + sys.executable, + "-um", + "mechanical_bull.add_user", + "--accept", + self.handle_name, + self.domain, + ] + subprocess.check_call( + cmd, + cwd=self.workspace, + ) + logger.info("Actor client config created") + config_toml_obj = tomli.loads(config_toml_path.read_text()) # Enable handler() function in this file for this actor - config_toml_path = pathlib.Path(self.workspace, "config.toml") - config_toml_obj = toml.loads(config_toml_path.read_text()) config_toml_obj[self.handle_name]["handlers"][ inspect.getmodule(sys.modules[__name__]).__spec__.name ] = { - "federate_created_entries_socket_path": self.federate_created_entries_socket_path, + "federate_created_entries_socket_path": str( + self.federate_created_entries_socket_path.resolve() + ), + "following": self.config.get("following", {}), } - config_toml_path.write_text(toml.dumps(config_toml_obj)) + config_toml_path.write_text(tomli_w.dumps(config_toml_obj)) + # Extract public key from private key in config file + did_key = bovine.crypto.private_key_to_did_key( + config_toml_obj[self.handle_name]["secret"], + ) + + # TODO This may not work if there is another instance of an event loop + # running. There shouldn't be but can we come up with a workaround in + # case that does happen? + actor_url = asyncio.run( + get_actor_url( + self.domain, + did_key=did_key, + ) + ) + # TODO take BOVINE_DB_URL from config, populate env on call to tool if + # NOT already set in env. + # Create the actor in the database, set + # BOVINE_DB_URL="sqlite://${HOME}/path/to/bovine.sqlite3" or see + # https://codeberg.org/bovine/bovine/src/branch/main/bovine_herd#configuration + # for more options. + if actor_url: + logger.info("Existing actor found. actor_url is %s", actor_url) + else: + logger.info("Actor not found, creating in database...") + cmd = [ + sys.executable, + "-um", + "bovine_tool.register", + self.handle_name, + "--domain", + self.domain, + ] + register_output = subprocess.check_output( + cmd, + cwd=self.workspace, + ) + bovine_name = register_output.decode().strip().split()[-1] + logger.info("Created actor with database name %s", bovine_name) + + cmd = [ + sys.executable, + "-um", + "bovine_tool.manage", + bovine_name, + "--did_key", + "key0", + did_key, + ] + subprocess.check_call( + cmd, + cwd=self.workspace, + ) + logger.info("Actor key added in database") + # Run client handlers cmd = [ sys.executable, "-um", @@ -145,10 +180,11 @@ def initialize_service(self): cmd, cwd=self.workspace, ) + atexit.register(self.mechanical_bull_proc.terminate) def created_entry(self, entry_id: str, receipt: bytes): with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.connect(self.federate_created_entries_socket_path) + client.connect(str(self.federate_created_entries_socket_path.resolve())) client.send(receipt) client.close() @@ -156,7 +192,10 @@ def created_entry(self, entry_id: str, receipt: bytes): async def handle( client: bovine.BovineClient, data: dict, + # config.toml arguments + following: dict[str, Follow] = None, federate_created_entries_socket_path: Path = None, + # handler arguments handler_event: HandlerEvent = None, handler_api_version: HandlerAPIVersion = HandlerAPIVersion.unstable, ): @@ -164,6 +203,14 @@ async def handle( logging.info(f"{__file__}:handle(handler_event={handler_event})") match handler_event: case HandlerEvent.OPENED: + await init_follow( + client, + id="acct:bob@localhost:5000", + domain="http://localhost:5000", + ) + if following: + for key, value in following.items(): + asyncio.create_task(init_follow(**value)) asyncio.create_task( federate_created_entries( client, federate_created_entries_socket_path @@ -186,29 +233,72 @@ async def handle( logger.error(json.dumps(data)) +async def init_follow(client, actor_id: str, domain: str = None): + # from config import actor, actor_object, make_id + print(actor_id) + remote = await lookup_uri_with_webfinger(client.session, actor_id, domain=domain) + print(remote) + return + actor = bovine.BovineActor( + actor_id=follow.id, + public_key_url=f"{follow.domain}#main-key", + ) + print(actor) + + return + remote_inbox = (await client.get(remote))["inbox"] + print(remote_inbox) + return + activity = client.activity_factory.create( + client.object_factory.follow( + dataclasses.asdict(follow), + ) + .as_public() + .build() + ).build() + logger.info("Following... %r", activity) + await client.post(remote_inbox, follow) + + async def federate_created_entries( client: bovine.BovineClient, socket_path: Path, ): async def federate_created_entry(reader, writer): - receipt = await reader.read() - note = ( - client.object_factory.note( - content=base64.b64encode(receipt), + try: + logger.info("federate_created_entry() Reading... %r", reader) + receipt = await reader.read() + logger.info("federate_created_entry() Read: %r", receipt) + note = ( + client.object_factory.note( + content=base64.b64encode(receipt).decode(), + ) + .as_public() + .build() ) - .as_public() - .build() - ) - activity = client.activity_factory.create(note).build() - logger.info("Sending... %r", activity) - await client.send_to_outbox(activity) + activity = client.activity_factory.create(note).build() + logger.info("Sending... %r", activity) + await client.send_to_outbox(activity) + + writer.close() + await writer.wait_closed() - writer.close() - await writer.wait_closed() + # DEBUG NOTE Dumping outbox + print("client:", client) + outbox = client.outbox() + print("outbox:", outbox) + count_messages = 0 + async for message in outbox: + count_messages += 1 + print(f"Message {count_messages} in outbox:", message) + print(f"End of messages in outbox, total: {count_messages}") + except: + logger.error(traceback.format_exc()) + logger.info("Attempting UNIX bind at %r", socket_path) server = await asyncio.start_unix_server( federate_created_entry, - path=str(socket_path.resolve()), + path=str(Path(socket_path).resolve()), ) async with server: logger.info("Awaiting receipts to federate at %r", socket_path) diff --git a/scitt_emulator/server.py b/scitt_emulator/server.py index f4d5e012..42475d31 100644 --- a/scitt_emulator/server.py +++ b/scitt_emulator/server.py @@ -46,16 +46,17 @@ def create_flask_app(config): federation = None if app.config.get("federation", None): - federation = app.config["federation"]( + app.federation = app.config["federation"]( config_path=app.config.get("federation_config_path", None), storage_path=storage_path, service_parameters_path=app.service_parameters_path ) + app.federation.initialize_service() app.scitt_service = clazz( storage_path=storage_path, service_parameters_path=app.service_parameters_path, - federation=federation, + federation=app.federation, ) app.scitt_service.initialize_service() print(f"Service parameters: {app.service_parameters_path}") diff --git a/setup.py b/setup.py index 1086f87d..47ec778b 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,9 @@ ], extras_require={ "federation-activitypub-bovine": [ + "tomli", + "tomli-w", + "aiohttp", "bovine", "bovine-tool", "mechanical-bull",