feat: hash did:web documents

scripts/export-dids.ts

@@ -1,3 +1,5 @@
+import crypto from 'node:crypto';
+
 import { BskyXRPC } from '@mary/bluesky-client';
 
 import * as v from '@badrap/valita';
@@ -217,33 +219,57 @@ let firehoseCursor: string | undefined = state?.firehose.cursor;
 					const signal = AbortSignal.timeout(15_000);
 					const res = await get(`https://${host}/.well-known/did.json`, signal);
 
-					const json = (await res.json()) as unknown;
-					const doc = didDocument.parse(json, { mode: 'passthrough' });
+					const text = await res.text();
+					const sha256sum = getHash('sha256', text);
+
+					if (obj.hash !== sha256sum) {
+						const json = JSON.parse(text);
+						const doc = didDocument.parse(json, { mode: 'passthrough' });
+
+						const pds = getPdsEndpoint(doc);
+						const labeler = getLabelerEndpoint(doc);
 
-					const pds = getPdsEndpoint(doc);
-					const labeler = getLabelerEndpoint(doc);
+						console.log(`  ${did}: pass (updated)`);
 
-					console.log(`  ${did}: pass`);
+						if (pds) {
+							const info = pdses.get(pds);
 
-					if (pds && obj.pds !== pds) {
-						if (!pdses.has(pds)) {
-							console.log(`    found pds: ${pds}`);
-							pdses.set(pds, {});
+							if (info === undefined) {
+								console.log(`    found pds: ${pds}`);
+								pdses.set(pds, {});
+							} else if (info.errorAt !== undefined) {
+								// reset `errorAt` if we encounter this PDS
+								console.log(`    found pds: ${pds} (errored)`);
+								info.errorAt = undefined;
+							}
 						}
-					}
 
-					if (labeler && obj.labeler !== labeler) {
-						if (!labelers.has(labeler)) {
-							console.log(`    found labeler: ${labeler}`);
-							labelers.set(labeler, { did });
-						} else {
-							labelers.get(labeler)!.did = did;
+						if (labeler) {
+							const info = labelers.get(labeler);
+
+							if (info === undefined) {
+								console.log(`    found labeler: ${labeler}`);
+								labelers.set(labeler, { did });
+							} else {
+								if (info.errorAt !== undefined) {
+									// reset `errorAt` if we encounter this labeler
+									console.log(`    found labeler: ${labeler} (errored)`);
+									info.errorAt = undefined;
+								}
+
+								info.did = did;
+							}
 						}
+
+						obj.hash = sha256sum;
+
+						obj.pds = pds;
+						obj.labeler = labeler;
+					} else {
+						console.log(`  ${did}: pass`);
 					}
 
 					obj.errorAt = undefined;
-					obj.pds = pds;
-					obj.labeler = labeler;
 				} catch (err) {
 					const errorAt = obj.errorAt;
 
@@ -255,7 +281,7 @@ let firehoseCursor: string | undefined = state?.firehose.cursor;
 						didWebs.delete(did);
 					}
 
-					console.log(`  ${did}: fail`);
+					console.log(`  ${did}: fail (${err})`);
 				}
 			});
 		}),
@@ -344,3 +370,10 @@ function getEndpoint(urlStr: string | undefined): string | undefined {
 
 	return url.href;
 }
+
+function getHash(algo: string, data: string) {
+	const hasher = crypto.createHash(algo);
+	hasher.update(data);
+
+	return hasher.digest('base64url');
+}

src/state.ts

@@ -17,6 +17,7 @@ export const serializedState = v.object({
 		didWebs: v.record(
 			v.object({
 				errorAt: dateInt.optional(),
+				hash: v.string().optional(),
 				pds: v.string().optional(),
 				labeler: v.string().optional(),
 			}),
@@ -46,6 +47,7 @@ export type SerializedState = v.Infer<typeof serializedState>;
 
 export interface DidWebInfo {
 	errorAt?: number;
+	hash?: string;
 	pds?: string;
 	labeler?: string;
 }