Merge pull request #5857 from guardian/mob/thrasher-tracker

Thrasher tracker

.github/workflows/fronts.yml → .github/workflows/scheduled.yml

@@ -1,4 +1,4 @@
-name: DCR Fronts Ophan Components
+name: Scheduled daily jobs
 on:
   schedule:
     # Every work day of the week at 08:08
@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  ophan-components:
+  scheduled:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -17,9 +17,9 @@ jobs:
       - name: Setup deno
         uses: denolib/setup-deno@v2
         with:
-          deno-version: v1.21.0
+          deno-version: v1.25.0
 
-      - name: Compare Ophan components
-        run: scripts/ci-ophan.sh
+      - name: Run Deno scripts
+        run: scripts/ci-deno.sh
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

scripts/ci-ophan.sh → scripts/ci-deno.sh

@@ -5,3 +5,9 @@ deno run \
 	--allow-net=www.theguardian.com,api.github.com \
 	--allow-env="GITHUB_TOKEN" \
 	scripts/deno/ophan-components.ts
+
+deno run \
+	--no-check=remote \
+	--allow-net=www.theguardian.com,api.github.com \
+	--allow-env="GITHUB_TOKEN" \
+	scripts/deno//thrasher-tracker.ts

scripts/deno/github.ts

@@ -3,7 +3,7 @@ import type { RestEndpointMethodTypes } from 'https://cdn.skypack.dev/@octokit/p
 
 /** Github token for Authentication */
 const token = Deno.env.get('GITHUB_TOKEN');
-if (!token) throw new Error('Missing GITHUB_TOKEN');
+if (!token) console.warn('Missing GITHUB_TOKEN');
 
 type OctokitWithRest = {
 	rest: {
@@ -18,5 +18,7 @@ type OctokitWithRest = {
 /**
  * A hydrated Octokit with types for the rest API.
  */
-// @ts-expect-error -- Octokit’s own types are not as good as ours
-export const octokit = new Octokit({ auth: token }) as OctokitWithRest;
+export const octokit = token
+	? // @ts-expect-error -- Octokit’s own types are not as good as ours
+	  (new Octokit({ auth: token }) as OctokitWithRest)
+	: undefined;

scripts/deno/iframe-titles.ts

@@ -165,6 +165,11 @@ ${formatter(ok, true)}
 
 const issue_number = 5510;
 
+if (!octokit) {
+	console.log(body);
+	Deno.exit(0);
+}
+
 try {
 	const {
 		data: { html_url },

scripts/deno/json.ts

@@ -0,0 +1,13 @@
+export const fetchJSON = async <T>(
+	url: Parameters<typeof fetch>[0],
+	{
+		headers,
+		parser,
+	}: {
+		headers?: HeadersInit;
+		parser: (data: unknown) => T | Promise<T>;
+	},
+): Promise<T> => {
+	const data: unknown = await fetch(url, { headers }).then((r) => r.json());
+	return parser(data);
+};

scripts/deno/ophan-components.ts

@@ -130,6 +130,11 @@ ${
 
 	const issue_number = issues[attribute];
 
+	if (!octokit) {
+		console.log(body);
+		return;
+	}
+
 	const {
 		data: { body: previousBody },
 	} = await octokit.rest.issues.get({

scripts/deno/surface-lighthouse-results.ts

@@ -150,6 +150,7 @@ const createLighthouseResultsMd = (): string => {
 };
 
 const getCommentID = async (): Promise<number | null> => {
+	if (!octokit) return null;
 	const { data: comments } = await octokit.rest.issues.listComments({
 		...GIHUB_PARAMS,
 	});
@@ -161,8 +162,14 @@ const getCommentID = async (): Promise<number | null> => {
 	return comment?.id ?? null;
 };
 
+const body = createLighthouseResultsMd();
+
+if (!octokit) {
+	console.log(body);
+	Deno.exit();
+}
+
 try {
-	const body = createLighthouseResultsMd();
 	const comment_id = await getCommentID();
 
 	const { data } = comment_id

scripts/deno/thrasher-tracker.ts

@@ -0,0 +1,179 @@
+import { array, object, string } from 'https://deno.land/x/[email protected]/mod.ts';
+import { fetchJSON } from './json.ts';
+import prettyBytes from 'https://esm.sh/pretty-bytes';
+import { octokit } from './github.ts';
+
+// -- Constants -- //
+
+const fronts = ['uk', 'us', 'international', 'au'] as const;
+
+const regex =
+	/(http|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:\/~+#-]*[\w@?^=%&\/~+#-])/g;
+
+const frontSchema = object({
+	pressedPage: object({
+		collections: array(
+			object({
+				id: string(),
+				collectionType: string(),
+				displayName: string(),
+				curated: array(
+					object({
+						enriched: object({
+							embedHtml: string().optional(),
+							embedCss: string().optional(),
+							embedJs: string().optional(),
+						}),
+					}),
+				),
+			}),
+		),
+	}),
+});
+
+/**
+ * We ignore all fonts extensions because browsers will only load
+ * `woff2`one of these resources at a time.
+ */
+const _fontsExtensions = ['woff', 'ttf'];
+
+const supportedResourceExtensions = [
+	'js',
+	'png',
+	'woff2',
+	'gif',
+	'jpg',
+	'mp4',
+	'css',
+];
+
+// -- Methods -- //
+
+const getExtension = (url: URL) => url.pathname.split('.').slice(-1)[0];
+
+const isSupportedResourceType = (url: URL): boolean =>
+	supportedResourceExtensions.includes(getExtension(url));
+
+const getResourceSize = async (url: URL): Promise<number> => {
+	const response = await fetch(url);
+	return (await response.blob()).size;
+};
+
+const getThrasherResources = (urls: URL[]) => {
+	return Promise.all(
+		urls.map(async (url) => {
+			const size = await getResourceSize(url);
+			return { url, size };
+		}),
+	);
+};
+
+const getFrontThrashers = async (path: string) => {
+	const url = new URL(`https://theguardian.com/${path}.json?dcr`);
+	const {
+		pressedPage: { collections },
+	} = await fetchJSON(url, { parser: frontSchema.parse });
+
+	const thrashers = collections.filter(
+		(collection) => collection.collectionType === 'fixed/thrasher',
+	);
+
+	const thrashersWithResources = thrashers.flatMap(
+		async ({ displayName, curated: [{ enriched }] }) => {
+			const resourceUrls = Object.values(enriched)
+				.flatMap((embed) =>
+					[...embed.matchAll(regex)].map(([url]) => new URL(url)),
+				)
+				.filter(isSupportedResourceType);
+
+			const resources = await getThrasherResources(resourceUrls);
+
+			const embedSize = new Blob([...Object.values(enriched)]).size;
+
+			const resourceSize = resources.reduce((map, { url, size }) => {
+				const ext = getExtension(url);
+				const acc = map.get(ext) ?? 0;
+				map.set(ext, acc + size);
+				return map;
+			}, new Map<string, number>());
+
+			return {
+				displayName,
+				resources,
+				embedSize,
+				resourceSize,
+				totalSize:
+					embedSize +
+					[...resourceSize.values()].reduce(
+						(acc, next) => acc + next,
+						0,
+					),
+			};
+		},
+	);
+	return Promise.all(thrashersWithResources);
+};
+
+const getTable = (data: Awaited<ReturnType<typeof getFrontThrashers>>) => {
+	const rows = data
+		.slice()
+		.sort((a, b) => b.totalSize - a.totalSize)
+		.map(
+			({ displayName, embedSize, resourceSize, totalSize }) =>
+				'| ' +
+				[
+					displayName,
+					prettyBytes(totalSize),
+					prettyBytes(embedSize),
+					[...resourceSize.entries()]
+						.sort(([, a], [, b]) => b - a)
+						.map(
+							([resourceType, size]) =>
+								`\`${resourceType}\`: ${prettyBytes(size)}`,
+						)
+						.join(', '),
+				].join(' | ') +
+				' |',
+		);
+	return [
+		'| Name | Total size | Embed size | Resources |',
+		'| ---- | ---------- | ---------- | --------- |',
+		...rows,
+	];
+};
+
+// -- Script -- //
+
+const lines = ['# Largest thrashers on network fronts'];
+
+for (const id of fronts) {
+	lines.push(
+		'',
+		'',
+		`## [${id.toUpperCase()} Front](https://www.theguardian.com/${id}) `,
+		'',
+	);
+	const data = await getFrontThrashers(id);
+
+	lines.push(...getTable(data));
+}
+
+const body = lines.join('\n');
+
+if (!octokit) {
+	console.log(body);
+	Deno.exit();
+}
+
+const {
+	data: { html_url },
+} = await octokit.rest.issues.update({
+	owner: 'guardian',
+	repo: 'dotcom-rendering',
+	issue_number: 5856,
+	body,
+});
+
+console.log('Updated issue:', html_url);
+
+Deno.exit();