test: add full routes test (DIYgod#14934)

* test: add critical routes test * fix: youtube channels config * test: add full routes test * test: output full routes test result * test: build routes before test * test: use one reporter * test: fix cache test errors * test: fix cache test errors * chore: puppeteerrc.cjs * chore: puppeteerrc.cjs * chore: puppeteerrc.cjs * fix: build script * chore: puppeteerrc rename * test: route test names * chore: pass test result to docs
axelburks · Mar 24, 2024 · b774c60 · b774c60
1 parent 2fc3b3a
commit b774c60
Show file tree

Hide file tree

Showing 17 changed files with 247 additions and 198 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -33,7 +33,11 @@ jobs:
       - name: Install dependencies (yarn)
         run: pnpm i
       - name: Build assets
-        run: npm run build
+        run: pnpm build
+      - name: Build full routes test result
+        run: pnpm vitest:fullroutes
+      - name: Build docs
+        run: pnpm build:docs
       - name: Commit files
         run: |
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -5,7 +5,6 @@ on:
     branches-ignore:
       - 'dependabot/**'
     paths:
-      - 'test/**'
       - 'lib/**'
       - 'package.json'
       - 'pnpm-lock.yaml'
@@ -43,6 +42,8 @@ jobs:
         run: pnpm i
       - name: Run postinstall script for dependencies
         run: pnpm rb
+      - name: Build routes
+        run: pnpm build
       - name: Test all and generate coverage
         run: pnpm run vitest:coverage
         env:
@@ -84,6 +85,8 @@ jobs:
         run: pnpm i
       - name: Run postinstall script for dependencies
         run: pnpm rb
+      - name: Build routes
+        run: pnpm build
       - name: Install Chromium
         if: ${{ matrix.chromium.dependency != '' }}
         # 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means

diff --git a/.puppeteerrc.js → .puppeteerrc.cjs b/.puppeteerrc.js → .puppeteerrc.cjs
diff --git a/Dockerfile b/Dockerfile
@@ -84,7 +84,7 @@ FROM node:21-bookworm-slim AS chromium-downloader
 # Yeah, downloading Chromium never needs those dependencies below.
 
 WORKDIR /app
-COPY ./.puppeteerrc.js /app/
+COPY ./.puppeteerrc.cjs /app/
 COPY --from=dep-version-parser /ver/.puppeteer_version /app/.puppeteer_version
 
 ARG TARGETPLATFORM

diff --git a/lib/middleware/cache.test.ts b/lib/middleware/cache.test.ts
@@ -1,23 +1,16 @@
-import { describe, expect, it, vi, afterEach, afterAll, beforeAll } from 'vitest';
+import { describe, expect, it, vi, afterEach } from 'vitest';
 import Parser from 'rss-parser';
 import wait from '@/utils/wait';
 
-const parser = new Parser();
+process.env.CACHE_EXPIRE = '1';
+process.env.CACHE_CONTENT_EXPIRE = '3';
 
-beforeAll(() => {
-    process.env.CACHE_EXPIRE = '1';
-    process.env.CACHE_CONTENT_EXPIRE = '3';
-});
+const parser = new Parser();
 
 afterEach(() => {
-    delete process.env.CACHE_TYPE;
     vi.resetModules();
 });
 
-afterAll(() => {
-    delete process.env.CACHE_EXPIRE;
-});
-
 describe('cache', () => {
     it('memory', async () => {
         process.env.CACHE_TYPE = 'memory';

diff --git a/lib/registry.test.ts b/lib/registry.test.ts
@@ -1,55 +1,8 @@
-import { describe, expect, it, afterAll } from 'vitest';
-process.env.SOCKET = 'socket';
-
+import { describe, expect, it } from 'vitest';
 import app from '@/app';
-import Parser from 'rss-parser';
-const parser = new Parser();
 import { config } from '@/config';
 
-afterAll(() => {
-    delete process.env.SOCKET;
-});
-
-async function checkRSS(response) {
-    const checkDate = (date) => {
-        expect(date).toEqual(expect.any(String));
-        expect(Date.parse(date)).toEqual(expect.any(Number));
-        expect(Date.now() - +new Date(date)).toBeGreaterThan(-1000 * 60 * 60 * 24 * 5);
-        expect(Date.now() - +new Date(date)).toBeLessThan(1000 * 60 * 60 * 24 * 30 * 12 * 10);
-    };
-
-    const parsed = await parser.parseString(await response.text());
-
-    expect(parsed).toEqual(expect.any(Object));
-    expect(parsed.title).toEqual(expect.any(String));
-    expect(parsed.title).not.toBe('RSSHub');
-    expect(parsed.description).toEqual(expect.any(String));
-    expect(parsed.link).toEqual(expect.any(String));
-    expect(parsed.lastBuildDate).toEqual(expect.any(String));
-    expect(parsed.ttl).toEqual(Math.trunc(config.cache.routeExpire / 60) + '');
-    expect(parsed.items).toEqual(expect.any(Array));
-    checkDate(parsed.lastBuildDate);
-
-    // check items
-    const guids: (string | undefined)[] = [];
-    for (const item of parsed.items) {
-        expect(item).toEqual(expect.any(Object));
-        expect(item.title).toEqual(expect.any(String));
-        expect(item.link).toEqual(expect.any(String));
-        expect(item.content).toEqual(expect.any(String));
-        expect(item.guid).toEqual(expect.any(String));
-        if (item.pubDate) {
-            expect(item.pubDate).toEqual(expect.any(String));
-            checkDate(item.pubDate);
-        }
-
-        // guid must be unique
-        expect(guids).not.toContain(item.guid);
-        guids.push(item.guid);
-    }
-}
-
-describe('router', () => {
+describe('registry', () => {
     // root
     it(`/`, async () => {
         const response = await app.request('/');
@@ -58,14 +11,6 @@ describe('router', () => {
         expect(response.headers.get('cache-control')).toBe('no-cache');
     });
 
-    // route
-    it(`/test/1`, async () => {
-        const response = await app.request('/test/1');
-        expect(response.status).toBe(200);
-
-        await checkRSS(response);
-    });
-
     // robots.txt
     it('/robots.txt', async () => {
         config.disallowRobot = false;

diff --git a/lib/registry.ts b/lib/registry.ts
@@ -7,8 +7,6 @@ import { serveStatic } from '@hono/node-server/serve-static';
 
 import index from '@/routes/index';
 import robotstxt from '@/routes/robots.txt';
-import { namespace as testNamespace } from './routes/test/namespace';
-import { route as testRoute } from '@/routes/test/index';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -27,18 +25,9 @@ let namespaces: Record<
 
 switch (process.env.NODE_ENV) {
     case 'test':
-        modules = {
-            '/test/namespace.ts': {
-                namespace: testNamespace,
-            },
-            '/test/index.ts': {
-                route: testRoute,
-            },
-        };
-        break;
     case 'production':
-        // eslint-disable-next-line n/no-unpublished-require
-        namespaces = require('../assets/build/routes.json');
+        // @ts-expect-error
+        namespaces = await import('../assets/build/routes.json');
         break;
     default:
         modules = directoryImport({

diff --git a/lib/routes.test.ts b/lib/routes.test.ts
@@ -0,0 +1,78 @@
+import { describe, expect, it } from 'vitest';
+import app from '@/app';
+import Parser from 'rss-parser';
+const parser = new Parser();
+import { config } from '@/config';
+
+process.env.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN = 'true';
+
+const routes = {
+    '/test/:id': '/test/1',
+};
+if (process.env.FULL_ROUTES_TEST) {
+    const { namespaces } = await import('@/registry');
+    for (const namespace in namespaces) {
+        for (const route in namespaces[namespace].routes) {
+            const requireConfig = namespaces[namespace].routes[route].features?.requireConfig;
+            let configs;
+            if (typeof requireConfig !== 'boolean') {
+                configs = requireConfig
+                    ?.filter((config) => !config.optional)
+                    .map((config) => config.name)
+                    .filter((name) => name !== 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN');
+            }
+            if (namespaces[namespace].routes[route].example && !configs?.length) {
+                routes[`/${namespace}${route}`] = namespaces[namespace].routes[route].example;
+            }
+        }
+    }
+}
+
+async function checkRSS(response) {
+    const checkDate = (date) => {
+        expect(date).toEqual(expect.any(String));
+        expect(Date.parse(date)).toEqual(expect.any(Number));
+        expect(Date.now() - +new Date(date)).toBeGreaterThan(-1000 * 60 * 60 * 24 * 5);
+        expect(Date.now() - +new Date(date)).toBeLessThan(1000 * 60 * 60 * 24 * 30 * 12 * 10);
+    };
+
+    const parsed = await parser.parseString(await response.text());
+
+    expect(parsed).toEqual(expect.any(Object));
+    expect(parsed.title).toEqual(expect.any(String));
+    expect(parsed.title).not.toBe('RSSHub');
+    expect(parsed.description).toEqual(expect.any(String));
+    expect(parsed.link).toEqual(expect.any(String));
+    expect(parsed.lastBuildDate).toEqual(expect.any(String));
+    expect(parsed.ttl).toEqual(Math.trunc(config.cache.routeExpire / 60) + '');
+    expect(parsed.items).toEqual(expect.any(Array));
+    checkDate(parsed.lastBuildDate);
+
+    // check items
+    const guids: (string | undefined)[] = [];
+    for (const item of parsed.items) {
+        expect(item).toEqual(expect.any(Object));
+        expect(item.title).toEqual(expect.any(String));
+        expect(item.link).toEqual(expect.any(String));
+        expect(item.content).toEqual(expect.any(String));
+        expect(item.guid).toEqual(expect.any(String));
+        if (item.pubDate) {
+            expect(item.pubDate).toEqual(expect.any(String));
+            checkDate(item.pubDate);
+        }
+
+        // guid must be unique
+        expect(guids).not.toContain(item.guid);
+        guids.push(item.guid);
+    }
+}
+
+describe('routes', () => {
+    for (const route in routes) {
+        it.concurrent(route, async () => {
+            const response = await app.request(routes[route]);
+            expect(response.status).toBe(200);
+            await checkRSS(response);
+        });
+    }
+});
diff --git a/lib/routes/afdian/dynamic.ts b/lib/routes/afdian/dynamic.ts
@@ -1,4 +1,5 @@
 import got from '@/utils/got';
+import type { Route } from '@/types';
 
 export const route: Route = {
     path: '/dynamic/:uid?',

diff --git a/lib/routes/bilibili/dynamic.ts b/lib/routes/bilibili/dynamic.ts
@@ -121,7 +121,6 @@ async function handler(ctx) {
         headers: {
             Referer: `https://space.bilibili.com/${uid}/`,
         },
-        transformResponse: [(data) => data],
     });
     const cards = JSONbig.parse(response.body).data.cards;
 

diff --git a/lib/routes/test/index.ts b/lib/routes/test/index.ts
@@ -79,13 +79,9 @@ async function handler(ctx) {
             break;
 
         case 'cache': {
-            const description = await cache.tryGet(
-                'test',
-                () => ({
-                    text: `Cache${++cacheIndex}`,
-                }),
-                config.cache.routeExpire * 2
-            );
+            const description = await cache.tryGet('test', () => ({
+                text: `Cache${++cacheIndex}`,
+            }));
             item.push({
                 title: 'Cache Title',
                 description: description.text,

diff --git a/lib/routes/youtube/channel.ts b/lib/routes/youtube/channel.ts
@@ -10,7 +10,12 @@ export const route: Route = {
     example: '/youtube/channel/UCDwDMPOZfxVV0x_dz0eQ8KQ',
     parameters: { id: 'YouTube channel id', embed: 'Default to embed the video, set to any value to disable embedding' },
     features: {
-        requireConfig: false,
+        requireConfig: [
+            {
+                name: 'YOUTUBE_KEY',
+                description: ' YouTube API Key, support multiple keys, split them with `,`, [API Key application](https://console.developers.google.com/)',
+            },
+        ],
         requirePuppeteer: false,
         antiCrawler: false,
         supportBT: false,

diff --git a/package.json b/package.json
@@ -19,20 +19,23 @@
     "files": [
         "lib"
     ],
+    "type": "module",
     "scripts": {
         "build": "tsx scripts/workflow/build-routes.ts",
-        "dev": "cross-env NODE_ENV=dev tsx watch --no-cache lib/index.ts",
-        "dev:cache": "cross-env NODE_ENV=production tsx watch lib/index.ts",
+        "build:docs": "tsx scripts/workflow/build-docs.ts",
+        "dev": "NODE_ENV=dev tsx watch --no-cache lib/index.ts",
+        "dev:cache": "NODE_ENV=production tsx watch lib/index.ts",
         "format": "eslint --cache --fix \"**/*.{ts,js,yml}\" && prettier \"**/*.{ts,js,json}\" --write",
         "format:check": "eslint --cache \"**/*.{ts,js,yml}\" && prettier \"**/*.{ts,js,json}\" --check",
         "format:staged": "lint-staged",
-        "vitest": "cross-env NODE_ENV=test vitest",
-        "vitest:coverage": "cross-env NODE_ENV=test vitest --coverage.enabled --reporter=junit",
-        "vitest:watch": "cross-env NODE_ENV=test vitest --watch",
+        "vitest": "NODE_ENV=test vitest",
+        "vitest:fullroutes": "NODE_ENV=test FULL_ROUTES_TEST=true vitest --reporter=json --reporter=default --outputFile=\"./assets/build/test-full-routes.json\" routes",
+        "vitest:coverage": "NODE_ENV=test vitest --coverage.enabled --reporter=junit",
+        "vitest:watch": "NODE_ENV=test vitest --watch",
         "lint": "eslint --cache .",
         "prepare": "husky || true",
         "profiling": "NODE_ENV=production tsx --prof lib/index.ts",
-        "start": "cross-env NODE_ENV=production tsx lib/index.ts",
+        "start": "NODE_ENV=production tsx lib/index.ts",
         "test": "npm run format:check && npm run vitest:coverage"
     },
     "lint-staged": {
@@ -147,7 +150,6 @@
         "@typescript-eslint/parser": "7.3.1",
         "@vercel/nft": "0.26.4",
         "@vitest/coverage-v8": "1.4.0",
-        "cross-env": "7.0.3",
         "eslint": "8.57.0",
         "eslint-config-prettier": "9.1.0",
         "eslint-nibble": "8.1.0",

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml