Fix toc plugin not picking up fragments (#583)

jpmorganchase · Mar 14, 2024 · c21f28e · c21f28e
1 parent 53894cd
commit c21f28e
Show file tree

Hide file tree

Showing 16 changed files with 448 additions and 200 deletions.
diff --git a/.changeset/gorgeous-melons-swim.md b/.changeset/gorgeous-melons-swim.md
@@ -0,0 +1,5 @@
+---
+'@jpmorganchase/mosaic-plugins': patch
+---
+
+Fix fragments not being picked up by the table of contents plugin
diff --git a/jest.config.server.js b/jest.config.server.js
@@ -26,6 +26,6 @@ module.exports = {
   ],
   transformIgnorePatterns: [
     // Ignore node_modules except for the following packages (required to run Plugin tests)
-    '/node_modules/(?!(unified|bail|trough|vfile.*|unist.*|remark.*|micromark.*|.*character-reference.*|estree-util.*|ccount|.*mdast.*|is-.*|.*entities.*)/)'
+    '/node_modules/(?!(unified|bail|trough|vfile.*|unist.*|remark.*|micromark.*|.*character-reference.*|estree-util.*|ccount|.*mdast.*|is-.*|.*entities.*|zwitch|longest-streak)/)'
   ]
 };
diff --git a/packages/plugins/package.json b/packages/plugins/package.json
@@ -64,7 +64,8 @@
     "remark-stringify": "^10.0.2",
     "rxjs": "^7.5.5",
     "unified": "^10.1.2",
-    "unist-util-visit": "^4.1.1",
+    "unist-util-visit": "^5.0.0",
+    "mdast-util-directive": "^3.0.0",
     "uuid": "^7.0.3",
     "vfile-reporter": "^7.0.5"
   }

diff --git a/packages/plugins/src/$AliasPlugin.ts b/packages/plugins/src/$AliasPlugin.ts
@@ -1,14 +1,7 @@
 import path from 'path';
-import { escapeRegExp } from 'lodash-es';
 import type { Page, Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import PluginError from './utils/PluginError.js';
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
+import { createPageTest } from './utils/createPageTest.js';
 
 interface AliasPluginPage extends Page {
   aliases?: string[];

diff --git a/packages/plugins/src/$CodeModPlugin.ts b/packages/plugins/src/$CodeModPlugin.ts
@@ -1,14 +1,6 @@
 import type { Page, Plugin as PluginType } from '@jpmorganchase/mosaic-types';
-import { escapeRegExp } from 'lodash-es';
-import path from 'path';
 import PluginError from './utils/PluginError.js';
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
+import { createPageTest } from './utils/createPageTest.js';
 
 interface CodeModPluginPage extends Page {
   frameOverrides?: Record<string, unknown>;

diff --git a/packages/plugins/src/$RefPlugin.ts b/packages/plugins/src/$RefPlugin.ts
@@ -1,13 +1,13 @@
-import path from 'node:path';
 import os from 'node:os';
-import { reduce, omit, escapeRegExp } from 'lodash-es';
+import { reduce, omit } from 'lodash-es';
 import { $RefParser } from '@apidevtools/json-schema-ref-parser';
 import type { Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import deepmerge from 'deepmerge';
 
 import normaliseRefs from './utils/normaliseRefs.js';
 import PluginError from './utils/PluginError.js';
 import { mergePageContent } from './utils/mergePageContent.js';
+import { createPageTest } from './utils/createPageTest.js';
 
 const isWindows = /^win/.test(os.platform());
 const windowsDrivePattern = /^([a-z]):/i;
@@ -65,13 +65,6 @@ const findKeys = (obj, targetProp, pathParts: string[] = []): [string[], string]
     []
   );
 
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
-
 interface RefsPluginConfigData {
   refs: { [key: string]: { $$path: string[]; $$value: string }[] };
   globalRefs: { [key: string]: { $$path: string[]; $$value: string }[] };

diff --git a/packages/plugins/src/$TagPlugin.ts b/packages/plugins/src/$TagPlugin.ts
@@ -1,14 +1,8 @@
 import path from 'path';
-import { escapeRegExp, reduce } from 'lodash-es';
+import { reduce } from 'lodash-es';
 import type { Page, Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import PluginError from './utils/PluginError.js';
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
+import { createPageTest } from './utils/createPageTest.js';
 
 const findKeys = (obj, targetProp, pathParts: string[] = []) =>
   reduce<string, { $$path: string[]; $$value: string }[]>(

diff --git a/packages/plugins/src/BreadcrumbsPlugin.ts b/packages/plugins/src/BreadcrumbsPlugin.ts
@@ -1,16 +1,9 @@
 import path from 'node:path';
-import { escapeRegExp } from 'lodash-es';
 import type { Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import PluginError from './utils/PluginError.js';
+import { createPageTest } from './utils/createPageTest.js';
 import { SidebarPluginPage } from './SidebarPlugin.js';
 
-const createPageTest = (ignorePages, pageExtensions) => {
-  const extTest = new RegExp(`${pageExtensions.map(ext => escapeRegExp(ext)).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(ignore => escapeRegExp(ignore)).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-};
-
 export type Breadcrumb = { label: string; path: string; id: string };
 
 export interface BreadcrumbsPluginPage extends SidebarPluginPage {

diff --git a/packages/plugins/src/FragmentPlugin.ts b/packages/plugins/src/FragmentPlugin.ts
@@ -1,127 +1,73 @@
-import path from 'path';
-import type { Plugin as PluginType } from '@jpmorganchase/mosaic-types';
-import { escapeRegExp } from 'lodash-es';
+import path from 'node:path';
+import type { Page, Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import { remark } from 'remark';
 import remarkMdx from 'remark-mdx';
 import remarkDirective from 'remark-directive';
-import { visitParents } from 'unist-util-visit-parents';
+import { visit } from 'unist-util-visit';
+import type { Content, Root } from 'mdast';
 import PluginError from './utils/PluginError.js';
+import { createPageTest } from './utils/createPageTest.js';
+
+function processFragments(
+  tree: Root,
+  pages: Page[],
+  isNonHiddenPage: (path: string) => boolean,
+  fullPath: string
+) {
+  visit(tree, (node, index, parent) => {
+    if (
+      node.type === 'containerDirective' ||
+      node.type === 'leafDirective' ||
+      node.type === 'textDirective'
+    ) {
+      if (node.name !== 'fragment') return;
+      const attributes = node.attributes ?? {};
+      const { src } = attributes;
+
+      if (!src) {
+        console.error("Fragment directive requires a 'src' attribute. Skipping.");
+        return;
+      }
 
-interface FragmentPluginPage {
-  fullPath: string;
-  content: string;
-}
-
-const createPageTest = (ignorePages, pageExtensions) => {
-  const extTest = new RegExp(`${pageExtensions.map(ext => escapeRegExp(ext)).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(ignore => escapeRegExp(ignore)).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-};
-
-function getFullPath(fullPath: string, relativePath: string): string {
-  const pathSegments = fullPath.split('/');
-  const relativeSegments = relativePath.split('/');
-
-  pathSegments.pop();
-
-  for (const segment of relativeSegments) {
-    if (segment === '..') {
-      pathSegments.pop();
-    } else if (segment !== '.') {
-      pathSegments.push(segment);
-    }
-  }
-  return pathSegments.join('/');
-}
-
-async function processTree(tree, serialiser, mutableFilesystem, fullPath, isNonHiddenPage) {
-  const nodesToProcess = [];
-
-  visitParents(tree, (node, ancestors) => {
-    if (node.type === 'code') {
-      return;
-    }
-
-    const match = node.name === 'fragment' && node.attributes.src;
-    if (match) {
-      const parent = ancestors[ancestors.length - 1];
-      const index = parent.children.indexOf(node);
-      nodesToProcess.push({ node, parent, index });
+      const fragmentFullPath = path.posix.join(path.dirname(fullPath), src);
+      const isHidden = !isNonHiddenPage(fragmentFullPath);
+      const fragmentPage = pages.find(page => page.fullPath === fragmentFullPath);
+      if (isHidden || !fragmentPage) {
+        console.warn(`Invalid file reference: '${node.attributes.src}'. Skipping.`);
+      } else {
+        // Create a new node with the content from fragmentPage.content
+        const newNode: Content = {
+          type: 'html',
+          value: fragmentPage.content
+        };
+        // Replace the original node with the newNode in the tree
+        parent.children.splice(index, 1, newNode);
+      }
     }
   });
-
-  for (const { node, parent, index } of nodesToProcess) {
-    const fragmentFullPath = getFullPath(fullPath, node.attributes.src);
-    if (!isNonHiddenPage(fragmentFullPath)) {
-      console.warn(`Invalid file reference: '${node.attributes.src}'. Skipping.`);
-    } else {
-      const fragmentPage = await serialiser.deserialise(
-        fragmentFullPath,
-        await mutableFilesystem.promises.readFile(fragmentFullPath)
-      );
-
-      // Create a new node with the content from fragmentPage.content
-      const newNode = {
-        type: 'html',
-        value: fragmentPage.content
-      };
-
-      // Replace the original node with the newNode in the tree
-      parent.children.splice(index, 1, newNode);
-    }
-  }
-  return tree;
 }
 
-const FragmentPlugin: PluginType<FragmentPluginPage, unknown, unknown> = {
-  async $beforeSend(mutableFilesystem, { serialiser, ignorePages, pageExtensions }) {
-    const pages = await Promise.all(
-      (
-        (await mutableFilesystem.promises.glob('**', {
-          onlyFiles: true,
-          ignore: ignorePages.map(ignore => `**/${ignore}`),
-          cwd: '/'
-        })) as string[]
-      ).map(async pagePath => {
-        const deserialisedPage = await serialiser.deserialise(
-          pagePath,
-          await mutableFilesystem.promises.readFile(pagePath)
-        );
-        return deserialisedPage;
-      })
-    );
+const processor = remark().use(remarkMdx).use(remarkDirective);
 
+const FragmentPlugin: PluginType = {
+  async $afterSource(pages, { ignorePages, pageExtensions }) {
     const isNonHiddenPage = createPageTest(ignorePages, pageExtensions);
-
     for (const page of pages) {
       try {
         const { fullPath } = page;
         if (!isNonHiddenPage(fullPath)) {
           continue;
         }
 
-        const tree = remark().use(remarkMdx).use(remarkDirective).parse(page.content);
-        const processedTree = await processTree(
-          tree,
-          serialiser,
-          mutableFilesystem,
-          fullPath,
-          isNonHiddenPage
-        );
-
-        page.content = remark()
-          .data('settings', { fences: true })
-          .use(remarkMdx)
-          .use(remarkDirective)
-          .stringify(processedTree);
-
-        const updatedFileData = await serialiser.serialise(fullPath, page);
-        await mutableFilesystem.promises.writeFile(fullPath, updatedFileData);
+        const tree = processor.parse(page.content);
+        processFragments(tree, pages, isNonHiddenPage, page.fullPath);
+        page.content = processor.stringify(tree);
       } catch (e) {
         throw new PluginError(e.message, page.fullPath);
       }
     }
+
+    return pages;
   }
 };
 

diff --git a/packages/plugins/src/LazyPagePlugin.ts b/packages/plugins/src/LazyPagePlugin.ts
@@ -1,17 +1,10 @@
 import type { Page, Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import fs from 'fs';
 import fsExtra from 'fs-extra';
-import { escapeRegExp } from 'lodash-es';
 import path from 'path';
 import { TDataOut } from 'memfs';
 import { mergePageContent } from './utils/mergePageContent.js';
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
+import { createPageTest } from './utils/createPageTest.js';
 
 function createFileGlob(url, pageExtensions) {
   if (pageExtensions.length === 1) {

diff --git a/packages/plugins/src/PagesWithoutFileExtPlugin.ts b/packages/plugins/src/PagesWithoutFileExtPlugin.ts
@@ -1,14 +1,7 @@
 import type { Plugin as PluginType } from '@jpmorganchase/mosaic-types';
 import { escapeRegExp } from 'lodash-es';
-import path from 'path';
 import PluginError from './utils/PluginError.js';
-
-function createPageTest(ignorePages, pageExtensions) {
-  const extTest = new RegExp(`${pageExtensions.map(escapeRegExp).join('|')}$`);
-  const ignoreTest = new RegExp(`${ignorePages.map(escapeRegExp).join('|')}$`);
-  return file =>
-    !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
-}
+import { createPageTest } from './utils/createPageTest.js';
 
 /**
  * Plugin that creates aliases without the file extension for every page emitted by the source.

diff --git a/packages/plugins/src/ReadingTimePlugin.ts b/packages/plugins/src/ReadingTimePlugin.ts
@@ -5,7 +5,7 @@ import getReadingTime from 'reading-time';
 import markdown from 'remark-parse';
 import { unified } from 'unified';
 import { visit } from 'unist-util-visit';
-import type { Node } from 'unist';
+import type { Literal, Node } from 'unist';
 
 const createPageTest = (ignorePages, pageExtensions) => {
   const extTest = new RegExp(`${pageExtensions.map(ext => escapeRegExp(ext)).join('|')}$`);
@@ -14,37 +14,33 @@ const createPageTest = (ignorePages, pageExtensions) => {
     !ignoreTest.test(file) && extTest.test(file) && !path.basename(file).startsWith('.');
 };
 
-type LeafNode = Node & {
-  value?: string;
-};
-
 export interface ReadingTimePluginPage extends Page {
   readingTime?: ReturnType<typeof getReadingTime>;
 }
 
+function isValid(node: Node): node is Literal {
+  return node.type === 'text' || node.type === 'code';
+}
+
+const processor = unified().use(markdown);
+
 /**
  * Calculates reading time of MDX pages and adds to frontmatter
  */
 const ReadingTimePlugin: PluginType<ReadingTimePluginPage> = {
   async $afterSource(pages, { ignorePages, pageExtensions }) {
-    const processor = unified().use(markdown);
-
     if (pageExtensions.includes('.mdx')) {
       for (const page of pages) {
         const isNonHiddenPage = createPageTest(ignorePages, ['.mdx']);
         if (!isNonHiddenPage(page.fullPath)) {
           continue;
         }
-        const tree: Node = await processor.parse(page.content);
+        const tree = processor.parse(page.content);
         let textContent = '';
 
-        visit(
-          tree,
-          node => node.type === 'text' || node.type === 'code',
-          (node: LeafNode) => {
-            textContent += node.value;
-          }
-        );
+        visit(tree, isValid, (node: Literal) => {
+          textContent += node.value;
+        });
         page.readingTime = getReadingTime(textContent);
       }
     }