📑 Add literalinclude directive

See jupyter-book/jupyterlab-myst#189

.changeset/flat-suits-allow.md

@@ -0,0 +1,5 @@
+---
+'myst-spec-ext': patch
+---
+
+Add `include` node, that implements the `literalinclude` directive

.changeset/gold-weeks-appear.md

@@ -0,0 +1,5 @@
+---
+'myst-directives': patch
+---
+
+Remove the codeBlockDirective, this is now the same as the `codeDirective`.

.changeset/orange-planes-promise.md

@@ -0,0 +1,6 @@
+---
+'myst-directives': patch
+'myst-cli': patch
+---
+
+Add `literalinclude` directive

docs/code.md

@@ -83,13 +83,79 @@ caption (string)
 name (string)
 : The target label for the code-block, can be used by `ref` and `numref` roles.
 
-```{note}
+```{note} Alternative implementations
 :class: dropdown
-# Alternative implementations
 
 The parser also supports the `docutils` implementation (see [docutils documentation](https://docutils.sourceforge.io/docs/ref/rst/directives.html#code)) of a `{code}` directive, which only supports the `number-lines` option.
 
 It is recommended to use the more fully featured `code-block` directive documented above, or a simple markdown code block.
 
 All implementations are resolved to the same `code` type in the abstract syntax tree.
 ```
+
+## Including Files
+
+If your code is in a separate file you can use the `literalinclude` directive (or the `include` directive with the `literal` flag).
+This directive is helpful for showing code snippets without duplicating your content.
+
+For example, a `literalinclude` of a snippet of the `myst.yml` such as:
+
+````markdown
+```{literalinclude} myst.yml
+:start-at: project
+:end-before: references
+:lineno-match:
+```
+````
+
+creates a snippet that has matching line numbers, and starts at a line including `"project"` and ends before the line including `"references"`.
+
+```{literalinclude} myst.yml
+:start-at: project
+:end-before: references
+:lineno-match:
+```
+
+:::{note} Auto Reload
+If you are working with the auto-reload (e.g. `myst start`), currently you will need to save the file that includes the code for the contents to update.
+:::
+
+## `include` Reference
+
+The argument of an include directive is the file path, relative to the file from which it was referenced.
+By default the file will be parsed using MyST, you can also set the file to be `literal`, which will show as a code-block; this is the same as using the `literalinclude` directive.
+If in literal mode, the directive also accepts all of the options from the `code-block` (e.g. `:linenos:`).
+
+To select a portion of the file to be shown using the `start-at`/`start-after` selectors with the `end-before`/`end-at`, which use a snippet of included text.
+Alternatively, you can explicitly select the lines (e.g. `1,3,5-10,20-`) or the `start-line`/`end-line` (which is zero based for compatibility with Sphinx).
+
+literal (boolean)
+: Flag the include block as literal, and show the contents as a code block. This can also be set automatically by setting the `language` or using the `literalinclude` directive.
+
+lang (string)
+: The language of the code to be highlighted as. If set, this automatically changes an `include` into a `literalinclude`.
+: You can alias this as `language` or `code`
+
+start-line (number)
+: Only the content starting from this line will be included. The first line has index 0 and negative values count from the end.
+
+start-at (string)
+: Only the content after and including the first occurrence of the specified text in the external data file will be included.
+
+start-after (string)
+: Only the content after the first occurrence of the specified text in the external data file will be included.
+
+end-line (number)
+: Only the content up to (but excluding) this line will be included.
+
+end-at (string)
+: Only the content up to and including the first occurrence of the specified text in the external data file (but after any start-after text) will be included.
+
+end-before (string)
+: Only the content before the first occurrence of the specified text in the external data file (but after any start-after text) will be included.
+
+lines (string)
+: Specify exactly which lines to include, starting at 1. For example, `1,3,5-10,20-` includes the lines 1, 3, 5 to 10 and lines 20 to the last line.
+
+lineno-match (boolean)
+: Display the original line numbers, correct only when the selection consists of contiguous lines.

packages/myst-cli/src/process/mdast.ts

@@ -155,7 +155,7 @@ export async function transformMdast(
   cache.$internalReferences[file] = state;
   // Import additional content from mdast or other files
   importMdastFromJson(session, file, mdast);
-  includeFilesDirective(session, file, mdast);
+  includeFilesDirective(session, vfile, file, mdast);
   // This needs to come before basic transformations since it may add labels to blocks
   liftCodeMetadataToBlock(session, file, mdast);
 

packages/myst-cli/src/transforms/include.spec.ts

@@ -0,0 +1,36 @@
+import { describe, expect, test } from 'vitest';
+import { filterIncludedContent } from './include';
+import { VFile } from 'vfile';
+
+describe('filterIncludedContent', () => {
+  test.each([
+    [{ startAt: 'ok' }, 'ok\nreally\ncool', 2, 0],
+    [{ startAt: 'ok', endBefore: 'cool' }, 'ok\nreally', 2, 0],
+    [{ startAt: 'ok', endBefore: 'ok' }, 'ok\nreally\ncool', 2, 1],
+    [{ startAt: 'ok', endAt: 'cool' }, 'ok\nreally\ncool', 2, 0],
+    [{ startAfter: 'k', endBefore: 'cool' }, 'really', 3, 0],
+    [{ endBefore: 'cool' }, 'some\nok\nreally', 1, 0],
+    [{ startAt: 'really' }, 'really\ncool', 3, 0],
+    [{ startAfter: 'really' }, 'cool', 4, 0],
+    [{ lines: [1, 3] }, 'some\nreally', 1, 0],
+    [{ lines: [[1, 3]] }, 'some\nok\nreally', 1, 0],
+    [{ lines: [1, [3]] }, 'some\nreally\ncool', 1, 0],
+    [{ lines: [2, 1, 2] }, 'ok\nsome\nok', 2, 0],
+    [{ lines: [1, -1] }, 'some\ncool', 1, 0],
+    [{ lines: [-1] }, 'cool', 4, 0],
+    [{ lines: [1, [-1]] }, 'some\ncool', 1, 0],
+    [{ lines: [1, [-2]] }, 'some\nreally\ncool', 1, 0],
+    [{ lines: [1, [-2, -1]] }, 'some\nreally\ncool', 1, 0],
+    [{ lines: [1, [-1, -2]] }, 'some', 1, 1],
+  ])('%s', (t, a, sln, w) => {
+    const vfile = new VFile();
+    const { content, startingLineNumber } = filterIncludedContent(
+      vfile,
+      t as any,
+      'some\nok\nreally\ncool',
+    );
+    expect(content).toEqual(a);
+    expect(startingLineNumber).toEqual(sln);
+    expect(vfile.messages.length).toBe(w);
+  });
+});

packages/myst-cli/src/transforms/include.ts

@@ -1,27 +1,166 @@
 import fs from 'node:fs';
-import type { GenericNode, GenericParent } from 'myst-common';
+import { fileError, fileWarn, type GenericNode, type GenericParent } from 'myst-common';
+import type { Code, Container, Include } from 'myst-spec-ext';
 import { parseMyst } from '../process/index.js';
 import { selectAll } from 'unist-util-select';
 import { join, dirname } from 'node:path';
 import type { ISession } from '../session/types.js';
+import type { Caption } from 'myst-spec';
+import type { VFile } from 'vfile';
 
 /**
  * This is the {include} directive, that loads from disk.
  *
  * RST documentation:
  *  - https://docutils.sourceforge.io/docs/ref/rst/directives.html#including-an-external-document-fragment
  */
-export function includeFilesDirective(session: ISession, filename: string, mdast: GenericParent) {
-  const includeNodes = selectAll('include', mdast) as GenericNode[];
+export function includeFilesDirective(
+  session: ISession,
+  vfile: VFile,
+  filename: string,
+  mdast: GenericParent,
+) {
+  const includeNodes = selectAll('include', mdast) as Include[];
   const dir = dirname(filename);
   includeNodes.forEach((node) => {
     const file = join(dir, node.file);
     if (!fs.existsSync(file)) {
-      session.log.error(`Include Directive: Could not find "${file}" in "${filename}"`);
+      fileError(vfile, `Include Directive: Could not find "${file}" in "${filename}"`);
       return;
     }
-    const content = fs.readFileSync(file).toString();
-    const children = parseMyst(session, content, filename).children as GenericNode[];
-    node.children = children;
+    const rawContent = fs.readFileSync(file).toString();
+    const { content, startingLineNumber } = filterIncludedContent(vfile, node.filter, rawContent);
+    let children: GenericNode[];
+    if (node.literal) {
+      const code: Code = {
+        type: 'code',
+        value: content,
+      };
+      if (node.startingLineNumber === 'match') {
+        // Replace the starting line number if it should match
+        node.startingLineNumber = startingLineNumber;
+      }
+      // Move the code attributes to the code block
+      (
+        [
+          'lang',
+          'emphasizeLines',
+          'showLineNumbers',
+          'startingLineNumber',
+          'label',
+          'identifier',
+        ] as const
+      ).forEach((attr) => {
+        if (!node[attr]) return;
+        code[attr] = node[attr] as any;
+        delete node[attr];
+      });
+      if (!node.caption) {
+        children = [code];
+      } else {
+        const caption: Caption = {
+          type: 'caption',
+          children: [
+            {
+              type: 'paragraph',
+              children: node.caption as any[],
+            },
+          ],
+        };
+        const container: Container = {
+          type: 'container',
+          kind: 'code' as any,
+          // Move the label to the container
+          label: code.label,
+          identifier: code.identifier,
+          children: [code as any, caption],
+        };
+        delete code.label;
+        delete code.identifier;
+        children = [container];
+      }
+    } else {
+      children = parseMyst(session, content, filename).children;
+    }
+    node.children = children as any;
   });
 }
+
+function index(n: number, total: number): [number, number] | null {
+  if (n > 0) return [n - 1, n];
+  if (n < 0) return [total + n, total + n + 1];
+  return null;
+}
+
+export function filterIncludedContent(
+  vfile: VFile,
+  filter: Include['filter'],
+  rawContent: string,
+): { content: string; startingLineNumber?: number } {
+  if (!filter || Object.keys(filter).length === 0) {
+    return { content: rawContent, startingLineNumber: undefined };
+  }
+  const lines = rawContent.split('\n');
+  let startingLineNumber: number | undefined;
+  if (filter.lines) {
+    const filtered = filter.lines.map((f) => {
+      if (typeof f === 'number') {
+        const ind = index(f, lines.length);
+        if (!ind) {
+          fileWarn(vfile, 'Invalid line number "0", indexing starts at 1');
+          return [];
+        }
+        if (!startingLineNumber) startingLineNumber = ind[0] + 1;
+        return lines.slice(...ind);
+      }
+      const ind0 = index(f[0], lines.length);
+      const ind1 = index(f[1] ?? lines.length, lines.length);
+      if (!ind0 || !ind1) {
+        fileWarn(vfile, 'Invalid line number "0", indexing starts at 1');
+        return [];
+      }
+      if (!startingLineNumber) startingLineNumber = ind0[0] + 1;
+      const slice = lines.slice(ind0[0], ind1[1]);
+      if (slice.length === 0) {
+        fileWarn(vfile, `Unexpected lines, from "${f[0]}" to "${f[1] ?? ''}"`);
+      }
+      return slice;
+    });
+    return { content: filtered.flat().join('\n'), startingLineNumber };
+  }
+  let startLine =
+    filter.startAt || filter.startAfter
+      ? lines.findIndex(
+          (line) =>
+            (filter.startAt && line.includes(filter.startAt)) ||
+            (filter.startAfter && line.includes(filter.startAfter)),
+        )
+      : 0;
+  if (startLine === -1) {
+    fileWarn(
+      vfile,
+      `Could not find starting line including "${filter.startAt || filter.startAfter}"`,
+    );
+    startLine = 0;
+  }
+  if (filter.startAfter) startLine += 1;
+  let endLine =
+    filter.endAt || filter.endBefore
+      ? lines
+          .slice(startLine + 1)
+          .findIndex(
+            (line) =>
+              (filter.endAt && line.includes(filter.endAt)) ||
+              (filter.endBefore && line.includes(filter.endBefore)),
+          )
+      : lines.length;
+  if (endLine === -1) {
+    fileWarn(vfile, `Could not find ending line including "${filter.endAt || filter.endBefore}"`);
+    endLine = lines.length;
+  } else if (filter.endAt || filter.endBefore) {
+    endLine += startLine;
+    if (filter.endAt) endLine += 1;
+  }
+  startingLineNumber = startLine + 1;
+  return { content: lines.slice(startLine, endLine + 1).join('\n'), startingLineNumber };
+}

packages/myst-directives/package.json

@@ -23,6 +23,8 @@
     "clean": "rimraf dist",
     "lint": "eslint \"src/**/!(*.spec).ts\" -c ./.eslintrc.cjs",
     "lint:format": "npx prettier --check \"src/**/*.ts\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
     "build:esm": "tsc --project ./tsconfig.json --module es2015 --outDir dist --declaration",
     "build": "npm-run-all -l clean -p build:esm"
   },

packages/myst-directives/src/code.spec.ts

@@ -0,0 +1,54 @@
+import { describe, expect, test } from 'vitest';
+import { getCodeBlockOptions } from './code.js';
+import { VFile } from 'vfile';
+
+describe('Code block options', () => {
+  test('default options', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({}, vfile);
+    expect(opts).toEqual({});
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 1 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines: 2', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 2 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true, startingLineNumber: 2 });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('number-lines clashes with lineno-start', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'number-lines': 1, 'lineno-start': 2 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true, startingLineNumber: 2 });
+    // Show warning!
+    expect(vfile.messages.length).toEqual(1);
+  });
+  test('lineno-start activates showLineNumbers', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'lineno-start': 1 }, vfile);
+    expect(opts).toEqual({ showLineNumbers: true });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  test('emphasize-lines', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'emphasize-lines': '3,5' }, vfile);
+    expect(opts).toEqual({ emphasizeLines: [3, 5] });
+    expect(vfile.messages.length).toEqual(0);
+  });
+  // See https://github.com/executablebooks/jupyterlab-myst/issues/174
+  test(':lineno-start: 10, :emphasize-lines: 12,13', () => {
+    const vfile = new VFile();
+    const opts = getCodeBlockOptions({ 'lineno-start': 10, 'emphasize-lines': '12,13' }, vfile);
+    expect(opts).toEqual({
+      showLineNumbers: true,
+      emphasizeLines: [12, 13],
+      startingLineNumber: 10,
+    });
+    expect(vfile.messages.length).toEqual(0);
+  });
+});