feat: support waterfall includes

src/css.js

@@ -1,6 +1,6 @@
 import path from 'path';
 
-import { resolve } from './resolver.js';
+import { handleTemplateFile } from './transforms/template-data.js';
 
 // TODO: add a regex for layer syntax
 const INCLUDE_REGEX = /@import [\"\']([\w:\/\\]+\.css)[\"\'];/g;
@@ -11,29 +11,15 @@ export default (config) => {
   config.addExtension('css', {
     outputFileExtension: 'css',
     compile: async function (inputContent, inputPath) {
-
       let parsed = path.parse(inputPath);
-      if (parsed.name.startsWith('_')) {
-        // Omit files prefixed with an underscore.
-        return;
-      }
-
-      return async () => {
+
+      return async (data) => {
         const includes = new Map();
         const matches = inputContent.matchAll(INCLUDE_REGEX);
         for (const [, file] of matches) {
-
-          const fullPath = path.resolve(config.dir.input, parsed.dir, file);
-          try {
-            const content = await (config.resolve || resolve)(fullPath);
-            includes.set(file, content);
-          } catch (err) {
-            console.error('error processing file:', fullPath, err);
-            // silently fail if there is no include
-            includes.set(file, `@import "${file}";`);
-          }
+          const tpl = await handleTemplateFile(config, data, path.join(parsed.dir, file));
+          includes.set(file, tpl ? tpl.content : `@import url("${file}");`);
         }
-
         return inputContent.replace(INCLUDE_REGEX, (_, file) => includes.get(file)) 
       };
     },

src/html.js

@@ -1,6 +1,7 @@
 import path from 'path';
-import { resolve } from './resolver.js';
-const INCLUDE_REGEX = /<html-include[\s\r\n]*src="([\w\.]+)"[\s\r\n]*\/?>/g;
+import { handleTemplateFile } from './transforms/template-data.js';
+
+const INCLUDE_REGEX = /<html-include[\s\r\n]*src="([\w\-\.]+)"[\s\r\n]*\/?>/g;
 
 export default (config) => {
   config.addTemplateFormats('html');
@@ -10,29 +11,21 @@ export default (config) => {
     compile: async function (inputContent, inputPath) {
 
       let parsed = path.parse(inputPath);
-      if (parsed.name.startsWith('_')) {
-        // Omit files prefixed with an underscore.
-        return;
-      }
 
-      return async () => {
+      return async (data) => {
         const includes = new Map();
-        const matches = inputContent.matchAll(INCLUDE_REGEX);
-        for (const [, file] of matches) {
-
-          const fullPath = path.join(config.dir.input, config.dir.includes, file);
-          try {
-            const content = await (config.resolve || resolve)(fullPath);
-            includes.set(file, content);
-          } catch (err) {
-            console.error('error processing file:', fullPath, err);
-            // silently fail if there is no include
-            includes.set(file, `<html-include src="${file}"/>`);
+        let content = inputContent, matches;
+
+        while ((matches = Array.from(content.matchAll(INCLUDE_REGEX))).length > 0) {
+          for (const [, file] of matches) {
+            const include = await handleTemplateFile(config, data, path.join(config.dir.includes, file));
+            includes.set(file, include ? include.content : `<!-- html-include src="${file}" -->`);
           }
-        }   
-        return inputContent.replace(INCLUDE_REGEX, (_, file) => {
-          return includes.get(file)
-        });
+          content = content.replace(INCLUDE_REGEX, (_, file) => {
+            return includes.get(file)
+          });
+        }
+        return content;
       };
     },
   });

src/md.js

@@ -1,6 +1,8 @@
 import path from 'path';
-import { resolve } from './resolver.js';
+
+import { handleTemplateFile } from './transforms/template-data.js';
 import { markdown } from './transforms/markdown.js';
+
 const INCLUDE_REGEX = /<html-include[\s\r\n]*src="([\w\.]+)"[\s\r\n]*\/?>/g;
 
 export default (config) => {
@@ -10,27 +12,13 @@ export default (config) => {
     outputFileExtension: 'html',
     compile: async function (inputContent, inputPath) {
 
-      let parsed = path.parse(inputPath);
-      if (parsed.name.startsWith('_')) {
-        // Omit files prefixed with an underscore.
-        return;
-      }
-
-      return async () => {
+      return async (data) => {
         const includes = new Map();
         const content = markdown(inputContent);
         const matches = content.matchAll(INCLUDE_REGEX);
         for (const [, file] of matches) {
-
-          const fullPath = path.join(config.dir.input, config.dir.includes, file);
-          try {
-            const content = await (config.resolve || resolve)(fullPath);
-            includes.set(file, content);
-          } catch (err) {
-            console.error('error processing file:', fullPath, err);
-            // silently fail if there is no include
-            includes.set(file, `<html-include src="${file}"/>`);
-          }
+          const include = await handleTemplateFile(config, data, path.join(config.dir.includes, file));
+          includes.set(file, include ? include.content : `<!-- html-include src="${file}" -->`);
         }   
         return content.replace(INCLUDE_REGEX, (_, file) => {
           return includes.get(file)

src/resolver.js

@@ -1,16 +1,29 @@
-import { readFile } from 'node:fs/promises'
+import { readFile, stat } from 'node:fs/promises'
 import path from 'node:path';
 
-export async function resolve(resource) {
-  if (/^\w+:\/\//.test(resource)) {
-    // seems to be an URI, fetch it
+/**
+ * Read a file from the input dir or from the internet.
+ * @param {string[]} paths
+ * @returns 
+ */
+export async function resolve(...paths) {
+  const last = paths.slice(-1)[0];
+  if (/^\w+:\/\//.test(last)) {
+    // seems to be an URL, fetch it
+    const resource = last;
     const response = await fetch(resource);
     const contentType = response.headers.get('Content-Type');
     if (!contentType || !contentType.startsWith('text')) {
       return await response.buffer();
     }
     return await response.text();
   }
+
   // otherwise, readFile it.
-  return await readFile(path.resolve(resource), 'utf8');
+  const resource = path.normalize(path.join(...paths));
+  const absResource = path.resolve(resource);
+  if ((await stat(absResource)).isDirectory()) {
+    return null;
+  }
+  return await readFile(absResource, 'utf8');
 }

src/sissi.js

@@ -6,8 +6,7 @@ import { SissiConfig } from './sissi-config.js';
 import { serve } from './httpd.js';
 import EventEmitter from 'node:stream';
 import { readDataDir } from './data.js';
-import { template } from './transforms/template-data.js'
-import { frontmatter } from './transforms/frontmatter.js';
+import { handleTemplateFile } from './transforms/template-data.js';
 
 export class Sissi {
 
@@ -30,9 +29,11 @@ export class Sissi {
         if (filter instanceof RegExp) return filter.test(file);
       }
     );
+    const writtenFiles = []
     for (const file of files) {
-      await this.processFile(file, eventEmitter);
+      writtenFiles.push(await this.processFile(file, eventEmitter));
     }
+    return writtenFiles.filter(Boolean);
   }
 
   /**
@@ -96,61 +97,27 @@ export class Sissi {
     if (! this.data) {
       this.data = await readDataDir(this.config);
     }
-    const absInputFileName = path.resolve(this.config.dir.input, inputFileName);
-    if (inputFileName.startsWith('_') || inputFileName.includes(path.sep + '_')) {
+    if (inputFileName.startsWith('_') || inputFileName.includes(path.sep + '_') || path.parse(inputFileName).name.startsWith('_')) {
       return;
     }
-    const stats = await stat(absInputFileName);
-    if (stats.isDirectory()) {
-      return;
-    }
-    let content = await readFile(absInputFileName, 'utf8');
-    const parsed = path.parse(inputFileName);
-    const extension = parsed.ext?.slice(1);
-
-    let ext = null;
-    if (this.config.extensions.has(extension)) {
-      ext = this.config.extensions.get(extension);
-      const { data: matterData, body } = frontmatter(content);
-      content = body;
-      const fileData = Object.assign({}, structuredClone(this.data), matterData);
-      const processor = await ext.compile(content, inputFileName);
-      content = template(await processor(fileData))(fileData);
-
-      if (fileData.layout) {
-        fileData.content = content;
-        const relLayoutDir = path.normalize(
-          path.join(this.config.dir.input, this.config.dir.layouts || '_layouts')
-        );
-        const absLayoutFilePath = path.resolve(relLayoutDir, fileData.layout);
-        const layoutExtKey = path.parse(absLayoutFilePath).ext?.slice(1);
-        let layoutContent = await readFile(absLayoutFilePath, 'utf8');
-
-        const layoutExt = layoutExtKey ? this.config.extensions.get(layoutExtKey) : null;
-        if (layoutExt) {
-          const processor = await layoutExt.compile(layoutContent, inputFileName);
-          layoutContent = await processor(fileData);
-        } 
-
-        content = template(layoutContent)(fileData);
-      }
-
 
+    const tpl = await handleTemplateFile(this.config, this.data, inputFileName);
+    if (! tpl) {
+      return null;
     }
-
-    let outputFileName =this.config.naming(this.config.dir.output, inputFileName, ext?.outputFileExtension);
-    console.log(`[write]\t${outputFileName}`);
+
+    console.log(`[write]\t${tpl.filename}`);
     if (eventEmitter) {
       eventEmitter.emit('watch-event', {
         eventType: 'change',
         filename: inputFileName
       });
     }
-    if (this.dryMode) {
-      return;
+    if (! this.dryMode) {
+      await mkdir(path.parse(tpl.filename).dir, {recursive: true});
+      await writeFile(tpl.filename, tpl.content, {});
     }
-    await mkdir(path.parse(outputFileName).dir, {recursive: true});
-    await writeFile(outputFileName, content, {});
+    return tpl.filename;
   }
 
   /**

src/transforms/bundle.js

@@ -0,0 +1,37 @@
+const SYNTAXES = {
+  html: /<html-include[\s\r\n]*src="([\w\-\.]+)"[\s\r\n]*\/?>/g,
+  css: /@import [\"\']([\w:\/\\]+\.css)[\"\'](?: layer\((\w+)\))?;/g,
+};
+
+/**
+ * Bundle assets into one file.
+ * 
+ * @param {string} inputContent
+ * @param {(resource: string) => Promise<string>} resolve 
+ * @param {'html'|'css'} syntax 
+ * @returns {Promise<string>} return the bundled resource
+ */
+async function bundle(inputContent, resolve, syntax, processor) {
+  const includes = new Map();
+  let content = inputContent, matches;
+  const includePattern = SYNTAXES[syntax];
+
+  while ((matches = Array.from(content.matchAll(includePattern))).length > 0) {
+    for (const [, file] of matches) {
+
+      const fullPath = path.join(config.dir.input, config.dir.includes, file);
+      try {
+        const content = await resolve(fullPath);
+
+        includes.set(file, await (await processor(content, file)).compile(data));
+      } catch (err) {
+        console.error('error processing file:', fullPath, err);
+        // silently fail if there is no include
+        includes.set(file, `<!-- html-include src="${file}" -->`);
+      }
+    }
+    content = content.replace(includePattern, (_, file) => {
+      return includes.get(file);
+    });
+  }
+}

src/transforms/template-data.js

@@ -1,3 +1,9 @@
+import path from 'node:path';
+
+import { frontmatter } from './frontmatter.js';
+import { resolve } from '../resolver.js';
+import { SissiConfig } from "../sissi-config.js";
+
 const TEMPLATE_REGEX = /\{\{\s*([\w\.\[\]]+)\s*\}\}/g;
 const JSON_PATH_REGEX = /^\w+((?:\.\w+)|(?:\[\d+\]))*$/
 const JSON_PATH_TOKEN = /(^\w+)|(\.\w+)|(\[\d+\])/g
@@ -41,3 +47,50 @@ export function template(str) {
     });
   }
 }
+
+/**
+ * Complete Template processing function
+ * @param {SissiConfig} config 
+ * @param {any} data 
+ * @param {string} inputFile 
+ * @returns {Promise<{content: Buffer|string, filename}>} the content file name and the output file name
+ */
+export async function handleTemplateFile(config, data, inputFile) {
+  const content = await (config.resolve || resolve)(config.dir.input, inputFile);
+  if (content === null) {
+    return null;
+  } 
+
+  const parsed = path.parse(inputFile);
+  const ext = parsed.ext?.slice(1);
+  if (! config.extensions.has(ext)) {
+    return {
+      content,
+      filename: config.naming(config.dir.output, inputFile)
+    };
+  }
+
+  const plugin = config.extensions.get(ext);
+
+  const { data: matterData, body } = frontmatter(content);
+  const fileData = Object.assign({}, structuredClone(data), matterData);
+
+  const outputFile = config.naming(config.dir.output, inputFile, plugin?.outputFileExtension);
+  Object.assign(fileData, {
+    inputFile,
+    outputFile,
+  })
+
+  const processor = await plugin.compile(body, inputFile);
+
+  let fileContent = template(await processor(fileData))(fileData);
+
+  if (fileData.layout) {
+    const layoutFilePath = path.normalize(path.join(config.dir.layouts, fileData.layout));
+    const l = await handleTemplateFile(config, 
+      {...fileData, content: fileContent, layout: null}, layoutFilePath);
+    fileContent = l.content;
+  }
+
+  return {content: fileContent, filename: outputFile};
+}

tests/css.test.js

@@ -1,5 +1,6 @@
 import { describe, it, before } from 'node:test';
 import assert from 'node:assert/strict';
+import path from 'node:path';
 
 import { SissiConfig } from '../src/sissi-config.js';
 import css from '../src/css.js'
@@ -18,7 +19,8 @@ describe('css plugin', () => {
   virtualFileSystem.set('A.css', '.a {color: red; }');
   virtualFileSystem.set('B.css', '.b {color: green }');
 
-  function dummyResolver(resource) {
+  function dummyResolver(...paths) {
+    const resource = path.normalize(path.join(...paths));
     const match = resource.match(/\\?\/?(\w+.css)$/);
     if (!match || !virtualFileSystem.has(match[1]) ) {
       throw new Error('Virtual File not found')
@@ -51,4 +53,4 @@ describe('css plugin', () => {
     assert(result === expectedFile);
   });
 
-});
+});

tests/data.test.js

@@ -9,7 +9,7 @@ describe('readDataDir', () => {
 
   before(() => {
     config = new SissiConfig({dir: {
-      input: 'tests/fixture',
+      input: 'tests/fixtures/data',
       data: '_data',
       output: 'dist'
     }});