MWPW-147219 Generate content for validator

blog-test/locales.json

@@ -0,0 +1,86 @@
+[
+  "",
+  "ae_ar",
+  "ae_en",
+  "africa",
+  "ar",
+  "at",
+  "au",
+  "be_en",
+  "be_fr",
+  "be_nl",
+  "bg",
+  "br",
+  "ca_fr",
+  "ca",
+  "ch_de",
+  "ch_fr",
+  "ch_it",
+  "cl",
+  "cn",
+  "co",
+  "cr",
+  "cy_en",
+  "cz",
+  "de",
+  "dk",
+  "ec",
+  "ee",
+  "eg_ar",
+  "eg_en",
+  "el",
+  "es",
+  "fi",
+  "fr",
+  "gr_el",
+  "gr_en",
+  "gt",
+  "hk_en",
+  "hk_zh",
+  "hu",
+  "id_en",
+  "id_id",
+  "ie",
+  "il_en",
+  "il_he",
+  "in_hi",
+  "in",
+  "it",
+  "jp",
+  "kr",
+  "kw_ar",
+  "kw_en",
+  "la",
+  "langstore",
+  "lt",
+  "lu_de",
+  "lu_en",
+  "lu_fr",
+  "lv",
+  "mena_ar",
+  "mena_en",
+  "mt",
+  "mx",
+  "my_en",
+  "my_ms",
+  "ng",
+  "nl",
+  "no",
+  "nz",
+  "pe",
+  "ph_en",
+  "ph_fil",
+  "se",
+  "sg",
+  "si",
+  "sk",
+  "th_en",
+  "th_th",
+  "tr",
+  "tw",
+  "ua",
+  "uk",
+  "vn_en",
+  "vn_vi",
+  "za"
+]

blog-test/migration.js

@@ -0,0 +1,95 @@
+import fs from 'fs';
+import { u } from 'unist-builder';
+import { BulkUpdate, ExcelReporter, loadListData, saveDocument } from '../bulk-update/index.js';
+import { selectBlock } from '../bulk-update/migration-tools/select.js';
+
+const { pathname } = new URL('.', import.meta.url);
+const dateString = ExcelReporter.getDateString();
+
+const config = {
+  list: [
+    'https://main--bacom-blog--adobecom.hlx.live/de/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/fr/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/au/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/uk/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/jp/blog/query-index.json',
+    'https://main--bacom-blog--adobecom.hlx.live/kr/blog/query-index.json',
+  ],
+  siteUrl: 'https://main--bacom-blog--adobecom.hlx.live',
+  stagePath: '/drafts/staged-content',
+  locales: JSON.parse(fs.readFileSync(`${pathname}locales.json`, 'utf8')),
+  prodSiteUrl: 'https://business.adobe.com',
+  reporter: new ExcelReporter(`${pathname}reports/blog-${dateString}.xlsx`, false),
+  outputDir: `${pathname}output`,
+  mdDir: `${pathname}md`,
+  mdCacheMs: 1 * 24 * 60 * 60 * 1000, // 1 day(s)
+  fetchWaitMs: 20,
+};
+
+/**
+ * Creates a block with the given name and fields.
+ *
+ * @param {string} name - The name of the block.
+ * @param {Object} fields - The fields of the block.
+ * @returns {Array} - The created block.
+ */
+export function createBlock(name, fields) {
+  const block = u('gridTable', [
+    u('gtBody', [
+      u('gtRow', [
+        u('gtCell', { colSpan: 2 }, [u('paragraph', [u('text', name)])]),
+      ]),
+      ...fields.map((values) => u('gtRow', values.map((value) => u('gtCell', [u('paragraph', [value ?? u('text', '')])])))),
+    ]),
+  ]);
+
+  return block;
+}
+
+/**
+ * Create or replace a hidden block, hide-block, with the entry and migration date
+ *
+ * @param {Object} document - The document to be migrated.
+ */
+export async function migrate(document) {
+  const { mdast, entry } = document;
+  if (!mdast || !mdast.children) return;
+
+  const fields = [
+    [u('text', 'Entry'), u('text', entry)],
+    [u('text', 'Date'), u('text', new Date().toISOString().split('T')[0])],
+  ];
+  const hiddenBlock = createBlock('Hide Block', fields); // This block is display none in Milo projects
+  const existingBlock = selectBlock(mdast, 'Hide Block');
+
+  if (existingBlock) {
+    existingBlock.children = hiddenBlock.children;
+    config.reporter.log('migration', 'update', 'Updated hide block');
+  } else {
+    mdast.children.push(hiddenBlock);
+    config.reporter.log('migration', 'create', 'Created hide block');
+  }
+
+  await saveDocument(document, config);
+}
+
+/**
+ * Initializes the migration process.
+ *
+ * @param {Array} list - The list of data to be migrated.
+ * @returns {Promise} - A promise that resolves to the configuration object.
+ */
+export async function init(list) {
+  config.list = await loadListData(list || config.list);
+
+  await BulkUpdate(config, migrate);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const args = process.argv.slice(2);
+  const [list] = args;
+
+  await init(list);
+  process.exit(0);
+}

blog-test/readme.md

@@ -0,0 +1,17 @@
+# Test Blog Migration
+
+Simple test migration to add a hidden block, "hide block", to each page.
+
+This migration add a hidden block so that there is modification to the page without changing the content.
+This is used to test the migration process and to validate that the migration process is working as expected.
+
+This can be used with the link validation to make sure links are not being shuffled.
+Other use cases may include adding migration information for authors.
+
+## Usage
+
+Run the migration script directly:
+
+```bash
+node blog-test/migration.js
+```

bulk-update/bulk-update.js

@@ -86,6 +86,22 @@ export async function loadListData(source, fetchFunction = fetch, fetchWaitMs =
   }
 }
 
+/**
+ * Generates the staged-content URL by localizing the stage path based on the entry path.
+ *
+ * @param {string} siteUrl - The base URL of the site.
+ * @param {string} entry - The entry path.
+ * @param {string} stagePath - The path to the stage.
+ * @param {string[]} locales - An array of supported locales.
+ * @returns {string} The staged URL.
+ */
+export function localizeStageUrl(siteUrl, entry, stagePath = '', locales = []) {
+  const currentLocale = locales.find((locale) => locale && entry.startsWith(`/${locale}/`));
+  const localizedPath = currentLocale ? entry.replace(`/${currentLocale}/`, `/${currentLocale}${stagePath}/`) : `${stagePath}${entry}`;
+
+  return `${siteUrl}${localizedPath}`;
+}
+
 /**
  * Executes a bulk update operation using the provided migration function
  * Loads data from various sources and executes bulk update operations from the migration function.
@@ -97,7 +113,19 @@ export async function loadListData(source, fetchFunction = fetch, fetchWaitMs =
  */
 export default async function main(config, migrate, reporter = null) {
   config.reporter = reporter || config.reporter;
-  const { length } = config.list;
+  const { list, outputDir, siteUrl, stagePath, locales } = config;
+  const { length } = list;
+
+  if (outputDir) {
+    fs.mkdirSync(outputDir, { recursive: true });
+    fs.writeFileSync(`${outputDir}/list.json`, JSON.stringify(config.list, null, 2));
+
+    if (stagePath) {
+      const stageOutput = list.map((entry) => [`${siteUrl}${entry}`, localizeStageUrl(siteUrl, entry, stagePath, locales)]);
+      fs.writeFileSync(`${outputDir}/staged.json`, JSON.stringify(stageOutput, null, 2));
+      fs.writeFileSync(`${outputDir}/staged.txt`, stageOutput.map((entry) => entry[1]).join('\n'));
+    }
+  }
 
   try {
     for (const [i, entry] of config.list.entries()) {

bulk-update/document-manager/document-manager.js

@@ -1,10 +1,15 @@
 /* eslint-disable max-len */
 import fs from 'fs';
+import path from 'path';
 import { fetch, timeoutSignal, AbortError } from '@adobe/fetch';
 import { mdast2docx } from '@adobe/helix-md2docx';
+import { mdast2md } from '@adobe/helix-docx2md';
 import parseMarkdown from '@adobe/helix-html-pipeline/src/steps/parse-markdown.js';
 import validateMdast from '../validation/mdast.js';
 
+const MD_SOURCE = 'source';
+const MD_UPDATED = 'updated';
+
 const delay = (milliseconds) => new Promise((resolve) => { setTimeout(resolve, milliseconds); });
 const { pathname } = new URL('.', import.meta.url);
 
@@ -17,9 +22,11 @@ const { pathname } = new URL('.', import.meta.url);
  * @returns {string} - The markdown path
  */
 export function entryToPath(entry) {
-  let path = entry.split(/\?|#/)[0].replace(/\/$/, '/index');
-  path = path.replace(/\.html$/, ''); // Remove .html extension
-  return path;
+  let filepath = entry.split(/\?|#/)[0].replace(/\/$/, '/index');
+  filepath = filepath.replace(/\.html$/, ''); // Remove .html extension
+  if (!filepath.startsWith('/')) filepath = `/${filepath}`;
+
+  return filepath;
 }
 
 /**
@@ -141,9 +148,9 @@ export async function loadDocument(entry, config, fetchFunction = fetch) {
   const document = { entry, path: entryToPath(entry) };
 
   document.url = new URL(document.path, siteUrl).href;
-  document.markdownFile = `${mdDir}${document.path}.md`;
 
   if (mdDir) {
+    document.markdownFile = path.join(mdDir, MD_SOURCE, `${document.path}.md`);
     document.markdown = loadMarkdownFromFile(document.markdownFile, mdCacheMs);
 
     if (document.markdown) {
@@ -170,17 +177,35 @@ export async function loadDocument(entry, config, fetchFunction = fetch) {
 }
 
 /**
- * Save a mdast as a docx file to the file system.
+ * Save a mdast as a md file to the specified file.
  *
  * @param {object} mdast
  * @param {string} outputFile
+ * @returns {Promise<void>}
+ */
+async function saveMd(mdast, output) {
+  const outputFolder = path.dirname(output);
+  fs.mkdirSync(outputFolder, { recursive: true });
+
+  const mdastCopy = JSON.parse(JSON.stringify(mdast));
+  const md = await mdast2md(mdastCopy, { gridtables: true });
+  fs.writeFileSync(output, md, 'utf-8');
+}
+
+/**
+ * Save a mdast as a docx file to the specified file.
+ *
+ * @param {object} mdast
+ * @param {string} outputFile
+ * @returns {Promise<void>}
  */
 async function saveDocx(mdast, output) {
-  const outputFolder = output.split('/').slice(0, -1).join('/');
+  const outputFolder = path.dirname(output);
   fs.mkdirSync(outputFolder, { recursive: true });
 
   const stylesXML = fs.readFileSync(`${pathname}styles.xml`, 'utf8');
-  const buffer = await mdast2docx(mdast, { stylesXML });
+  const mdastCopy = JSON.parse(JSON.stringify(mdast));
+  const buffer = await mdast2docx(mdastCopy, { stylesXML });
   fs.writeFileSync(output, buffer);
 }
 
@@ -197,23 +222,31 @@ async function saveDocx(mdast, output) {
  */
 export async function saveDocument(document, config) {
   const { mdast, entry } = document;
-  const { reporter, outputDir } = config;
+  const { reporter, mdDir, outputDir } = config;
   if (!outputDir) {
     reporter.log('save', 'error', 'No output directory specified. Skipping save.');
     return;
   }
   const documentPath = entryToPath(entry);
-  const output = `${outputDir}${documentPath}.docx`;
+  const outputDocx = path.join(outputDir, `${documentPath}.docx`);
+  const outputMd = path.join(mdDir, MD_UPDATED, `${documentPath}.md`);
   const issues = validateMdast(mdast);
 
   issues.forEach((issue) => {
     reporter.log('validation', 'error', issue, { entry });
   });
 
   try {
-    await saveDocx(mdast, output);
-    reporter.log('save', 'success', 'Saved docx', { entry });
+    await saveMd(mdast, outputMd);
+    reporter.log('save', 'md success', 'Saved markdown', { entry });
+  } catch (e) {
+    reporter.log('save', 'md error', e.message, { entry });
+  }
+
+  try {
+    await saveDocx(mdast, outputDocx);
+    reporter.log('save', 'docx success', 'Saved docx', { entry });
   } catch (e) {
-    reporter.log('save', 'error', e.message, { entry });
+    reporter.log('save', 'docx error', e.message, { entry });
   }
 }

test/bulk-update/bulk-update.test.js

@@ -1,6 +1,6 @@
 import { expect } from '@esm-bundle/chai';
 import { stub } from 'sinon';
-import BulkUpdate, { loadListData } from '../../bulk-update/bulk-update.js';
+import BulkUpdate, { loadListData, localizeStageUrl } from '../../bulk-update/bulk-update.js';
 import BaseReporter from '../../bulk-update/reporter/reporter.js';
 
 const { pathname } = new URL('.', import.meta.url);
@@ -119,4 +119,28 @@ describe('BulkUpdater', () => {
       expect(totals).to.deep.equal({ load: { success: 2 } });
     });
   });
+  describe('localizeStageUrl', () => {
+    it('generates the correct staged URL without locales', () => {
+      const siteUrl = 'https://example.com';
+      const entry = '/test/path';
+      const stagePath = '/stage';
+
+      const expectedUrl = 'https://example.com/stage/test/path';
+      const result = localizeStageUrl(siteUrl, entry, stagePath);
+
+      expect(result).to.equal(expectedUrl);
+    });
+
+    it('generates the correct staged URL', () => {
+      const siteUrl = 'https://example.com';
+      const entry = '/fr/test/path';
+      const stagePath = '/staged-content';
+      const locales = ['fr', 'de'];
+
+      const expectedUrl = 'https://example.com/fr/staged-content/test/path';
+      const result = localizeStageUrl(siteUrl, entry, stagePath, locales);
+
+      expect(result).to.equal(expectedUrl);
+    });
+  });
 });