feat(wiki): add parser (#54)

.github/workflows/unit-test.yml

@@ -11,6 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - uses: pnpm/[email protected]
         with:
           version: '6'

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "packages/utils/wiki/__test__/wiki-syntax-spec"]
+	path = packages/utils/wiki/__test__/wiki-syntax-spec
+	url = https://github.com/bangumi/wiki-syntax-spec

packages/utils/index.ts

@@ -0,0 +1 @@
+export { default as parseWiki } from './wiki/parser'

packages/utils/package.json

@@ -1,5 +1,10 @@
 {
   "name": "@bangumi/utils",
   "version": "0.0.0",
-  "private": true
+  "private": true,
+  "main": "index.ts",
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.5",
+    "js-yaml": "^4.1.0"
+  }
 }

packages/utils/wiki/__test__/parser.test.ts

@@ -0,0 +1,45 @@
+import path from 'path'
+import fs from 'fs'
+import parse from '../parser'
+import yaml from 'js-yaml'
+
+const testsDir = path.resolve(__dirname, './wiki-syntax-spec/tests/')
+const validTestDir = path.resolve(testsDir, 'valid')
+const invalidTestDir = path.resolve(testsDir, 'invalid')
+
+const validTestFiles = fs.readdirSync(validTestDir)
+const inValidTestFiles = fs.readdirSync(invalidTestDir)
+
+describe('Wiki syntax parser expected to be valid', () => {
+  validTestFiles.forEach(file => {
+    const prefix = file.split('.')[0]
+    const suffix = file.split('.')[1]
+    if (suffix !== 'wiki') {
+      return
+    }
+    it(`${prefix} should be valid`, () => {
+      const testFilePath = path.resolve(validTestDir, file)
+      const expectedFilePath = path.resolve(validTestDir, `${prefix}.yaml`)
+
+      const testContent = fs.readFileSync(testFilePath, 'utf8')
+      const expectedContent = fs.readFileSync(expectedFilePath, 'utf8')
+
+      const result = parse(testContent)
+      const expected = yaml.load(expectedContent)
+
+      expect(result).toEqual(expected)
+    })
+  })
+})
+
+describe('Wiki syntax parser expected to be inValid', () => {
+  inValidTestFiles.forEach(file => {
+    const prefix = file.split('.')[0]
+    it(`${prefix} should be invalid`, () => {
+      const testFilePath = path.resolve(invalidTestDir, file)
+      const testContent = fs.readFileSync(testFilePath, 'utf8')
+
+      expect(() => parse(testContent)).toThrowError()
+    })
+  })
+})

packages/utils/wiki/error.ts

@@ -0,0 +1,17 @@
+export const GlobalPrefixError = 'missing prefix \'{{Infobox\' at the start'
+export const GlobalSuffixError = 'missing suffix \'}}\' at the end'
+export const ArrayNoCloseError = 'array should be closed by \'}\''
+export const ArrayItemWrappedError = 'array item should be wrapped by \'[]\''
+export const ExpectingNewFieldError = 'missing \'|\' to start a new field'
+export const ExpectingSignEqualError = 'missing \'=\' to separate field name and value'
+
+export class WikiSyntaxError extends Error {
+  line: string | null
+  lino: number | null
+
+  constructor (lino: number | null, line: string | null, message: string) {
+    super(message)
+    this.line = line
+    this.lino = lino
+  }
+}

packages/utils/wiki/parser.ts

@@ -0,0 +1,108 @@
+import { Wiki, WikiArrayItem, WikiItem, WikiItemType } from './types'
+import {
+  ArrayItemWrappedError,
+  ArrayNoCloseError,
+  ExpectingNewFieldError,
+  ExpectingSignEqualError,
+  GlobalPrefixError,
+  GlobalSuffixError,
+  WikiSyntaxError
+} from './error'
+
+/* should start with `{{Infobox` and end with `}}` */
+const prefix = '{{Infobox'
+const suffix = '}}'
+
+export default function parse (s: string): Wiki {
+  const wiki: Wiki = { type: '', data: [] }
+
+  const strTrim = s.trim().replace(/\r\n/g, '\n')
+
+  if (strTrim === '') {
+    return wiki
+  }
+
+  if (!strTrim.startsWith(prefix)) {
+    throw new WikiSyntaxError(null, null, GlobalPrefixError)
+  }
+  if (!strTrim.endsWith(suffix)) {
+    throw new WikiSyntaxError(null, null, GlobalSuffixError)
+  }
+
+  const arr = strTrim.split('\n')
+  wiki.type = parseType(arr[0])
+
+  /* split content between {{Infobox xxx and }} */
+  const fields = arr.slice(1, -1)
+
+  let inArray = false
+  for (let i = 0; i < fields.length; ++i) {
+    const line = fields[i].trim()
+    const lino = i + 2
+
+    if (line === '') {
+      continue
+    }
+    /* new field */
+    if (line[0] === '|') {
+      if (inArray) {
+        throw new WikiSyntaxError(lino, line, ArrayNoCloseError)
+      }
+      const meta = parseNewField(lino, line)
+      inArray = meta[2] === 'array'
+      const field = new WikiItem(...meta)
+      wiki.data.push(field)
+      /* is Array item */
+    } else if (inArray) {
+      if (line[0] === '}') {
+        inArray = false
+        continue
+      }
+      if (i === fields.length - 1) {
+        throw new WikiSyntaxError(lino, line, ArrayNoCloseError)
+      }
+      wiki.data[wiki.data.length - 1]!.values!.push(
+        new WikiArrayItem(
+          ...parseArrayItem(lino, line)
+        )
+      )
+    } else {
+      throw new WikiSyntaxError(lino, line, ExpectingNewFieldError)
+    }
+  }
+  return wiki
+}
+
+const parseType = (line: string): string => {
+  return line.slice(prefix.length, !line.includes('}}') ? undefined : line.indexOf('}}')).trim()
+}
+
+const parseNewField = (lino: number, line: string): [string, string, WikiItemType] => {
+  const str = line.slice(1)
+  const index = str.indexOf('=')
+
+  if (index === -1) {
+    throw new WikiSyntaxError(lino, line, ExpectingSignEqualError)
+  }
+
+  const key = str.slice(0, index).trim()
+  const value = str.slice(index + 1).trim()
+  switch (value) {
+    case '{':
+      return [key, '', 'array']
+    default :
+      return [key, value, 'object']
+  }
+}
+
+const parseArrayItem = (lino: number, line: string): [string, string] => {
+  if (line[0] !== '[' || line[line.length - 1] !== ']') {
+    throw new WikiSyntaxError(lino, line, ArrayItemWrappedError)
+  }
+  const content = line.slice(1, line.length - 1)
+  const index = content.indexOf('|')
+  if (index === -1) {
+    return ['', content.trim()]
+  }
+  return [content.slice(0, index).trim(), content.slice(index + 1).trim()]
+}

packages/utils/wiki/types.ts

@@ -0,0 +1,36 @@
+export interface Wiki {
+  type: string
+  data: WikiItem[]
+}
+
+export type WikiItemType = 'array' | 'object'
+
+export class WikiArrayItem {
+  k?: string
+  v?: string
+
+  constructor (k: string, v: string) {
+    k && (this.k = k)
+    this.v = v
+  }
+}
+
+export class WikiItem {
+  key?: string
+  value?: string
+  array?: boolean
+  values?: WikiArrayItem[]
+
+  constructor (key: string, value: string, type: WikiItemType) {
+    this.key = key
+    switch (type) {
+      case 'array':
+        this.array = true
+        this.values = []
+        break
+      case 'object':
+        this.value = value
+        break
+    }
+  }
+}