Feature/parallel fetch data

.editorconfig

@@ -0,0 +1,11 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+
+[*.{js,jsx,ts,tsx,json,mjs}]
+charset = utf-8
+indent_style = space
+trim_trailing_whitespace = true
+indent_size = 2

.eslintrc

@@ -4,11 +4,12 @@
     "node": true,
     "es6": true
   },
-  "ignorePatterns": ["packages/_archive/*", "examples/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"],
+  "ignorePatterns": ["packages/_archive/*", "**/.contentlayer/*", "**/dist/*", "**/.nyc_output/*"],
   "parser": "@typescript-eslint/parser",
-  "plugins": ["@typescript-eslint", "simple-import-sort", "import"],
+  "plugins": ["prettier", "@typescript-eslint", "simple-import-sort", "import"],
   "extends": ["plugin:react-hooks/recommended", "plugin:@typescript-eslint/recommended", "prettier"],
   "rules": {
+    "prettier/prettier": "error",
     "simple-import-sort/imports": "error",
     "import/no-duplicates": "warn",
     // "func-style": ["warn", "expression"],

examples/gatsbydocs/.gitignore

@@ -0,0 +1,4 @@
+content
+.next
+.contentlayer
+node_modules

examples/gatsbydocs/contentlayer.config.ts

@@ -0,0 +1,80 @@
+import rehypeShiki from '@stefanprobst/rehype-shiki'
+import type { FieldDef } from 'contentlayer/source-files'
+import { defineDocumentType, defineNestedType, makeSource } from 'contentlayer/source-files'
+import { createRequire } from 'module'
+import * as path from 'path'
+import * as shiki from 'shiki'
+
+const Example = defineNestedType(() => ({
+  name: 'Example',
+  fields: {
+    label: { type: 'string' },
+    href: { type: 'string' },
+  },
+}))
+
+const fields: Record<string, FieldDef> = {
+  title: {
+    type: 'string',
+    required: true,
+  },
+  jsdoc: {
+    type: 'list',
+    of: { type: 'string' },
+  },
+  tableOfContentsDepth: { type: 'number' },
+  showTopLevelSignatures: { type: 'boolean' },
+  date: { type: 'string' },
+  version: { type: 'string' },
+  canonicalLink: { type: 'string' },
+  apiCalls: { type: 'string' },
+  contentsHeading: { type: 'string' },
+  description: { type: 'string' },
+  examples: {
+    type: 'list',
+    of: Example,
+  },
+}
+
+const Reference = defineDocumentType(() => ({
+  name: 'Reference',
+  filePathPattern: 'docs/reference/**/*.md',
+  fields,
+}))
+
+const HowTo = defineDocumentType(() => ({
+  name: 'HowTo',
+  filePathPattern: 'docs/how-to/**/*.md',
+  fields,
+}))
+
+const Conceptual = defineDocumentType(() => ({
+  name: 'Conceptual',
+  filePathPattern: 'docs/conceptual/**/*.md',
+  fields,
+}))
+
+const Tutorial = defineDocumentType(() => ({
+  name: 'Tutorial',
+  filePathPattern: 'tutorial/**/*.md',
+  fields,
+}))
+
+export default makeSource(async () => {
+  // const require = createRequire(import.meta.url)
+  // const shikiPkgPath = (dir: string) => path.join(require.resolve('shiki'), '..', '..', dir, path.sep)
+  // const highlighter = await shiki.getHighlighter({
+  //   paths: { languages: shikiPkgPath('languages'), themes: shikiPkgPath('themes') },
+  //   theme: 'github-light',
+  // })
+
+  return {
+    contentDirPath: 'content',
+    documentTypes: [Reference, HowTo, Conceptual, Tutorial],
+    // onUnknownDocuments: 'skip-ignore',
+    // markdown: {
+    //   // '@stefanprobst/rehype-shiki', {}
+    //   rehypePlugins: [[rehypeShiki, { highlighter }]],
+    // },
+  }
+})

examples/gatsbydocs/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/basic-features/typescript for more information.

examples/gatsbydocs/next.config.js

@@ -0,0 +1,4 @@
+const { withContentlayer } = require('next-contentlayer')
+
+// module.exports = {}
+module.exports = withContentlayer({})

examples/gatsbydocs/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "examples-gatsby-docs",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "next": "12.1.6",
+    "react": "18.1.0",
+    "react-dom": "18.1.0"
+  },
+  "devDependencies": {
+    "@leafac/rehype-shiki": "^1.3.1",
+    "@stefanprobst/rehype-shiki": "^2.0.4",
+    "@types/react": "18.0.9",
+    "@types/react-dom": "^17.0.9",
+    "contentlayer": "workspace:*",
+    "eslint-config-next": "^11.0.1",
+    "next-contentlayer": "workspace:*",
+    "shiki": "^0.9.4",
+    "typescript": "4.6.4"
+  }
+}

examples/gatsbydocs/pages/[...id].tsx

@@ -0,0 +1,27 @@
+import type { DocumentTypes } from 'contentlayer/generated'
+import { allDocuments } from 'contentlayer/generated'
+import Head from 'next/head'
+import type { FC } from 'react'
+
+export const getStaticPaths = () => {
+  const paths = allDocuments.map((_) => `/${_._raw.flattenedPath}`)
+
+  return { paths, fallback: false }
+}
+
+export const getStaticProps = (context: any) => {
+  const doc = allDocuments.find((_) => _._raw.flattenedPath === context.params.id.join('/'))
+
+  return { props: { doc } }
+}
+
+const Page: FC<{ doc: DocumentTypes }> = ({ doc }) => (
+  <>
+    <Head>
+      <title>{doc.title}</title>
+    </Head>
+    <div dangerouslySetInnerHTML={{ __html: doc.body.html }} />
+  </>
+)
+
+export default Page

examples/gatsbydocs/pages/index.tsx

@@ -0,0 +1,21 @@
+import { allDocuments } from 'contentlayer/generated'
+import type { InferGetStaticPropsType } from 'next'
+import type { FC } from 'react'
+
+export const getStaticProps = () => {
+  const docs = allDocuments.map((_) => ({ path: _._raw.flattenedPath, title: _.title }))
+
+  return { props: { docs } }
+}
+
+const Page: FC<InferGetStaticPropsType<typeof getStaticProps>> = ({ docs }) => (
+  <div>
+    {docs.map((doc) => (
+      <a style={{ display: 'block' }} key={doc.path} href={`/${doc.path}`}>
+        {doc.title}
+      </a>
+    ))}
+  </div>
+)
+
+export default Page

examples/gatsbydocs/tsconfig.json

@@ -0,0 +1,41 @@
+{
+  "compilerOptions": {
+    "jsx": "preserve",
+    "allowSyntheticDefaultImports": true,
+    "rootDir": ".",
+    "allowJs": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "esModuleInterop": true,
+    "resolveJsonModule": true,
+    "incremental": false,
+    "composite": false,
+    "noEmit": true,
+    "target": "es5",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
+    "strict": false,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "isolatedModules": true,
+    "baseUrl": ".",
+    "paths": {
+      "contentlayer/generated": [
+        "./.contentlayer/generated"
+      ]
+    }
+  },
+  "include": [
+    "next-env.d.ts",
+    "**/*.tsx",
+    "**/*.ts",
+    ".contentlayer/generated"
+  ],
+  "exclude": [
+    "node_modules",
+    "gatsby"
+  ]
+}

package.json

@@ -34,6 +34,7 @@
     "eslint": "^8.15.0",
     "eslint-config-prettier": "^8.5.0",
     "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-react-hooks": "^4.5.0",
     "eslint-plugin-simple-import-sort": "^7.0.0",
     "prettier": "^2.6.2",

packages/@contentlayer/source-files/package.json

@@ -43,6 +43,7 @@
     "fast-glob": "^3.2.11",
     "gray-matter": "^4.0.3",
     "micromatch": "^4.0.5",
+    "piscina": "^3.2.0",
     "ts-pattern": "^4.0.2",
     "unified": "^10.1.2",
     "yaml": "^1.10.2"

packages/@contentlayer/source-files/src/__test__/fetchData/utils.ts

@@ -54,6 +54,10 @@ export const runTest = async ({
           previousCache: undefined,
           contentTypeMap,
         }),
+        // TODO: DocumentTypeMap is now explicitly returned from
+        // makeCacheItemFromFilePath in [1]. Verify that it is tested somewhere
+        // else, as we're throwing it away here.
+        T.map((_) => _.tuple[0]),
         These.effectToEither,
       ),
     )

packages/@contentlayer/source-files/src/errors/index.ts

@@ -355,6 +355,47 @@ This is possibly a bug in Contentlayer. Please open an issue.`
 
     renderLine = () => `"${this.documentFilePath}": ${errorToString(this.error)}`
   }
+
+  const tagToError = {
+    InvalidFrontmatterError: InvalidFrontmatterError,
+    InvalidMarkdownFileError: InvalidMarkdownFileError,
+    InvalidYamlFileError: InvalidYamlFileError,
+    InvalidJsonFileError: InvalidJsonFileError,
+    ComputedValueError: ComputedValueError,
+    UnsupportedFileExtension: UnsupportedFileExtension,
+    FileExtensionMismatch: FileExtensionMismatch,
+    NoSuchDocumentTypeError: NoSuchDocumentTypeError,
+    NoSuchNestedDocumentTypeError: NoSuchNestedDocumentTypeError,
+    CouldNotDetermineDocumentTypeError: CouldNotDetermineDocumentTypeError,
+    MissingRequiredFieldsError: MissingRequiredFieldsError,
+    ExtraFieldDataError: ExtraFieldDataError,
+    ReferencedFileDoesNotExistError: ReferencedFileDoesNotExistError,
+    IncompatibleFieldDataError: IncompatibleFieldDataError,
+    SingletonDocumentNotFoundError: SingletonDocumentNotFoundError,
+    UnexpectedError: UnexpectedError,
+  }
+
+  export const fromSerialized = ({
+    _tag,
+    ...rest
+  }: {
+    _tag: keyof typeof tagToError
+    // FIXME: Obviously, this type is a lie, arguments would be a union of
+    // class params, instead of an intersection.
+    readonly error: unknown
+    readonly documentFilePath: PosixFilePath
+    readonly documentTypeName: string
+    readonly fieldName: string
+    readonly extension: string
+    readonly filePath: string
+    readonly typeFieldName: string
+    readonly contentType: DocumentContentType
+    readonly referencedFilePath: PosixFilePath
+    readonly validNestedTypeNames: readonly string[]
+    readonly fieldDefsWithMissingData: core.FieldDef[]
+    readonly extraFieldEntries: readonly (readonly [fieldKey: string, fieldValue: any])[]
+    readonly incompatibleFieldData: readonly (readonly [fieldKey: string, fieldValue: any])[]
+  }): FetchDataError => new tagToError[_tag](rest)
 }
 
 export type SchemaError = DuplicateBodyFieldError

packages/@contentlayer/source-files/src/fetchData/DocumentTypeMap.ts

@@ -1,8 +1,8 @@
 import type { PosixFilePath } from '@contentlayer/utils'
-import type { Has } from '@contentlayer/utils/effect'
-import { HashMap, O, pipe, State, T, Tagged } from '@contentlayer/utils/effect'
+import type { Has, Option, These } from '@contentlayer/utils/effect'
+import { HashMap, O, pipe, State, T, Tagged, Tuple } from '@contentlayer/utils/effect'
 
-type DocumentTypeName = string
+export type DocumentTypeName = string
 
 export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{
   readonly map: HashMap.HashMap<DocumentTypeName, PosixFilePath[]>
@@ -29,6 +29,21 @@ export class DocumentTypeMap extends Tagged('@local/DocumentTypeMap')<{
  */
 export const DocumentTypeMapState = State.State<DocumentTypeMap>(DocumentTypeMap._tag)
 
+export const serialize = () =>
+  pipe(
+    DocumentTypeMapState.get,
+    // FIXME: unsafe
+    T.chain((map) => T.succeedWith(() => JSON.stringify(map))),
+  )
+
+export function provideFromSerialized(serialized: string) {
+  return pipe(
+    // FIXME: unsafe
+    T.succeedWith<DocumentTypeMap>(() => JSON.parse(serialized)),
+    T.map((map) => T.provideSomeLayer(DocumentTypeMapState.Live(new DocumentTypeMap(map)))),
+  )
+}
+
 export const provideDocumentTypeMapState = T.provideSomeLayer(DocumentTypeMapState.Live(DocumentTypeMap.init()))
 
 export type HasDocumentTypeMapState = Has<State.State<DocumentTypeMap>>