feat: add types and use refine taxonomies structure for sources (#56)

* feat: create sources page (#41)

* feat: create sources page

* add svg assets

* update gitignore and tailwind wind config

* add reusable functions

* create sources data

* feat: add transcript details card

* fix: update breadcrumb functionality

* chore: update sources page to use nested directories

* chore: add media responsiveness, fix explore navigation and breadcrumb active state

* chore: reduce size of body

* scale down size of source data

* filter index from getStaticPaths

* fix: make changes to generateStaticParams function, add correct type for slugParams in contentLayer

* chore: add source link to page header

* fix: inconsistent page scroll for large number of items

* fix: scaling for transcript details cards

* Add custom error page (#48)

* feat: add custom error page

* create: re-usable error skeleton, add no search result component

* fix: scroll, header and add footer (#47)

* fix: scroll, header and add footer

* fix: switch to fixed height for explore content

* fix: switch layout to template

* Feat/update contentlayer algo (#51)

* build: initialize submodules

* add source directory

* modify types algorithm, contentTree algorithm

* move sources to base route

* fix: correct back buttton

* fix: text corrections, explore navigation state

* chore: add additional sources

* fix: capitalize types title

* sort transcript page

* chore: fix count and sort for types and categories

* chore: route types to types page

* chore: point submodule to main and delete refine-taxonomies submodule branch (#55)

---------

Co-authored-by: Solomon eze <[email protected]>

.gitignore

@@ -25,6 +25,8 @@
 /public/aliases.json
 /public/topics-data.json
 /public/speaker-data.json
+/public/source-count-data.json
+/public/sources-data.json
 
 # misc
 .DS_Store

.gitmodules

@@ -4,4 +4,4 @@
 [submodule "public/gh-pages"]
 	path = public/gh-pages
 	url = https://github.com/bitcointranscripts/bitcointranscripts.github.io.git
-	branch = gh-pages
+	branch = gh-pages

contentlayer.config.ts

@@ -1,13 +1,8 @@
-import { createSlug, SpeakerData, TopicsData } from "./src/utils";
-import {
-  defineDocumentType,
-  defineNestedType,
-  makeSource,
-} from "contentlayer2/source-files";
-import { writeFileSync } from "fs";
 import path from "path";
 import * as fs from "fs";
-import { Transcript as ContentTranscriptType } from "./.contentlayer/generated/types";
+import { createSlug, createText, SpeakerData, TopicsData, unsluggify } from "./src/utils";
+import { defineDocumentType, defineNestedType, makeSource } from "contentlayer2/source-files";
+import { Transcript as ContentTranscriptType, Source as ContentSourceType } from "./.contentlayer/generated/types";
 
 const Resources = defineNestedType(() => ({
   name: "Resources",
@@ -16,7 +11,6 @@ const Resources = defineNestedType(() => ({
     url: { type: "string" },
   },
 }));
-
 export interface CategoryInfo {
   title: string;
   slug: string;
@@ -32,6 +26,10 @@ interface TagInfo {
   count: number;
 }
 
+interface ContentTree {
+  [key: string]: ContentTree | ContentTranscriptType[];
+}
+
 /**
  * Count the occurrences of all tags across transcripts and write to json file
  */
@@ -65,7 +63,7 @@ const getTranscriptAliases = (allTranscripts: ContentTranscriptType[]) => {
     }
   }
 
-  writeFileSync("./public/aliases.json", JSON.stringify(aliases));
+  fs.writeFileSync("./public/aliases.json", JSON.stringify(aliases));
 };
 
 const getCategories = () => {
@@ -96,11 +94,7 @@ function organizeTags(transcripts: ContentTranscriptType[]) {
   });
 
   // Process all tags at once
-  const allTags = new Set(
-    transcripts.flatMap(
-      (transcript) => transcript.tags?.map((tag) => tag) || []
-    )
-  );
+  const allTags = new Set(transcripts.flatMap((transcript) => transcript.tags?.map((tag) => tag) || []));
 
   allTags.forEach((tag) => {
     const catInfo = categoryMap.get(tag);
@@ -122,21 +116,19 @@ function organizeTags(transcripts: ContentTranscriptType[]) {
 
   // Add "Miscellaneous" category with remaining uncategorized tags
   if (tagsWithoutCategory.size > 0) {
-    tagsByCategory["Miscellaneous"] = Array.from(tagsWithoutCategory).map(
-      (tag) => ({
-        name: tag,
-        slug: tag,
-        count: tagCounts[tag] || 0,
-      })
-    );
+    tagsByCategory["Miscellaneous"] = Array.from(tagsWithoutCategory).map((tag) => ({
+      name: tag,
+      slug: tag,
+      count: tagCounts[tag] || 0,
+    }));
   }
 
   // Sort tags alphabetically within each category
   Object.keys(tagsByCategory).forEach((category) => {
     tagsByCategory[category].sort((a, b) => a.name.localeCompare(b.name));
   });
 
-  writeFileSync("./public/tag-data.json", JSON.stringify(tagsByCategory));
+  fs.writeFileSync("./public/tag-data.json", JSON.stringify(tagsByCategory));
   return { tagsByCategory, tagsWithoutCategory };
 }
 
@@ -146,11 +138,11 @@ function organizeTopics(transcripts: ContentTranscriptType[]) {
 
   transcripts.forEach((transcript) => {
     const slugTags = transcript.tags?.map((tag) => ({
-      slug:createSlug(tag),
-      name:tag
+      slug: createSlug(tag),
+      name: tag,
     }));
 
-    slugTags?.forEach(({slug, name}) => {
+    slugTags?.forEach(({ slug, name }) => {
       if (slugTopics[slug] !== undefined) {
         const index = slugTopics[slug];
         topicsArray[index].count += 1;
@@ -166,56 +158,20 @@ function organizeTopics(transcripts: ContentTranscriptType[]) {
     });
   });
 
-  writeFileSync("./public/topics-data.json", JSON.stringify(topicsArray));
+  fs.writeFileSync("./public/topics-data.json", JSON.stringify(topicsArray));
 }
-/**
- * Count the occurrences of all types across transcripts and write to json file
- */
-const createTypesCount = (allTranscripts: ContentTranscriptType[]) => {
-  const typesAndCount: Record<string, number> = {};
-  const relevantTypes = [
-    "video",
-    "core-dev-tech",
-    "podcast",
-    "conference",
-    "meeting",
-    "club",
-    "meetup",
-    "hackathon",
-    "workshop",
-    "residency",
-    "developer-tools",
-  ];
-
-  allTranscripts.forEach((transcript) => {
-    if (transcript.categories) {
-      transcript.categories.forEach((type: string) => {
-        const formattedType = createSlug(type);
-        if (relevantTypes.includes(formattedType)) {
-          if (formattedType in typesAndCount) {
-            typesAndCount[formattedType] += 1;
-          } else {
-            typesAndCount[formattedType] = 1;
-          }
-        }
-      });
-    }
-  });
-
-  writeFileSync("./public/types-data.json", JSON.stringify(typesAndCount));
-};
 
 function createSpeakers(transcripts: ContentTranscriptType[]) {
   const slugSpeakers: any = {};
   const speakerArray: SpeakerData[] = [];
 
   transcripts.forEach((transcript) => {
     const slugSpeakersArray = transcript.speakers?.map((speaker) => ({
-      slug:createSlug(speaker),
+      slug: createSlug(speaker),
       name: speaker,
     }));
 
-    slugSpeakersArray?.forEach(({slug, name}) => {
+    slugSpeakersArray?.forEach(({ slug, name }) => {
       if (slugSpeakers[slug] !== undefined) {
         const index = slugSpeakers[slug];
         speakerArray[index].count += 1;
@@ -231,8 +187,95 @@ function createSpeakers(transcripts: ContentTranscriptType[]) {
     });
   });
 
+  fs.writeFileSync("./public/speaker-data.json", JSON.stringify(speakerArray));
+}
+
+function generateSourcesCount(transcripts: ContentTranscriptType[], sources: ContentSourceType[]) {
+  const sourcesArray: TagInfo[] = [];
+  const slugSources: Record<string, number> = {};
+
+  transcripts.forEach((transcript) => {
+    const slug = transcript._raw.flattenedPath.split("/")[0];
 
-  writeFileSync("./public/speaker-data.json", JSON.stringify(speakerArray));
+    if (slugSources[slug] !== undefined) {
+      sourcesArray[slugSources[slug]].count += 1;
+    } else {
+      const sourcesLength = sourcesArray.length;
+      slugSources[slug] = sourcesLength;
+
+      const getSourceName = (slug: string) =>
+        sources.find((source) => source.language === "en" && source.slugAsParams[0] === slug)?.title ?? unsluggify(slug);
+
+      sourcesArray[sourcesLength] = {
+        slug,
+        name: getSourceName(slug),
+        count: 1,
+      };
+    }
+  });
+
+  fs.writeFileSync("./public/source-count-data.json", JSON.stringify(sourcesArray));
+  return { sourcesArray, slugSources };
+}
+
+const createTypesCount = (transcripts: ContentTranscriptType[], sources: ContentSourceType[]) => {
+  const { sourcesArray, slugSources } = generateSourcesCount(transcripts, sources);
+  const nestedTypes: any = {};
+
+  sources.forEach((transcript) => {
+    if (transcript.types) {
+      transcript.types.forEach((type) => {
+        const slugType = type.charAt(0).toUpperCase() + type.slice(1);
+        const slug = transcript.slugAsParams[0];
+
+        const sourceIndex = slugSources[slug];
+        const getSource = sourcesArray[sourceIndex] ?? null;
+
+        if (!nestedTypes[slugType]) {
+          nestedTypes[slugType] = [];
+        } else {
+          if (nestedTypes[slugType].includes(getSource) || getSource === null) return;
+          nestedTypes[slugType].push(getSource);
+        }
+      });
+    }
+  });
+
+  fs.writeFileSync("./public/types-data.json", JSON.stringify(nestedTypes));
+};
+
+function organizeContent(transcripts: ContentTranscriptType[]) {
+  const tree: ContentTree = {};
+
+  transcripts.forEach((transcript) => {
+    const parts = transcript.slugAsParams;
+    let current = tree;
+
+    const isNonEnglishDir = /\w+\.[a-z]{2}\b/.test(parts[parts.length - 1]);
+    if (isNonEnglishDir) return;
+
+    for (let i = 0; i < parts.length - 1; i++) {
+      if (!current[parts[i]]) {
+        current[parts[i]] = i === parts.length - 2 ? [] : {};
+      }
+      current = current[parts[i]] as ContentTree;
+    }
+
+    (current as unknown as any[]).push({
+      title: transcript.title,
+      speakers: transcript.speakers,
+      date: transcript.date,
+      tags: transcript.tags,
+      sourceFilePath: transcript._raw.sourceFilePath,
+      flattenedPath: transcript._raw.flattenedPath,
+      summary: transcript.summary,
+      body: createText(transcript.body),
+      source: transcript.source,
+    });
+  });
+
+  // Save the result as JSON
+  fs.writeFileSync("./public/sources-data.json", JSON.stringify(tree, null, 2));
 }
 
 export const Transcript = defineDocumentType(() => ({
@@ -261,22 +304,58 @@ export const Transcript = defineDocumentType(() => ({
     aditional_resources: { type: "list", of: Resources },
     additional_resources: { type: "list", of: Resources },
     weight: { type: "number" },
+    types: { type: "list", of: { type: "string" } },
+    source_file: { type: "string" },
   },
   computedFields: {
     url: {
       type: "string",
       resolve: (doc) => `/${doc._raw.flattenedPath}`,
     },
     slugAsParams: {
-      type: "string",
+      type: "list",
       resolve: (doc) => doc._raw.flattenedPath.split("/"),
     },
   },
 }));
 
+export const Source = defineDocumentType(() => ({
+  name: "Source",
+  filePathPattern: `**/_index{,.??}.md`,
+  contentType: "markdown",
+  fields: {
+    title: { type: "string", required: true },
+    source: { type: "string" },
+    transcription_coverage: { type: "string" },
+    hosts: { type: "list", of: { type: "string" } },
+    weight: { type: "number" },
+    website: { type: "string" },
+    types: { type: "list", of: { type: "string" } },
+    additional_resources: { type: "list", of: Resources },
+  },
+  computedFields: {
+    url: {
+      type: "string",
+      resolve: (doc) => `/${doc._raw.flattenedPath.split("/").slice(0, -1).join("/")}`,
+    },
+    language: {
+      type: "string",
+      resolve: (doc) => {
+        const index = doc._raw.flattenedPath.split("/").pop();
+        const lan = index?.split(".").length === 2 ? index?.split(".")[1] : "en";
+        return lan;
+      },
+    },
+    slugAsParams: {
+      type: "list",
+      resolve: (doc) => doc._raw.flattenedPath.split("/").slice(0, -1),
+    },
+  },
+}));
+
 export default makeSource({
   contentDirPath: path.join(process.cwd(), "public", "bitcoin-transcript"),
-  documentTypes: [Transcript],
+  documentTypes: [Source, Transcript],
   contentDirExclude: [
     ".github",
     ".gitignore",
@@ -288,11 +367,13 @@ export default makeSource({
     "2018-08-17-richard-bondi-bitcoin-cli-regtest.es.md",
   ],
   onSuccess: async (importData) => {
-    const { allDocuments } = await importData();
-    organizeTags(allDocuments);
-    createTypesCount(allDocuments);
-    organizeTopics(allDocuments);
-    getTranscriptAliases(allDocuments);
-    createSpeakers(allDocuments);
+    const { allTranscripts, allSources } = await importData();
+    organizeTags(allTranscripts);
+    createTypesCount(allTranscripts, allSources);
+    organizeTopics(allTranscripts);
+    getTranscriptAliases(allTranscripts);
+    createSpeakers(allTranscripts);
+    generateSourcesCount(allTranscripts, allSources);
+    organizeContent(allTranscripts);
   },
 });

next.config.mjs

@@ -5,8 +5,8 @@ const nextConfig = {
     return {
       fallback: [
         {
-          source: '/:path*.:ext([^/]+)', // intercept all paths ending with a file extension
-          destination: '/gh-pages/:path*.:ext', // rewrite to gh-pages/[path_here].ext
+          source: "/:path*.:ext([^/]+)", // intercept all paths ending with a file extension
+          destination: "/gh-pages/:path*.:ext", // rewrite to gh-pages/[path_here].ext
         },
         {
           source: "/transcripts",
@@ -20,8 +20,12 @@ const nextConfig = {
           source: "/:path*",
           destination: "/gh-pages/:path*/index.html",
         },
-      ]
-    }
+        {
+          source: "/sources/:path((?!.*\\.[^/]+).*)", // Matches /source/[any path without a file extension]
+          destination: "/[...slug]/:path*", // Replace with your catch-all route
+        },
+      ],
+    };
   },
 };