feat(search): searchable content

Cargo.toml

@@ -13,6 +13,7 @@ members = [
   "mwp-core",
   "mwp-web",
   "mwp-scraper",
+  "mwp-content",
 ]
 
 [workspace.package]
@@ -37,3 +38,4 @@ sled = "0.34.7"
 serde = "1.0.195"
 serde_json = "1.0.111"
 mwp-scraper = { path="./mwp-scraper" }
+mwp-content = { path="./mwp-content" }

mwp-content/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "mwp-content"
+authors.workspace = true
+version.workspace = true
+edition.workspace = true
+categories.workspace = true
+repository.workspace = true
+
+[dependencies]
+pulldown-cmark = "0.9.3"
+walkdir = "2.4.0"

mwp-content/src/lib.rs

@@ -0,0 +1,117 @@
+use std::{
+    collections::HashMap,
+    fmt::Write,
+    fs,
+    path::{Path, PathBuf},
+};
+
+use pulldown_cmark::{html, Options, Parser};
+use walkdir::WalkDir;
+
+#[derive(Clone)]
+pub struct Page {
+    pub title: String,
+    pub path: String,
+    pub html: String,
+    pub text: String,
+    pub tags: Vec<String>,
+    pub hiearchy: String,
+}
+
+fn remove_extension(path: &Path) -> PathBuf {
+    let mut new_path = PathBuf::new();
+    let parent_dir = path.parent().unwrap();
+    if let Some(file_name) = path.file_stem() {
+        new_path.push(parent_dir);
+        new_path.push(file_name);
+    }
+    new_path
+}
+
+pub async fn read_dir(src: &str) -> HashMap<String, Page> {
+    let mut entries = HashMap::new();
+
+    let mut options = Options::empty();
+    options.insert(Options::ENABLE_STRIKETHROUGH);
+
+    for entry in WalkDir::new(src)
+        .into_iter()
+        .filter_map(Result::ok)
+        .filter(|e| !e.file_type().is_dir())
+        .filter(|e| {
+            e.path()
+                .extension()
+                .is_some_and(|ext| ext.to_str() == Some("md"))
+        })
+    {
+        let contents =
+            fs::read_to_string(entry.path()).expect("Something went wrong reading the file");
+
+        let mut text_output = String::new();
+
+        let mut in_heading = false;
+        let mut title = String::new();
+        #[allow(clippy::unnecessary_filter_map)]
+        let parser = Parser::new_ext(&contents, options).filter_map(|event| {
+            match event.clone() {
+                pulldown_cmark::Event::Text(text) => {
+                    write!(&mut text_output, "{}", text).expect("write text output");
+                    if in_heading && title.is_empty() {
+                        title = text.to_string();
+                    }
+                }
+                pulldown_cmark::Event::Start(pulldown_cmark::Tag::Heading(
+                    pulldown_cmark::HeadingLevel::H1,
+                    _,
+                    _,
+                )) => {
+                    in_heading = true;
+                }
+                pulldown_cmark::Event::End(pulldown_cmark::Tag::Heading(
+                    pulldown_cmark::HeadingLevel::H1,
+                    _,
+                    _,
+                )) => {
+                    in_heading = false;
+                }
+                _ => (),
+            }
+            Some(event)
+        });
+
+        let mut html_output = String::new();
+        html::push_html(&mut html_output, parser);
+
+        let clean_path =
+            Path::new("/").join(remove_extension(entry.path().strip_prefix(src).unwrap()));
+
+        let parent = clean_path.parent().unwrap();
+
+        let tags = parent
+            .iter()
+            .filter_map(|component| {
+                if *component == Path::new("/") {
+                    None
+                } else {
+                    Some(component.to_str().unwrap().into())
+                }
+            })
+            .collect::<Vec<String>>();
+
+        let clean_path = clean_path.display().to_string();
+
+        entries.insert(
+            clean_path.to_owned(),
+            Page {
+                title,
+                path: clean_path,
+                html: html_output,
+                text: text_output,
+                tags,
+                hiearchy: parent.display().to_string(),
+            },
+        );
+    }
+
+    entries
+}

mwp-web/Cargo.toml

@@ -15,5 +15,7 @@ serde = "1.0.195"
 serde_json = "1.0.111"
 tantivy = "0.21.1"
 
+mwp-content = { path="../mwp-content" }
+
 [build-dependencies]
 grass = "0.13.1"

mwp-web/src/main.rs

@@ -1,5 +1,11 @@
+use std::collections::HashMap;
+
 use actix_files::Files;
-use actix_web::{get, web, App, HttpServer, Result as AwResult};
+use actix_web::{
+    get,
+    guard::{Guard, GuardContext},
+    web, App, HttpServer, Result as AwResult,
+};
 use maud::{html, Markup, PreEscaped};
 use serde::Deserialize;
 use tantivy::{
@@ -149,19 +155,78 @@ async fn tag_page(tag: web::Path<String>, index: web::Data<Index>) -> AwResult<M
     })
 }
 
+async fn content_page(
+    path: web::Path<Vec<String>>,
+    content: web::Data<Content>,
+) -> AwResult<Markup> {
+    match content
+        .docs
+        .get(format!("/{}", path.join("/").as_str()).as_str())
+    {
+        Some(mwp_content::Page { html: content, .. }) => Ok(html! {
+            html {
+                (render::header("Content | MWP"))
+                body {
+                    h1 { (path.join(",")) };
+                    main {
+                        article {
+                            (PreEscaped(content))
+                        }
+                    }
+                }
+            }
+        }),
+        None => Ok(html! {
+            html {
+                (render::header("Not found | MWP"))
+                body {
+                    h1 { "Not found" };
+                }
+            }
+        }),
+    }
+}
+
+#[derive(Clone)]
+struct Content {
+    pub docs: HashMap<String, mwp_content::Page>,
+}
+
+struct ContentGuard {
+    pub contents: Vec<String>,
+}
+
+impl Guard for ContentGuard {
+    fn check(&self, req: &GuardContext<'_>) -> bool {
+        self.contents.contains(&req.head().uri.path().to_string())
+    }
+}
+
 #[actix_web::main]
 async fn main() -> std::io::Result<()> {
     env_logger::init_from_env(env_logger::Env::new().default_filter_or("info"));
 
     let index_path = "../index";
     let index = Index::open_in_dir(index_path).unwrap();
+    let content = Content {
+        docs: mwp_content::read_dir("../../wiki").await,
+    };
 
     HttpServer::new(move || {
         App::new()
             .app_data(web::Data::new(index.clone()))
+            .app_data(web::Data::new(content.clone()))
             .service(index_page)
             .service(tag_page)
             .service(search_page)
+            .route(
+                "/{path:.*}",
+                web::get()
+                    .guard(ContentGuard {
+                        contents: content.docs.keys().cloned().collect(),
+                    })
+                    .to(content_page),
+            )
             .service(Files::new("/", "./static/"))
     })
     .bind(("127.0.0.1", 4444))?