Skip to content

Commit

Permalink
feat: docker
Browse files Browse the repository at this point in the history
  • Loading branch information
matoous committed Mar 3, 2024
1 parent 3811585 commit e34ca2b
Show file tree
Hide file tree
Showing 7 changed files with 82 additions and 73 deletions.
31 changes: 31 additions & 0 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,34 @@ jobs:
run: cargo doc --no-deps --workspace --document-private-items
env:
RUSTDOCFLAGS: -D warnings

build:
name: Build and push
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Log in to the Container registry
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}

- name: Build and push Docker image
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 1 addition & 20 deletions mwp-content/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ impl Page {
let mut link_title = String::new();
let mut links: Vec<Link> = Vec::new();
let mut open_tags: Vec<Tag> = Vec::new();
let mut got_resources: bool = false;

#[allow(clippy::unnecessary_filter_map)]
let parser = Parser::new_ext(&content, options).filter_map(|event| match event.clone() {
Expand All @@ -93,25 +92,10 @@ impl Page {
link_title = text.to_string();
}

// once we detect "Resources" section of the markdown we ignore the rest of the document
if text.as_ref() == "Resources" {
got_resources = true;
return Some(event);
}

if got_resources {
return None;
}

Some(event)
}
Event::Start(tag) => {
open_tags.push(tag);

if got_resources {
return None;
}

Some(event)
}
Event::End(tag) => {
Expand All @@ -137,10 +121,7 @@ impl Page {

Some(event)
}
event => match got_resources {
true => None,
false => Some(event),
},
event => Some(event),
});

let mut html_output = String::new();
Expand Down
34 changes: 15 additions & 19 deletions mwp-search/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,21 @@ impl Doc {
}

impl SearchIndex {
pub fn new(dir: &str) -> Result<Self, Box<dyn std::error::Error>> {
let index = if let Ok(index) = Index::open_in_dir(dir) {
index
} else {
let mut schema_builder = Schema::builder();
schema_builder.add_text_field("url", STRING | STORED);
schema_builder.add_text_field("title", TEXT | STORED);
schema_builder.add_text_field("body", TEXT);
schema_builder.add_text_field("domain", STRING | STORED);
schema_builder.add_text_field("tags", STRING | STORED | FAST);
let schema = schema_builder.build();
Index::builder()
.schema(schema)
.settings(tantivy::IndexSettings {
docstore_blocksize: 32_000_000,
..tantivy::IndexSettings::default()
})
.create_in_dir(dir)?
};
pub fn new() -> Result<Self, Box<dyn std::error::Error>> {
let mut schema_builder = Schema::builder();
schema_builder.add_text_field("url", STRING | STORED);
schema_builder.add_text_field("title", TEXT | STORED);
schema_builder.add_text_field("body", TEXT);
schema_builder.add_text_field("domain", STRING | STORED);
schema_builder.add_text_field("tags", STRING | STORED | FAST);
let schema = schema_builder.build();
let index = Index::builder()
.schema(schema)
.settings(tantivy::IndexSettings {
docstore_blocksize: 32_000_000,
..tantivy::IndexSettings::default()
})
.create_in_ram()?;

Ok(SearchIndex { index })
}
Expand Down
2 changes: 2 additions & 0 deletions mwp-web/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,10 @@ maud = { version = "0.26.0", features = ["actix-web"] }
serde = "1.0.196"
serde_json = "1.0.113"
tantivy = "0.21.1"
rusqlite = { version = "0.30.0", features = ["time", "url"]}

mwp-content = { path="../mwp-content" }
mwp-search = { path="../mwp-search" }

[build-dependencies]
grass = "0.13.2"
36 changes: 30 additions & 6 deletions mwp-web/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ use actix_web::{
};
use maud::{html, Markup, PreEscaped};
use mwp_content::Content;
use mwp_search::{Doc, SearchIndex};
use rusqlite::Connection;
use serde::Deserialize;
use tantivy::{
query::{AllQuery, QueryParser, TermQuery},
Expand Down Expand Up @@ -161,9 +163,6 @@ async fn content_page(
}
}
article { (PreEscaped(html)) }
.links {
(listing(searcher, schema, result.docs))
}
}
))
}
Expand All @@ -185,13 +184,38 @@ impl Guard for ContentGuard {
async fn main() -> std::io::Result<()> {
env_logger::init_from_env(env_logger::Env::new().default_filter_or("info"));

let index_path = "./index";
let index = Index::open_in_dir(index_path).unwrap();
let index = SearchIndex::new().unwrap();

let conn = Connection::open("./my_db.db3").unwrap();
let mut stmt = conn
.prepare("SELECT title, url, domain, body, tags, created_at, scraped_at FROM links")
.unwrap();
let docs_iter = stmt
.query_map([], |row| {
Ok(Doc {
title: row.get(0)?,
url: row.get(1)?,
domain: row.get(2)?,
body: row.get(3)?,
tags: row.get::<usize, Option<String>>(4).map(|res| {
res.map(|s| s.split(';').map(|s| s.into()).collect::<Vec<String>>())
})?,
created_at: row.get(5)?,
scraped_at: row.get(6)?,
})
})
.unwrap();

let mut builder = index.builder();
for doc in docs_iter {
builder.add(doc.unwrap()).unwrap();
}
builder.commit();
let content = Content::from_dir("../wiki").await;

HttpServer::new(move || {
App::new()
.app_data(web::Data::new(index.clone()))
.app_data(web::Data::new(index.index.clone()))
.app_data(web::Data::new(content.clone()))
.service(tag_page)
.service(search_page)
Expand Down
29 changes: 1 addition & 28 deletions src/main.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use mwp_content::Link;
use mwp_search::{Doc, SearchIndex};
use mwp_search::Doc;
use rusqlite::Connection;
use time::OffsetDateTime;

Expand All @@ -21,34 +21,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
(), // empty list of parameters.
)?;

let index = SearchIndex::new("./index")?;

let content = mwp_content::Content::from_dir("../wiki").await;
index.add_content(content.values())?;

let mut stmt =
conn.prepare("SELECT title, url, domain, body, tags, created_at, scraped_at FROM links")?;
let docs_iter = stmt.query_map([], |row| {
Ok(Doc {
title: row.get(0)?,
url: row.get(1)?,
domain: row.get(2)?,
body: row.get(3)?,
tags: row
.get::<usize, Option<String>>(4)
.map(|res| res.map(|s| s.split(';').map(|s| s.into()).collect::<Vec<String>>()))?,
created_at: row.get(5)?,
scraped_at: row.get(6)?,
})
})?;

let mut builder = index.builder();
for doc in docs_iter {
builder.add(doc.unwrap())?;
}
builder.commit();

return Ok(());

// only needed before links from content are migrated to bookmarking system
let links = content
Expand Down

0 comments on commit e34ca2b

Please sign in to comment.