refactor(document): add web document service api

ee/tabby-db/src/lib.rs

@@ -21,6 +21,7 @@ use user_completions::UserCompletionDailyStatsDAO;
 pub use user_events::UserEventDAO;
 pub use users::UserDAO;
 pub use web_crawler::WebCrawlerUrlDAO;
+pub use web_documents::WebDocumentDAO;
 
 pub mod cache;
 mod email_setting;

ee/tabby-schema/graphql/schema.graphql

@@ -99,6 +99,11 @@ input CodeSearchParamsOverrideInput {
   numToScore: Int
 }
 
+input CreateCustomDocumentInput {
+  name: String!
+  url: String!
+}
+
 input CreateIntegrationInput {
   displayName: String!
   accessToken: String!
@@ -182,6 +187,11 @@ input SecuritySettingInput {
   disableClientSideTelemetry: Boolean!
 }
 
+input SetPresetDocumentActiveInput {
+  name: String!
+  active: Boolean!
+}
+
 input ThreadRunDebugOptionsInput {
   codeSearchParamsOverride: CodeSearchParamsOverrideInput = null
 }
@@ -475,6 +485,9 @@ type Mutation {
   triggerJobRun(command: String!): ID!
   createWebCrawlerUrl(input: CreateWebCrawlerUrlInput!): ID!
   deleteWebCrawlerUrl(id: ID!): Boolean!
+  createCustomDocument(input: CreateCustomDocumentInput!): ID!
+  deleteCustomDocument(id: ID!): Boolean!
+  setPresetDocumentActive(input: SetPresetDocumentActiveInput!): ID!
 }
 
 type NetworkSetting {
@@ -566,6 +579,8 @@ type Query {
     Thread is public within an instance, so no need to check for ownership.
   """
   threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection!
+  customWebDocuments(after: String, before: String, first: Int, last: Int): WebDocumentConnection!
+  presetWebDocuments(active: Boolean!): [WebDocument!]!
 }
 
 type RefreshTokenResponse {
@@ -710,3 +725,20 @@ type WebCrawlerUrlEdge {
   node: WebCrawlerUrl!
   cursor: String!
 }
+
+type WebDocument {
+  url: String!
+  id: ID!
+  createdAt: DateTime!
+  jobInfo: JobInfo!
+}
+
+type WebDocumentConnection {
+  edges: [WebDocumentEdge!]!
+  pageInfo: PageInfo!
+}
+
+type WebDocumentEdge {
+  node: WebDocument!
+  cursor: String!
+}

ee/tabby-schema/src/schema/mod.rs

@@ -10,6 +10,7 @@ pub mod setting;
 pub mod thread;
 pub mod user_event;
 pub mod web_crawler;
+pub mod web_documents;
 pub mod worker;
 
 use std::sync::Arc;
@@ -51,7 +52,9 @@ use self::{
     },
     user_event::{UserEvent, UserEventService},
     web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl},
+    web_documents::{CreateCustomDocumentInput, WebDocument, WebDocumentService},
 };
+use crate::web_documents::SetPresetDocumentActiveInput;
 use crate::{
     env,
     juniper::relay::{self, query_async, Connection},
@@ -71,6 +74,7 @@ pub trait ServiceLocator: Send + Sync {
     fn analytic(&self) -> Arc<dyn AnalyticService>;
     fn user_event(&self) -> Arc<dyn UserEventService>;
     fn web_crawler(&self) -> Arc<dyn WebCrawlerService>;
+    fn web_documents(&self) -> Arc<dyn WebDocumentService>;
     fn thread(&self) -> Arc<dyn ThreadService>;
 }
 
@@ -564,6 +568,34 @@ impl Query {
         )
         .await
     }
+
+    async fn custom_web_documents(
+        ctx: &Context,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<i32>,
+        last: Option<i32>,
+    ) -> Result<Connection<WebDocument>> {
+        query_async(
+            after,
+            before,
+            first,
+            last,
+            |after, before, first, last| async move {
+                ctx.locator
+                    .web_documents()
+                    .list_custom_web_documents(after, before, first, last)
+                    .await
+            },
+        )
+        .await
+    }
+    async fn preset_web_documents(ctx: &Context, active: bool) -> Result<Vec<WebDocument>> {
+        ctx.locator
+            .web_documents()
+            .list_preset_web_documents(active)
+            .await
+    }
 }
 
 #[derive(GraphQLObject)]
@@ -916,6 +948,37 @@ impl Mutation {
         ctx.locator.web_crawler().delete_web_crawler_url(id).await?;
         Ok(true)
     }
+
+    async fn create_custom_document(ctx: &Context, input: CreateCustomDocumentInput) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .create_custom_web_document(input.name, input.url)
+            .await?;
+        Ok(id)
+    }
+
+    async fn delete_custom_document(ctx: &Context, id: ID) -> Result<bool> {
+        ctx.locator
+            .web_documents()
+            .delete_custom_web_document(id)
+            .await?;
+        Ok(true)
+    }
+
+    async fn set_preset_document_active(
+        ctx: &Context,
+        input: SetPresetDocumentActiveInput,
+    ) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .set_preset_web_documents_active(input.name, input.active)
+            .await?;
+        Ok(id)
+    }
 }
 
 async fn check_analytic_access(ctx: &Context, users: &[ID]) -> Result<(), CoreError> {

ee/tabby-schema/src/schema/web_documents.rs

@@ -0,0 +1,72 @@
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use juniper::{GraphQLInputObject, GraphQLObject, ID};
+use validator::Validate;
+
+use crate::{job::JobInfo, juniper::relay::NodeType, Context, Result};
+
+#[derive(GraphQLObject)]
+#[graphql(context = Context)]
+pub struct WebDocument {
+    pub url: String,
+    pub id: ID,
+    pub created_at: DateTime<Utc>,
+    pub job_info: JobInfo,
+}
+
+impl WebDocument {
+    pub fn source_id(&self) -> String {
+        Self::format_source_id(&self.id)
+    }
+
+    pub fn format_source_id(id: &ID) -> String {
+        format!("web_document:{}", id)
+    }
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct CreateCustomDocumentInput {
+    #[validate(url(code = "name", message = "Invalid Name"))]
+    pub name: String,
+    #[validate(url(code = "url", message = "Invalid URL"))]
+    pub url: String,
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct SetPresetDocumentActiveInput {
+    #[validate(url(code = "name", message = "Invalid Name"))]
+    pub name: String,
+    pub active: bool,
+}
+
+impl NodeType for WebDocument {
+    type Cursor = String;
+
+    fn cursor(&self) -> Self::Cursor {
+        self.id.to_string()
+    }
+
+    fn connection_type_name() -> &'static str {
+        "WebDocumentConnection"
+    }
+
+    fn edge_type_name() -> &'static str {
+        "WebDocumentEdge"
+    }
+}
+
+#[async_trait]
+pub trait WebDocumentService: Send + Sync {
+    async fn list_custom_web_documents(
+        &self,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<usize>,
+        last: Option<usize>,
+    ) -> Result<Vec<WebDocument>>;
+
+    async fn create_custom_web_document(&self, name: String, url: String) -> Result<ID>;
+    async fn delete_custom_web_document(&self, id: ID) -> Result<()>;
+    async fn list_preset_web_documents(&self, active: bool) -> Result<Vec<WebDocument>>;
+    async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<ID>;
+}

ee/tabby-webserver/src/service/mod.rs

@@ -12,6 +12,7 @@ mod setting;
 mod thread;
 mod user_event;
 pub mod web_crawler;
+pub mod web_documents;
 
 use std::sync::Arc;
 
@@ -43,6 +44,7 @@ use tabby_schema::{
     thread::ThreadService,
     user_event::UserEventService,
     web_crawler::WebCrawlerService,
+    web_documents::WebDocumentService,
     worker::WorkerService,
     AsID, AsRowid, CoreError, Result, ServiceLocator,
 };
@@ -60,6 +62,7 @@ struct ServerContext {
     user_event: Arc<dyn UserEventService>,
     job: Arc<dyn JobService>,
     web_crawler: Arc<dyn WebCrawlerService>,
+    web_documents: Arc<dyn WebDocumentService>,
     thread: Arc<dyn ThreadService>,
 
     logger: Arc<dyn EventLogger>,
@@ -77,6 +80,7 @@ impl ServerContext {
         repository: Arc<dyn RepositoryService>,
         integration: Arc<dyn IntegrationService>,
         web_crawler: Arc<dyn WebCrawlerService>,
+        web_documents: Arc<dyn WebDocumentService>,
         job: Arc<dyn JobService>,
         answer: Option<Arc<AnswerService>>,
         db_conn: DbConn,
@@ -105,6 +109,7 @@ impl ServerContext {
                 setting.clone(),
             )),
             web_crawler,
+            web_documents,
             thread,
             license,
             repository,
@@ -260,6 +265,10 @@ impl ServiceLocator for ArcServerContext {
         self.0.web_crawler.clone()
     }
 
+    fn web_documents(&self) -> Arc<dyn WebDocumentService> {
+        self.0.web_documents.clone()
+    }
+
     fn thread(&self) -> Arc<dyn ThreadService> {
         self.0.thread.clone()
     }
@@ -271,6 +280,7 @@ pub async fn create_service_locator(
     repository: Arc<dyn RepositoryService>,
     integration: Arc<dyn IntegrationService>,
     web_crawler: Arc<dyn WebCrawlerService>,
+    web_documents: Arc<dyn WebDocumentService>,
     job: Arc<dyn JobService>,
     answer: Option<Arc<AnswerService>>,
     db: DbConn,
@@ -283,6 +293,7 @@ pub async fn create_service_locator(
             repository,
             integration,
             web_crawler,
+            web_documents,
             job,
             answer,
             db,

ee/tabby-webserver/src/service/web_documents.rs

@@ -0,0 +1,50 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use juniper::ID;
+use tabby_db::{DbConn, WebDocumentDAO};
+use tabby_schema::{
+    job::{JobInfo, JobService},
+    web_documents::{WebDocument, WebDocumentService},
+    AsID, AsRowid, Result,
+};
+
+use super::{background_job::BackgroundJobEvent, graphql_pagination_to_filter};
+
+pub fn create(db: DbConn, job_service: Arc<dyn JobService>) -> impl WebDocumentService {
+    WebDocumentServiceImpl { db, job_service }
+}
+
+struct WebDocumentServiceImpl {
+    db: DbConn,
+    job_service: Arc<dyn JobService>,
+}
+
+#[async_trait]
+impl WebDocumentService for WebDocumentServiceImpl {
+    async fn list_custom_web_documents(
+        &self,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<usize>,
+        last: Option<usize>,
+    ) -> Result<Vec<WebDocument>> {
+        Ok(vec![])
+    }
+
+    async fn create_custom_web_document(&self, name: String, url: String) -> Result<ID> {
+        Ok(ID::new("0"))
+    }
+
+    async fn delete_custom_web_document(&self, id: ID) -> Result<()> {
+        Ok(())
+    }
+
+    async fn list_preset_web_documents(&self, active: bool) -> Result<Vec<WebDocument>> {
+        Ok(vec![])
+    }
+
+    async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<ID> {
+        Ok(ID::new("0"))
+    }
+}