refactor(document): add web document service api

ee/tabby-db/src/lib.rs

@@ -21,6 +21,7 @@ use user_completions::UserCompletionDailyStatsDAO;
 pub use user_events::UserEventDAO;
 pub use users::UserDAO;
 pub use web_crawler::WebCrawlerUrlDAO;
+pub use web_documents::WebDocumentDAO;
 
 pub mod cache;
 mod email_setting;

ee/tabby-schema/graphql/schema.graphql

@@ -99,6 +99,11 @@ input CodeSearchParamsOverrideInput {
   numToScore: Int
 }
 
+input CreateCustomDocumentInput {
+  name: String!
+  url: String!
+}
+
 input CreateIntegrationInput {
   displayName: String!
   accessToken: String!
@@ -182,6 +187,11 @@ input SecuritySettingInput {
   disableClientSideTelemetry: Boolean!
 }
 
+input SetPresetDocumentActiveInput {
+  name: String!
+  active: Boolean!
+}
+
 input ThreadRunDebugOptionsInput {
   codeSearchParamsOverride: CodeSearchParamsOverrideInput = null
 }
@@ -232,6 +242,24 @@ type CompletionStats {
   selects: Int!
 }
 
+type CustomDocumentConnection {
+  edges: [CustomDocumentEdge!]!
+  pageInfo: PageInfo!
+}
+
+type CustomDocumentEdge {
+  node: CustomWebDocument!
+  cursor: String!
+}
+
+type CustomWebDocument {
+  url: String!
+  name: String!
+  id: ID!
+  createdAt: DateTime!
+  jobInfo: JobInfo!
+}
+
 type DiskUsage {
   filepath: [String!]!
   "Size in kilobytes."
@@ -475,6 +503,9 @@ type Mutation {
   triggerJobRun(command: String!): ID!
   createWebCrawlerUrl(input: CreateWebCrawlerUrlInput!): ID!
   deleteWebCrawlerUrl(id: ID!): Boolean!
+  createCustomDocument(input: CreateCustomDocumentInput!): ID!
+  deleteCustomDocument(id: ID!): Boolean!
+  setPresetDocumentActive(input: SetPresetDocumentActiveInput!): ID!
 }
 
 type NetworkSetting {
@@ -495,6 +526,23 @@ type PageInfo {
   endCursor: String
 }
 
+type PresetDocumentConnection {
+  edges: [PresetDocumentEdge!]!
+  pageInfo: PageInfo!
+}
+
+type PresetDocumentEdge {
+  node: PresetWebDocument!
+  cursor: String!
+}
+
+type PresetWebDocument {
+  name: String!
+  id: ID!
+  active: Boolean!
+  jobInfo: JobInfo
+}
+
 type ProvidedRepository {
   id: ID!
   integrationId: ID!
@@ -566,6 +614,8 @@ type Query {
     Thread is public within an instance, so no need to check for ownership.
   """
   threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection!
+  customWebDocuments(after: String, before: String, first: Int, last: Int): CustomDocumentConnection!
+  presetWebDocuments(after: String, before: String, first: Int, last: Int, active: Boolean!): PresetDocumentConnection!
 }
 
 type RefreshTokenResponse {

ee/tabby-schema/src/schema/mod.rs

@@ -10,6 +10,7 @@ pub mod setting;
 pub mod thread;
 pub mod user_event;
 pub mod web_crawler;
+pub mod web_documents;
 pub mod worker;
 
 use std::sync::Arc;
@@ -51,7 +52,9 @@ use self::{
     },
     user_event::{UserEvent, UserEventService},
     web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl},
+    web_documents::{CreateCustomDocumentInput, CustomWebDocument, WebDocumentService},
 };
+use crate::web_documents::{PresetWebDocument, SetPresetDocumentActiveInput};
 use crate::{
     env,
     juniper::relay::{self, query_async, Connection},
@@ -71,6 +74,7 @@ pub trait ServiceLocator: Send + Sync {
     fn analytic(&self) -> Arc<dyn AnalyticService>;
     fn user_event(&self) -> Arc<dyn UserEventService>;
     fn web_crawler(&self) -> Arc<dyn WebCrawlerService>;
+    fn web_documents(&self) -> Arc<dyn WebDocumentService>;
     fn thread(&self) -> Arc<dyn ThreadService>;
 }
 
@@ -564,6 +568,46 @@ impl Query {
         )
         .await
     }
+
+    async fn custom_web_documents(
+        ctx: &Context,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<i32>,
+        last: Option<i32>,
+    ) -> Result<Connection<CustomWebDocument>> {
+        query_async(
+            after,
+            before,
+            first,
+            last,
+            |after, before, first, last| async move {
+                ctx.locator
+                    .web_documents()
+                    .list_custom_web_documents(after, before, first, last)
+                    .await
+            },
+        )
+        .await
+    }
+    async fn preset_web_documents(ctx: &Context,
+                                  after: Option<String>,
+                                  before: Option<String>,
+                                  first: Option<i32>,
+                                  last: Option<i32>,
+                                  active: bool) -> Result<Connection<PresetWebDocument>> {
+        query_async(
+            after,
+            before,
+            first,
+            last,
+            |after, before, first, last| async move {
+                ctx.locator
+                    .web_documents()
+                    .list_preset_web_documents(after, before, first, last, active)
+                    .await
+            }).await
+    }
 }
 
 #[derive(GraphQLObject)]
@@ -916,6 +960,37 @@ impl Mutation {
         ctx.locator.web_crawler().delete_web_crawler_url(id).await?;
         Ok(true)
     }
+
+    async fn create_custom_document(ctx: &Context, input: CreateCustomDocumentInput) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .create_custom_web_document(input.name, input.url)
+            .await?;
+        Ok(id)
+    }
+
+    async fn delete_custom_document(ctx: &Context, id: ID) -> Result<bool> {
+        ctx.locator
+            .web_documents()
+            .delete_custom_web_document(id)
+            .await?;
+        Ok(true)
+    }
+
+    async fn set_preset_document_active(
+        ctx: &Context,
+        input: SetPresetDocumentActiveInput,
+    ) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .set_preset_web_documents_active(input.name, input.active)
+            .await?;
+        Ok(id)
+    }
 }
 
 async fn check_analytic_access(ctx: &Context, users: &[ID]) -> Result<(), CoreError> {

ee/tabby-schema/src/schema/web_documents.rs

@@ -0,0 +1,111 @@
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use juniper::{GraphQLInputObject, GraphQLObject, ID};
+use validator::Validate;
+
+use crate::{job::JobInfo, juniper::relay::NodeType, Context, Result};
+
+#[derive(GraphQLObject)]
+#[graphql(context = Context)]
+pub struct CustomWebDocument {
+    pub url: String,
+    pub name: String,
+    pub id: ID,
+    pub created_at: DateTime<Utc>,
+    pub job_info: JobInfo,
+}
+
+#[derive(GraphQLObject)]
+#[graphql(context = Context)]
+pub struct PresetWebDocument {
+    pub name: String,
+    pub id: ID,
+    pub active: bool,
+    pub job_info: Option<JobInfo>,
+}
+
+impl CustomWebDocument {
+    pub fn source_id(&self) -> String {
+        Self::format_source_id(&self.id)
+    }
+
+    pub fn format_source_id(id: &ID) -> String {
+        format!("web_document:{}", id)
+    }
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct CreateCustomDocumentInput {
+    #[validate(regex(
+        code = "name",
+        path = "*crate::schema::constants::USERNAME_REGEX",
+        message = "Invalid repository name"
+    ))]
+    pub name: String,
+    #[validate(url(code = "url", message = "Invalid URL"))]
+    pub url: String,
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct SetPresetDocumentActiveInput {
+    #[validate(regex(
+        code = "name",
+        path = "*crate::schema::constants::USERNAME_REGEX",
+        message = "Invalid repository name"
+    ))]
+    pub name: String,
+    pub active: bool,
+}
+
+impl NodeType for CustomWebDocument {
+    type Cursor = String;
+
+    fn cursor(&self) -> Self::Cursor {
+        self.id.to_string()
+    }
+
+    fn connection_type_name() -> &'static str {
+        "CustomDocumentConnection"
+    }
+
+    fn edge_type_name() -> &'static str {
+        "CustomDocumentEdge"
+    }
+}
+
+impl NodeType for PresetWebDocument {
+    type Cursor = String;
+
+    fn cursor(&self) -> Self::Cursor {
+        self.name.clone()
+    }
+
+    fn connection_type_name() -> &'static str {
+        "PresetDocumentConnection"
+    }
+
+    fn edge_type_name() -> &'static str {
+        "PresetDocumentEdge"
+    }
+}
+
+#[async_trait]
+pub trait WebDocumentService: Send + Sync {
+    async fn list_custom_web_documents(
+        &self,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<usize>,
+        last: Option<usize>,
+    ) -> Result<Vec<CustomWebDocument>>;
+
+    async fn create_custom_web_document(&self, name: String, url: String) -> Result<ID>;
+    async fn delete_custom_web_document(&self, id: ID) -> Result<()>;
+    async fn list_preset_web_documents(&self,
+                                       after: Option<String>,
+                                       before: Option<String>,
+                                       first: Option<usize>,
+                                       last: Option<usize>,
+                                       active: bool) -> Result<Vec<PresetWebDocument>>;
+    async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<ID>;
+}

ee/tabby-webserver/src/service/mod.rs

@@ -12,6 +12,7 @@ mod setting;
 mod thread;
 mod user_event;
 pub mod web_crawler;
+pub mod web_documents;
 
 use std::sync::Arc;
 
@@ -43,6 +44,7 @@ use tabby_schema::{
     thread::ThreadService,
     user_event::UserEventService,
     web_crawler::WebCrawlerService,
+    web_documents::WebDocumentService,
     worker::WorkerService,
     AsID, AsRowid, CoreError, Result, ServiceLocator,
 };
@@ -60,6 +62,7 @@ struct ServerContext {
     user_event: Arc<dyn UserEventService>,
     job: Arc<dyn JobService>,
     web_crawler: Arc<dyn WebCrawlerService>,
+    web_documents: Arc<dyn WebDocumentService>,
     thread: Arc<dyn ThreadService>,
 
     logger: Arc<dyn EventLogger>,
@@ -77,6 +80,7 @@ impl ServerContext {
         repository: Arc<dyn RepositoryService>,
         integration: Arc<dyn IntegrationService>,
         web_crawler: Arc<dyn WebCrawlerService>,
+        web_documents: Arc<dyn WebDocumentService>,
         job: Arc<dyn JobService>,
         answer: Option<Arc<AnswerService>>,
         db_conn: DbConn,
@@ -105,6 +109,7 @@ impl ServerContext {
                 setting.clone(),
             )),
             web_crawler,
+            web_documents,
             thread,
             license,
             repository,
@@ -260,6 +265,10 @@ impl ServiceLocator for ArcServerContext {
         self.0.web_crawler.clone()
     }
 
+    fn web_documents(&self) -> Arc<dyn WebDocumentService> {
+        self.0.web_documents.clone()
+    }
+
     fn thread(&self) -> Arc<dyn ThreadService> {
         self.0.thread.clone()
     }
@@ -271,6 +280,7 @@ pub async fn create_service_locator(
     repository: Arc<dyn RepositoryService>,
     integration: Arc<dyn IntegrationService>,
     web_crawler: Arc<dyn WebCrawlerService>,
+    web_documents: Arc<dyn WebDocumentService>,
     job: Arc<dyn JobService>,
     answer: Option<Arc<AnswerService>>,
     db: DbConn,
@@ -283,6 +293,7 @@ pub async fn create_service_locator(
             repository,
             integration,
             web_crawler,
+            web_documents,
             job,
             answer,
             db,