refactor(document): add web document service api

ee/tabby-db/src/lib.rs

@@ -21,6 +21,7 @@ use user_completions::UserCompletionDailyStatsDAO;
 pub use user_events::UserEventDAO;
 pub use users::UserDAO;
 pub use web_crawler::WebCrawlerUrlDAO;
+pub use web_documents::WebDocumentDAO;
 
 pub mod cache;
 mod email_setting;

ee/tabby-schema/graphql/schema.graphql

@@ -99,6 +99,11 @@ input CodeSearchParamsOverrideInput {
   numToScore: Int
 }
 
+input CreateCustomDocumentInput {
+  name: String!
+  url: String!
+}
+
 input CreateIntegrationInput {
   displayName: String!
   accessToken: String!
@@ -182,6 +187,11 @@ input SecuritySettingInput {
   disableClientSideTelemetry: Boolean!
 }
 
+input SetPresetDocumentActiveInput {
+  name: String!
+  active: Boolean!
+}
+
 input ThreadRunDebugOptionsInput {
   codeSearchParamsOverride: CodeSearchParamsOverrideInput = null
 }
@@ -232,6 +242,24 @@ type CompletionStats {
   selects: Int!
 }
 
+type CustomDocumentConnection {
+  edges: [CustomDocumentEdge!]!
+  pageInfo: PageInfo!
+}
+
+type CustomDocumentEdge {
+  node: CustomWebDocument!
+  cursor: String!
+}
+
+type CustomWebDocument {
+  url: String!
+  name: String!
+  id: ID!
+  createdAt: DateTime!
+  jobInfo: JobInfo!
+}
+
 type DiskUsage {
   filepath: [String!]!
   "Size in kilobytes."
@@ -475,6 +503,9 @@ type Mutation {
   triggerJobRun(command: String!): ID!
   createWebCrawlerUrl(input: CreateWebCrawlerUrlInput!): ID!
   deleteWebCrawlerUrl(id: ID!): Boolean!
+  createCustomDocument(input: CreateCustomDocumentInput!): ID!
+  deleteCustomDocument(id: ID!): Boolean!
+  setPresetDocumentActive(input: SetPresetDocumentActiveInput!): ID!
 }
 
 type NetworkSetting {
@@ -495,6 +526,24 @@ type PageInfo {
   endCursor: String
 }
 
+type PresetDocumentConnection {
+  edges: [PresetDocumentEdge!]!
+  pageInfo: PageInfo!
+}
+
+type PresetDocumentEdge {
+  node: PresetWebDocument!
+  cursor: String!
+}
+
+type PresetWebDocument {
+  name: String!
+  id: ID!
+  active: Boolean!
+  updatedAt: DateTime
+  jobInfo: JobInfo
+}
+
 type ProvidedRepository {
   id: ID!
   integrationId: ID!
@@ -566,6 +615,8 @@ type Query {
     Thread is public within an instance, so no need to check for ownership.
   """
   threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection!
+  customWebDocuments(after: String, before: String, first: Int, last: Int): CustomDocumentConnection!
+  presetWebDocuments(after: String, before: String, first: Int, last: Int, active: Boolean!): PresetDocumentConnection!
 }
 
 type RefreshTokenResponse {

ee/tabby-schema/src/schema/constants.rs

@@ -5,6 +5,7 @@ lazy_static! {
     pub static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[a-zA-Z][\\w.-]+$").unwrap();
     pub static ref USERNAME_REGEX: Regex =
         Regex::new(r"^[^0-9±!@£$%^&*_+§¡€#¢¶•ªº«\\/<>?:;|=.,]{2,20}$").unwrap();
+    pub static ref DOCUMENT_NAME_REGEX: Regex = Regex::new(r"^[A-Za-z][A-Za-z0-9]*$").unwrap();
 }
 
 #[cfg(test)]

ee/tabby-schema/src/schema/mod.rs

@@ -10,6 +10,7 @@ pub mod setting;
 pub mod thread;
 pub mod user_event;
 pub mod web_crawler;
+pub mod web_documents;
 pub mod worker;
 
 use std::sync::Arc;
@@ -51,7 +52,9 @@ use self::{
     },
     user_event::{UserEvent, UserEventService},
     web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl},
+    web_documents::{CreateCustomDocumentInput, CustomWebDocument, WebDocumentService},
 };
+use crate::web_documents::{PresetWebDocument, SetPresetDocumentActiveInput};
 use crate::{
     env,
     juniper::relay::{self, query_async, Connection},
@@ -71,6 +74,7 @@ pub trait ServiceLocator: Send + Sync {
     fn analytic(&self) -> Arc<dyn AnalyticService>;
     fn user_event(&self) -> Arc<dyn UserEventService>;
     fn web_crawler(&self) -> Arc<dyn WebCrawlerService>;
+    fn web_documents(&self) -> Arc<dyn WebDocumentService>;
     fn thread(&self) -> Arc<dyn ThreadService>;
 }
 
@@ -564,6 +568,50 @@ impl Query {
         )
         .await
     }
+
+    async fn custom_web_documents(
+        ctx: &Context,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<i32>,
+        last: Option<i32>,
+    ) -> Result<Connection<CustomWebDocument>> {
+        query_async(
+            after,
+            before,
+            first,
+            last,
+            |after, before, first, last| async move {
+                ctx.locator
+                    .web_documents()
+                    .list_custom_web_documents(after, before, first, last)
+                    .await
+            },
+        )
+        .await
+    }
+    async fn preset_web_documents(
+        ctx: &Context,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<i32>,
+        last: Option<i32>,
+        active: bool,
+    ) -> Result<Connection<PresetWebDocument>> {
+        query_async(
+            after,
+            before,
+            first,
+            last,
+            |after, before, first, last| async move {
+                ctx.locator
+                    .web_documents()
+                    .list_preset_web_documents(after, before, first, last, active)
+                    .await
+            },
+        )
+        .await
+    }
 }
 
 #[derive(GraphQLObject)]
@@ -916,6 +964,37 @@ impl Mutation {
         ctx.locator.web_crawler().delete_web_crawler_url(id).await?;
         Ok(true)
     }
+
+    async fn create_custom_document(ctx: &Context, input: CreateCustomDocumentInput) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .create_custom_web_document(input.name, input.url)
+            .await?;
+        Ok(id)
+    }
+
+    async fn delete_custom_document(ctx: &Context, id: ID) -> Result<bool> {
+        ctx.locator
+            .web_documents()
+            .delete_custom_web_document(id)
+            .await?;
+        Ok(true)
+    }
+
+    async fn set_preset_document_active(
+        ctx: &Context,
+        input: SetPresetDocumentActiveInput,
+    ) -> Result<ID> {
+        input.validate()?;
+        let id = ctx
+            .locator
+            .web_documents()
+            .set_preset_web_documents_active(input.name, input.active)
+            .await?;
+        Ok(id)
+    }
 }
 
 async fn check_analytic_access(ctx: &Context, users: &[ID]) -> Result<(), CoreError> {

ee/tabby-schema/src/schema/web_documents.rs

@@ -0,0 +1,115 @@
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use juniper::{GraphQLInputObject, GraphQLObject, ID};
+use validator::Validate;
+
+use crate::{job::JobInfo, juniper::relay::NodeType, Context, Result};
+
+#[derive(GraphQLObject)]
+#[graphql(context = Context)]
+pub struct CustomWebDocument {
+    pub url: String,
+    pub name: String,
+    pub id: ID,
+    pub created_at: DateTime<Utc>,
+    pub job_info: JobInfo,
+}
+
+#[derive(GraphQLObject)]
+#[graphql(context = Context)]
+pub struct PresetWebDocument {
+    pub name: String,
+    pub id: ID,
+    pub active: bool,
+    /// `updated_at` is only filled when the preset is active.
+    pub updated_at: Option<DateTime<Utc>>,
+    pub job_info: Option<JobInfo>,
+}
+
+impl CustomWebDocument {
+    pub fn source_id(&self) -> String {
+        Self::format_source_id(&self.id)
+    }
+
+    pub fn format_source_id(id: &ID) -> String {
+        format!("web_document:{}", id)
+    }
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct CreateCustomDocumentInput {
+    #[validate(regex(
+        code = "name",
+        path = "*crate::schema::constants::DOCUMENT_NAME_REGEX",
+        message = "Invalid document name"
+    ))]
+    pub name: String,
+    #[validate(url(code = "url", message = "Invalid URL"))]
+    pub url: String,
+}
+
+#[derive(Validate, GraphQLInputObject)]
+pub struct SetPresetDocumentActiveInput {
+    #[validate(regex(
+        code = "name",
+        path = "*crate::schema::constants::DOCUMENT_NAME_REGEX",
+        message = "Invalid document name"
+    ))]
+    pub name: String,
+    pub active: bool,
+}
+
+impl NodeType for CustomWebDocument {
+    type Cursor = String;
+
+    fn cursor(&self) -> Self::Cursor {
+        self.id.to_string()
+    }
+
+    fn connection_type_name() -> &'static str {
+        "CustomDocumentConnection"
+    }
+
+    fn edge_type_name() -> &'static str {
+        "CustomDocumentEdge"
+    }
+}
+
+impl NodeType for PresetWebDocument {
+    type Cursor = String;
+
+    fn cursor(&self) -> Self::Cursor {
+        self.name.clone()
+    }
+
+    fn connection_type_name() -> &'static str {
+        "PresetDocumentConnection"
+    }
+
+    fn edge_type_name() -> &'static str {
+        "PresetDocumentEdge"
+    }
+}
+
+#[async_trait]
+pub trait WebDocumentService: Send + Sync {
+    async fn list_custom_web_documents(
+        &self,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<usize>,
+        last: Option<usize>,
+    ) -> Result<Vec<CustomWebDocument>>;
+
+    async fn create_custom_web_document(&self, name: String, url: String) -> Result<ID>;
+    async fn delete_custom_web_document(&self, id: ID) -> Result<()>;
+    async fn list_preset_web_documents(
+        &self,
+        after: Option<String>,
+        before: Option<String>,
+        first: Option<usize>,
+        last: Option<usize>,
+        active: bool,
+    ) -> Result<Vec<PresetWebDocument>>;
+    async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<ID>;
+}