diff --git a/ee/tabby-db/migrations/0036_web_document.down.sql b/ee/tabby-db/migrations/0036_web_document.down.sql new file mode 100644 index 00000000000..cfdbee70bfd --- /dev/null +++ b/ee/tabby-db/migrations/0036_web_document.down.sql @@ -0,0 +1 @@ +DROP TABLE web_documents; diff --git a/ee/tabby-db/migrations/0036_web_document.up.sql b/ee/tabby-db/migrations/0036_web_document.up.sql new file mode 100644 index 00000000000..0a2f0d948f3 --- /dev/null +++ b/ee/tabby-db/migrations/0036_web_document.up.sql @@ -0,0 +1,10 @@ +CREATE TABLE web_documents( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + is_preset BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + CONSTRAINT idx_name UNIQUE(name), + CONSTRAINT idx_url UNIQUE(url) +); diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 6f19779ffb8..a38c2d52929 100644 Binary files a/ee/tabby-db/schema.sqlite and b/ee/tabby-db/schema.sqlite differ diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql index 34a20b3acba..15bead2fad4 100644 --- a/ee/tabby-db/schema/schema.sql +++ b/ee/tabby-db/schema/schema.sql @@ -190,3 +190,13 @@ CREATE TABLE thread_messages( updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), FOREIGN KEY(thread_id) REFERENCES threads(id) ON DELETE CASCADE ); +CREATE TABLE web_documents( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + is_preset BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + CONSTRAINT idx_name UNIQUE(name), + CONSTRAINT idx_url UNIQUE(url) +); diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs index cd239e49e7f..ef6d06d8eb1 100644 --- a/ee/tabby-db/src/lib.rs +++ b/ee/tabby-db/src/lib.rs @@ -22,6 +22,7 @@ use user_completions::UserCompletionDailyStatsDAO; pub use user_events::UserEventDAO; pub use users::UserDAO; pub use web_crawler::WebCrawlerUrlDAO; +pub use web_documents::WebDocumentDAO; pub mod cache; mod email_setting; @@ -41,6 +42,7 @@ mod user_completions; mod user_events; mod users; mod web_crawler; +mod web_documents; use anyhow::Result; use sql_query_builder as sql; diff --git a/ee/tabby-db/src/web_documents.rs b/ee/tabby-db/src/web_documents.rs new file mode 100644 index 00000000000..c361cc485be --- /dev/null +++ b/ee/tabby-db/src/web_documents.rs @@ -0,0 +1,77 @@ +use anyhow::{anyhow, Result}; +use chrono::{DateTime, Utc}; +use sqlx::{prelude::FromRow, query}; +use tabby_db_macros::query_paged_as; + +use crate::DbConn; + +#[allow(unused)] +#[derive(FromRow)] +pub struct WebDocumentDAO { + pub id: i64, + pub name: String, + pub url: String, + pub is_preset: bool, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl DbConn { + pub async fn list_web_documents( + &self, + limit: Option, + skip_id: Option, + backwards: bool, + is_preset: bool, + ) -> Result> { + let condition = Some(format!("is_preset={}", is_preset)); + + let urls = query_paged_as!( + WebDocumentDAO, + "web_documents", + ["id", "name", "url", "is_preset", "created_at" as "created_at!: DateTime", "updated_at" as "updated_at!: DateTime"], + limit, + skip_id, + backwards, + condition + ).fetch_all(&self.pool) + .await?; + + Ok(urls) + } + + pub async fn create_web_document( + &self, + name: String, + url: String, + is_preset: bool, + ) -> Result { + let res = query!( + "INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", + name, + url, + is_preset + ) + .execute(&self.pool) + .await?; + + Ok(res.last_insert_rowid()) + } + + pub async fn deactivate_preset_web_document(&self, name: String) -> Result<()> { + let res = query!("DELETE FROM web_documents WHERE name = ?;", name) + .execute(&self.pool) + .await?; + if res.rows_affected() != 1 { + return Err(anyhow!("No preset web document to deactivate")); + } + Ok(()) + } + + pub async fn delete_web_document(&self, id: i64) -> Result<()> { + query!("DELETE FROM web_documents WHERE id = ?;", id) + .execute(&self.pool) + .await?; + Ok(()) + } +} diff --git a/ee/tabby-schema/graphql/schema.graphql b/ee/tabby-schema/graphql/schema.graphql index 918cf99c869..f1b9b72b5ba 100644 --- a/ee/tabby-schema/graphql/schema.graphql +++ b/ee/tabby-schema/graphql/schema.graphql @@ -99,6 +99,11 @@ input CodeSearchParamsOverrideInput { numToScore: Int } +input CreateCustomDocumentInput { + name: String! + url: String! +} + input CreateIntegrationInput { displayName: String! accessToken: String! @@ -188,6 +193,11 @@ input SecuritySettingInput { disableClientSideTelemetry: Boolean! } +input SetPresetDocumentActiveInput { + name: String! + active: Boolean! +} + input ThreadRunDebugOptionsInput { codeSearchParamsOverride: CodeSearchParamsOverrideInput = null } @@ -238,6 +248,25 @@ type CompletionStats { selects: Int! } +type CustomDocumentConnection { + edges: [CustomDocumentEdge!]! + pageInfo: PageInfo! +} + +type CustomDocumentEdge { + node: CustomWebDocument! + cursor: String! +} + +type CustomWebDocument { + url: String! + name: String! + id: ID! + createdAt: DateTime! + updatedAt: DateTime! + jobInfo: JobInfo! +} + type DiskUsage { filepath: [String!]! "Size in kilobytes." @@ -490,6 +519,9 @@ type Mutation { deleteWebCrawlerUrl(id: ID!): Boolean! "Delete pair of user message and bot response in a thread." deleteThreadMessagePair(threadId: ID!, userMessageId: ID!, assistantMessageId: ID!): Boolean! + createCustomDocument(input: CreateCustomDocumentInput!): ID! + deleteCustomDocument(id: ID!): Boolean! + setPresetDocumentActive(input: SetPresetDocumentActiveInput!): Boolean! } type NetworkSetting { @@ -510,6 +542,25 @@ type PageInfo { endCursor: String } +type PresetDocumentConnection { + edges: [PresetDocumentEdge!]! + pageInfo: PageInfo! +} + +type PresetDocumentEdge { + node: PresetWebDocument! + cursor: String! +} + +type PresetWebDocument { + id: ID! + name: String! + "`updated_at` is only filled when the preset is active." + updatedAt: DateTime + "`job_info` is only filled when the preset is active." + jobInfo: JobInfo +} + type ProvidedRepository { id: ID! integrationId: ID! @@ -581,6 +632,8 @@ type Query { Thread is public within an instance, so no need to check for ownership. """ threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection! + customWebDocuments(after: String, before: String, first: Int, last: Int): CustomDocumentConnection! + presetWebDocuments(after: String, before: String, first: Int, last: Int, isActive: Boolean!): PresetDocumentConnection! } type RefreshTokenResponse { diff --git a/ee/tabby-schema/src/schema/constants.rs b/ee/tabby-schema/src/schema/constants.rs index 66270f823e2..ca2931effdc 100644 --- a/ee/tabby-schema/src/schema/constants.rs +++ b/ee/tabby-schema/src/schema/constants.rs @@ -5,6 +5,8 @@ lazy_static! { pub static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[a-zA-Z][\\w.-]+$").unwrap(); pub static ref USERNAME_REGEX: Regex = Regex::new(r"^[^0-9±!@£$%^&*_+§¡€#¢¶•ªº«\\/<>?:;|=.,]{2,20}$").unwrap(); + pub static ref WEB_DOCUMENT_NAME_REGEX: Regex = + Regex::new(r"^[A-Za-z][A-Za-z0-9]*(\ [A-Za-z0-9]+)*$").unwrap(); } #[cfg(test)] @@ -40,4 +42,23 @@ mod tests { assert_eq!(result, expected, "Failed for name: {}", name); } } + + #[test] + fn test_web_document_name_regex() { + let test_cases = vec![ + ("John", true), // English name + ("Müller", false), // German name + ("abc123", true), + ("Abc 123", true), + (" abc 123", false), + ("abc123*", false), + ("abc123_", false), + ("abc 123", false), // two space + ]; + + for (name, expected) in test_cases { + let result = WEB_DOCUMENT_NAME_REGEX.is_match(name); + assert_eq!(result, expected, "Failed for name: {}", name); + } + } } diff --git a/ee/tabby-schema/src/schema/mod.rs b/ee/tabby-schema/src/schema/mod.rs index c0bcf954787..35411cff312 100644 --- a/ee/tabby-schema/src/schema/mod.rs +++ b/ee/tabby-schema/src/schema/mod.rs @@ -10,6 +10,7 @@ pub mod setting; pub mod thread; pub mod user_event; pub mod web_crawler; +pub mod web_documents; pub mod worker; use std::sync::Arc; @@ -51,10 +52,12 @@ use self::{ }, user_event::{UserEvent, UserEventService}, web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl}, + web_documents::{CreateCustomDocumentInput, CustomWebDocument, WebDocumentService}, }; use crate::{ env, juniper::relay::{self, query_async, Connection}, + web_documents::{PresetWebDocument, SetPresetDocumentActiveInput}, }; pub trait ServiceLocator: Send + Sync { @@ -71,6 +74,7 @@ pub trait ServiceLocator: Send + Sync { fn analytic(&self) -> Arc; fn user_event(&self) -> Arc; fn web_crawler(&self) -> Arc; + fn web_documents(&self) -> Arc; fn thread(&self) -> Arc; } @@ -580,6 +584,50 @@ impl Query { ) .await } + + async fn custom_web_documents( + ctx: &Context, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + query_async( + after, + before, + first, + last, + |after, before, first, last| async move { + ctx.locator + .web_documents() + .list_custom_web_documents(after, before, first, last) + .await + }, + ) + .await + } + async fn preset_web_documents( + ctx: &Context, + after: Option, + before: Option, + first: Option, + last: Option, + is_active: bool, + ) -> Result> { + query_async( + after, + before, + first, + last, + |after, before, first, last| async move { + ctx.locator + .web_documents() + .list_preset_web_documents(after, before, first, last, is_active) + .await + }, + ) + .await + } } #[derive(GraphQLObject)] @@ -959,6 +1007,36 @@ impl Mutation { .await?; Ok(true) } + + async fn create_custom_document(ctx: &Context, input: CreateCustomDocumentInput) -> Result { + input.validate()?; + let id = ctx + .locator + .web_documents() + .create_custom_web_document(input.name, input.url) + .await?; + Ok(id) + } + + async fn delete_custom_document(ctx: &Context, id: ID) -> Result { + ctx.locator + .web_documents() + .delete_custom_web_document(id) + .await?; + Ok(true) + } + + async fn set_preset_document_active( + ctx: &Context, + input: SetPresetDocumentActiveInput, + ) -> Result { + input.validate()?; + ctx.locator + .web_documents() + .set_preset_web_documents_active(input.name, input.active) + .await?; + Ok(true) + } } async fn check_analytic_access(ctx: &Context, users: &[ID]) -> Result<(), CoreError> { diff --git a/ee/tabby-schema/src/schema/web_documents.rs b/ee/tabby-schema/src/schema/web_documents.rs new file mode 100644 index 00000000000..eb958a2561b --- /dev/null +++ b/ee/tabby-schema/src/schema/web_documents.rs @@ -0,0 +1,117 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use juniper::{GraphQLInputObject, GraphQLObject, ID}; +use validator::Validate; + +use crate::{job::JobInfo, juniper::relay::NodeType, Context, Result}; + +#[derive(GraphQLObject)] +#[graphql(context = Context)] +pub struct CustomWebDocument { + pub url: String, + pub name: String, + pub id: ID, + pub created_at: DateTime, + pub updated_at: DateTime, + pub job_info: JobInfo, +} + +#[derive(GraphQLObject)] +#[graphql(context = Context)] +pub struct PresetWebDocument { + pub id: ID, + + pub name: String, + /// `updated_at` is only filled when the preset is active. + pub updated_at: Option>, + /// `job_info` is only filled when the preset is active. + pub job_info: Option, +} + +impl CustomWebDocument { + pub fn source_id(&self) -> String { + Self::format_source_id(&self.id) + } + + pub fn format_source_id(id: &ID) -> String { + format!("web_document:{}", id) + } +} + +#[derive(Validate, GraphQLInputObject)] +pub struct CreateCustomDocumentInput { + #[validate(regex( + code = "name", + path = "*crate::schema::constants::WEB_DOCUMENT_NAME_REGEX", + message = "Invalid document name" + ))] + pub name: String, + #[validate(url(code = "url", message = "Invalid URL"))] + pub url: String, +} + +#[derive(Validate, GraphQLInputObject)] +pub struct SetPresetDocumentActiveInput { + #[validate(regex( + code = "name", + path = "*crate::schema::constants::WEB_DOCUMENT_NAME_REGEX", + message = "Invalid document name" + ))] + pub name: String, + pub active: bool, +} + +impl NodeType for CustomWebDocument { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.id.to_string() + } + + fn connection_type_name() -> &'static str { + "CustomDocumentConnection" + } + + fn edge_type_name() -> &'static str { + "CustomDocumentEdge" + } +} + +impl NodeType for PresetWebDocument { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.id.to_string() + } + + fn connection_type_name() -> &'static str { + "PresetDocumentConnection" + } + + fn edge_type_name() -> &'static str { + "PresetDocumentEdge" + } +} + +#[async_trait] +pub trait WebDocumentService: Send + Sync { + async fn list_custom_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result>; + + async fn create_custom_web_document(&self, name: String, url: String) -> Result; + async fn delete_custom_web_document(&self, id: ID) -> Result<()>; + async fn list_preset_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + is_active: bool, + ) -> Result>; + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<()>; +} diff --git a/ee/tabby-webserver/src/service/mod.rs b/ee/tabby-webserver/src/service/mod.rs index f9e5c10def3..af0bb3c5a59 100644 --- a/ee/tabby-webserver/src/service/mod.rs +++ b/ee/tabby-webserver/src/service/mod.rs @@ -7,11 +7,13 @@ pub mod event_logger; pub mod integration; pub mod job; mod license; +mod preset_web_documents_data; pub mod repository; mod setting; mod thread; mod user_event; pub mod web_crawler; +pub mod web_documents; use std::sync::Arc; @@ -43,6 +45,7 @@ use tabby_schema::{ thread::ThreadService, user_event::UserEventService, web_crawler::WebCrawlerService, + web_documents::WebDocumentService, worker::WorkerService, AsID, AsRowid, CoreError, Result, ServiceLocator, }; @@ -60,6 +63,7 @@ struct ServerContext { user_event: Arc, job: Arc, web_crawler: Arc, + web_documents: Arc, thread: Arc, logger: Arc, @@ -77,6 +81,7 @@ impl ServerContext { repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, answer: Option>, db_conn: DbConn, @@ -105,6 +110,7 @@ impl ServerContext { setting.clone(), )), web_crawler, + web_documents, thread, license, repository, @@ -260,6 +266,10 @@ impl ServiceLocator for ArcServerContext { self.0.web_crawler.clone() } + fn web_documents(&self) -> Arc { + self.0.web_documents.clone() + } + fn thread(&self) -> Arc { self.0.thread.clone() } @@ -271,6 +281,7 @@ pub async fn create_service_locator( repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, answer: Option>, db: DbConn, @@ -283,6 +294,7 @@ pub async fn create_service_locator( repository, integration, web_crawler, + web_documents, job, answer, db, diff --git a/ee/tabby-webserver/src/service/preset_web_documents_data.rs b/ee/tabby-webserver/src/service/preset_web_documents_data.rs new file mode 100644 index 00000000000..02d83fe36c7 --- /dev/null +++ b/ee/tabby-webserver/src/service/preset_web_documents_data.rs @@ -0,0 +1,384 @@ +pub const PRESET_WEB_DOCUMENTS_DATA: &str = r#"[{ "name": "CodeMirror", "crawlerStart": "https://codemirror.net/docs/", "crawlerPrefix": "https://codemirror.net/docs/" }, +{ "name": "React", "crawlerStart": "https://react.dev/reference/", "crawlerPrefix": "https://react.dev/reference/" }, +{ "name": "Tailwind CSS", "crawlerStart": "https://tailwindcss.com/docs/", "crawlerPrefix": "https://tailwindcss.com/docs/" }, +{ "name": "Qwik", "crawlerStart": "https://qwik.builder.io/docs/", "crawlerPrefix": "https://qwik.builder.io/docs/" }, +{ "name": "Stripe", "crawlerStart": "https://stripe.com/docs/api", "crawlerPrefix": "https://stripe.com/docs/api" }, +{ "name": "PostHog", "crawlerStart": "https://posthog.com/docs", "crawlerPrefix": "https://posthog.com/docs" }, +{ "name": "Express", "crawlerStart": "https://expressjs.com/en/5x/api.html", "crawlerPrefix": "https://expressjs.com/en/5x/" }, +{ "name": "Boto3", "crawlerStart": "https://boto3.amazonaws.com/v1/documentation/api/latest/index.html", "crawlerPrefix": "https://boto3.amazonaws.com/v1/documentation/api/latest/" }, +{ "name": "Redux", "crawlerStart": "https://redux.js.org/api/", "crawlerPrefix": "https://redux.js.org/api/" }, +{ "name": "Electron", "crawlerStart": "https://www.electronjs.org/docs/latest/", "crawlerPrefix": "https://www.electronjs.org/docs/latest/" }, +{ "name": "Tauri", "crawlerStart": "https://tauri.app/v1/api/js/", "crawlerPrefix": "https://tauri.app/v1/api/js/" }, +{ "name": "selenium-python", "crawlerStart": "https://selenium-python.readthedocs.io/", "crawlerPrefix": "https://selenium-python.readthedocs.io/" }, +{ "name": "SolidJS", "crawlerStart": "https://www.solidjs.com/docs/", "crawlerPrefix": "https://www.solidjs.com/docs/" }, +{ "name": "Lexical", "crawlerStart": "https://lexical.dev/docs/concepts/editor-state", "crawlerPrefix": "https://lexical.dev/docs/concepts/editor-state" }, +{ "name": "Lexical Packages", "crawlerStart": "https://lexical.dev/docs/packages/lexical", "crawlerPrefix": "https://lexical.dev/docs/packages/lexical" }, +{ "name": "Prisma", "crawlerStart": "https://www.prisma.io/docs", "crawlerPrefix": "https://www.prisma.io/docs" }, +{ "name": "Kubernetes", "crawlerStart": "https://kubernetes.io/docs/", "crawlerPrefix": "https://kubernetes.io/docs/" }, +{ "name": "Sentry-Python", "crawlerStart": "https://docs.sentry.io/platforms/python/", "crawlerPrefix": "https://docs.sentry.io/platforms/python/" }, +{ "name": "Pytorch", "crawlerStart": "https://pytorch.org/docs/stable/", "crawlerPrefix": "https://pytorch.org/docs/stable/" }, +{ "name": "Transformers", "crawlerStart": "https://huggingface.co/docs/transformers/", "crawlerPrefix": "https://huggingface.co/docs/transformers/" }, +{ "name": "Hugging-Face-Transformers", "crawlerStart": "https://huggingface.co/docs/transformers/", "crawlerPrefix": "https://huggingface.co/docs/transformers/" }, +{ "name": "Langchain", "crawlerStart": "https://python.langchain.com/docs/", "crawlerPrefix": "https://python.langchain.com/docs/" }, +{ "name": "Vue", "crawlerStart": "https://vuejs.org/guide/", "crawlerPrefix": "https://vuejs.org/guide/" }, +{ "name": "Angular", "crawlerStart": "https://angular.io/docs", "crawlerPrefix": "https://angular.io/docs" }, +{ "name": "Astro", "crawlerStart": "https://docs.astro.build/en/", "crawlerPrefix": "https://docs.astro.build/en/" }, +{ "name": "Elixir", "crawlerStart": "https://elixir-lang.org/docs.html", "crawlerPrefix": "https://elixir-lang.org/docs.html" }, +{ "name": "Svelte", "crawlerStart": "https://svelte.dev/docs", "crawlerPrefix": "https://svelte.dev/docs" }, +{ "name": "SvelteKit", "crawlerStart": "https://kit.svelte.dev/docs/introduction", "crawlerPrefix": "https://kit.svelte.dev/docs" }, +{ "name": "PineconeDB", "crawlerStart": "https://docs.pinecone.io/docs/", "crawlerPrefix": "https://docs.pinecone.io/docs/" }, +{ "name": "Redis", "crawlerStart": "https://redis.io/docs/", "crawlerPrefix": "https://redis.io/docs/" }, +{ "name": "KeyDB", "crawlerStart": "https://docs.keydb.dev/docs/", "crawlerPrefix": "https://docs.keydb.dev/docs/" }, +{ "name": "Yugabyte", "crawlerStart": "https://docs.yugabyte.com/", "crawlerPrefix": "https://docs.yugabyte.com/" }, +{ "name": "Tensorflow", "crawlerStart": "https://www.tensorflow.org/api_docs/python/tf/", "crawlerPrefix": "https://www.tensorflow.org/api_docs/python/tf/" }, +{ "name": "Tmux", "crawlerStart": "https://tmuxguide.readthedocs.io/en/latest/tmux/tmux.html", "crawlerPrefix": "https://tmuxguide.readthedocs.io/en/latest/tmux/tmux.html" }, +{ "name": "FFmpeg", "crawlerStart": "https://ffmpeg.org/ffmpeg.html", "crawlerPrefix": "https://ffmpeg.org/ffmpeg.html" }, +{ "name": "Flutter", "crawlerStart": "https://docs.flutter.dev/", "crawlerPrefix": "https://docs.flutter.dev/" }, +{ "name": "Pandas", "crawlerStart": "https://pandas.pydata.org/docs/", "crawlerPrefix": "https://pandas.pydata.org/docs/" }, +{ "name": "Julia", "crawlerStart": "https://docs.julialang.org/en/v1/", "crawlerPrefix": "https://docs.julialang.org/en/v1/" }, +{ "name": "Django", "crawlerStart": "https://docs.djangoproject.com/en/4.2/", "crawlerPrefix": "https://docs.djangoproject.com/en/4.2/" }, +{ "name": "Flask", "crawlerStart": "https://flask.palletsprojects.com/en/2.3.x/", "crawlerPrefix": "https://flask.palletsprojects.com/en/2.3.x/" }, +{ "name": "Keras", "crawlerStart": "https://keras.io/api/", "crawlerPrefix": "https://keras.io/api/" }, +{ "name": "Langchain-JS", "crawlerStart": "https://js.langchain.com/docs/", "crawlerPrefix": "https://js.langchain.com/docs/" }, +{ "name": "requests", "crawlerStart": "https://requests.readthedocs.io/en/latest/api/", "crawlerPrefix": "https://requests.readthedocs.io/en/latest/api/" }, +{ "name": "ggplot2", "crawlerStart": "https://ggplot2.tidyverse.org/reference/", "crawlerPrefix": "https://ggplot2.tidyverse.org/reference/" }, +{ "name": "RubyOnRails", "crawlerStart": "https://api.rubyonrails.org/", "crawlerPrefix": "https://api.rubyonrails.org/" }, +{ "name": "Spark", "crawlerStart": "https://spark.apache.org/docs/latest/", "crawlerPrefix": "https://spark.apache.org/docs/latest/" }, +{ "name": "PySpark", "crawlerStart": "https://spark.apache.org/docs/latest/api/python/", "crawlerPrefix": "https://spark.apache.org/docs/latest/api/python/" }, +{ "name": "LanguageServerProtocol", "crawlerStart": "https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/", "crawlerPrefix": "https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/" }, +{ "name": "axios", "crawlerStart": "https://axios-http.com/docs/api_intro", "crawlerPrefix": "https://axios-http.com/docs/api_intro" }, +{ "name": "Jquery", "crawlerStart": "https://api.jqueryui.com/1.12/", "crawlerPrefix": "https://api.jqueryui.com/1.12/" }, +{ "name": "lodash", "crawlerStart": "https://lodash.com/docs/", "crawlerPrefix": "https://lodash.com/docs/" }, +{ "name": "java.swing", "crawlerStart": "https://docs.oracle.com/javase%2F7%2Fdocs%2Fapi%2F%2F/javax/swing/package-summary.html", "crawlerPrefix": "https://docs.oracle.com/javase%2F7%2Fdocs%2Fapi%2F%2F/javax/swing/" }, +{ "name": "Matplotlib", "crawlerStart": "https://matplotlib.org/stable/api/", "crawlerPrefix": "https://matplotlib.org/stable/api/" }, +{ "name": "NodeJS", "crawlerStart": "https://nodejs.org/api/", "crawlerPrefix": "https://nodejs.org/api/" }, +{ "name": "Tomcat", "crawlerStart": "https://tomcat.apache.org/tomcat-8.5-doc/", "crawlerPrefix": "https://tomcat.apache.org/tomcat-8.5-doc/" }, +{ "name": "redb", "crawlerStart": "https://docs.rs/redb/latest/redb/", "crawlerPrefix": "https://docs.rs/redb/latest/redb/" }, +{ "name": "OpenAI", "crawlerStart": "https://platform.openai.com/docs/", "crawlerPrefix": "https://platform.openai.com/docs/" }, +{ "name": "Notion", "crawlerStart": "https://developers.notion.com/reference/", "crawlerPrefix": "https://developers.notion.com/reference/" }, +{ "name": "GCP CLI", "crawlerStart": "https://cloud.google.com/sdk/docs", "crawlerPrefix": "https://cloud.google.com/sdk/docs" }, +{ "name": "AWS CLI", "crawlerStart": "https://docs.aws.amazon.com/cli/latest/reference/", "crawlerPrefix": "https://docs.aws.amazon.com/cli/latest/reference/" }, +{ "name": "React Native", "crawlerStart": "https://reactnative.dev/docs/appstate", "crawlerPrefix": "https://reactnative.dev/docs/" }, +{ "name": "Lottie", "crawlerStart": "http://airbnb.io/lottie/#/README", "crawlerPrefix": "http://airbnb.io/lottie/#/" }, +{ "name": "Expo", "crawlerStart": "https://docs.expo.dev/", "crawlerPrefix": "https://docs.expo.dev/" }, +{ "name": "Goreplay", "crawlerStart": "https://github.com/buger/goreplay/wiki", "crawlerPrefix": "https://github.com/buger/goreplay/wiki" }, +{ "name": "NextJS", "crawlerStart": "https://nextjs.org/docs", "crawlerPrefix": "https://nextjs.org/docs" }, +{ "name": "Jest", "crawlerStart": "https://jestjs.io/docs/getting-started", "crawlerPrefix": "https://jestjs.io/docs/getting-started" }, +{ "name": "Vite", "crawlerStart": "https://vitejs.dev/guide/", "crawlerPrefix": "https://vitejs.dev/guide/" }, +{ "name": "Webpack", "crawlerStart": "https://webpack.js.org/concepts/", "crawlerPrefix": "https://webpack.js.org/concepts/" }, +{ "name": "ESBuild", "crawlerStart": "https://esbuild.github.io/api/", "crawlerPrefix": "https://esbuild.github.io/api/" }, +{ "name": "D3", "crawlerStart": "https://d3js.org/getting-started", "crawlerPrefix": "https://d3js.org/" }, +{ "name": "Deno API", "crawlerStart": "https://deno.land/api@v1.35.0", "crawlerPrefix": "https://deno.land/api@v1.35.0" }, +{ "name": "Deno", "crawlerStart": "https://deno.land/manual@v1.35.0/introduction", "crawlerPrefix": "https://deno.land/manual@v1.35.0/" }, +{ "name": "Puppeteer", "crawlerStart": "https://pptr.dev/", "crawlerPrefix": "https://pptr.dev/" }, +{ "name": "Laravel", "crawlerStart": "https://laravel.com/docs/10.x", "crawlerPrefix": "https://laravel.com/docs/10.x" }, +{ "name": "FontAwesome", "crawlerStart": "https://fontawesome.com/docs/web/", "crawlerPrefix": "https://fontawesome.com/docs/web/" }, +{ "name": "OpenCV", "crawlerStart": "https://docs.opencv.org/4.x/", "crawlerPrefix": "https://docs.opencv.org/4.x/" }, +{ "name": "Nginx", "crawlerStart": "http://nginx.org/en/docs/", "crawlerPrefix": "http://nginx.org/en/docs/" }, +{ "name": "Hugo", "crawlerStart": "https://gohugo.io/documentation/", "crawlerPrefix": "https://gohugo.io/documentation/" }, +{ "name": "MongoDB", "crawlerStart": "https://www.mongodb.com/docs/manual/", "crawlerPrefix": "https://www.mongodb.com/docs/manual/" }, +{ "name": "Phoenix", "crawlerStart": "https://hexdocs.pm/phoenix/", "crawlerPrefix": "https://hexdocs.pm/phoenix/" }, +{ "name": "Spring Boot", "crawlerStart": "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/", "crawlerPrefix": "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/" }, +{ "name": "FastAPI", "crawlerStart": "https://fastapi.tiangolo.com/tutorial/", "crawlerPrefix": "https://fastapi.tiangolo.com/tutorial/" }, +{ "name": "Nuxt", "crawlerStart": "https://nuxt.com/docs", "crawlerPrefix": "https://nuxt.com/docs" }, +{ "name": "Postgres", "crawlerStart": "https://www.postgresql.org/docs/current/index.html", "crawlerPrefix": "https://www.postgresql.org/docs/current/" }, +{ "name": "Streamlit", "crawlerStart": "https://docs.streamlit.io/", "crawlerPrefix": "https://docs.streamlit.io/" }, +{ "name": "AngularJS", "crawlerStart": "https://docs.angularjs.org/guide/", "crawlerPrefix": "https://docs.angularjs.org/guide/" }, +{ "name": "Fontello", "crawlerStart": "https://github.com/fontello/fontello/wiki/", "crawlerPrefix": "https://github.com/fontello/fontello/wiki/" }, +{ "name": "Go", "crawlerStart": "https://go.dev/doc", "crawlerPrefix": "https://go.dev/doc" }, +{ "name": "Maven", "crawlerStart": "https://maven.apache.org/guides/", "crawlerPrefix": "https://maven.apache.org/guides/" }, +{ "name": "Vagrant", "crawlerStart": "https://developer.hashicorp.com/vagrant/docs", "crawlerPrefix": "https://developer.hashicorp.com/vagrant/docs" }, +{ "name": "ESLint", "crawlerStart": "https://eslint.org/docs/latest/", "crawlerPrefix": "https://eslint.org/docs/latest/" }, +{ "name": "Swagger", "crawlerStart": "https://swagger.io/docs/", "crawlerPrefix": "https://swagger.io/docs/" }, +{ "name": "JUnit 5", "crawlerStart": "https://junit.org/junit5/docs/current/user-guide/", "crawlerPrefix": "https://junit.org/junit5/docs/current/user-guide/" }, +{ "name": "NUnit", "crawlerStart": "https://docs.nunit.org/", "crawlerPrefix": "https://docs.nunit.org/articles/" }, +{ "name": "TestNG", "crawlerStart": "https://testng.org/doc/documentation-main.html", "crawlerPrefix": "https://testng.org/doc/" }, +{ "name": "Jasmine", "crawlerStart": "https://jasmine.github.io/pages/docs_home.html/", "crawlerPrefix": "https://jasmine.github.io/api/" }, +{ "name": "Mocha", "crawlerStart": "https://mochajs.org/api/", "crawlerPrefix": "https://mochajs.org/api/" }, +{ "name": "Cypress", "crawlerStart": "https://docs.cypress.io/guides/overview/why-cypress", "crawlerPrefix": "https://docs.cypress.io/guides/" }, +{ "name": "TestComplete", "crawlerStart": "https://support.smartbear.com/testcomplete/docs/", "crawlerPrefix": "https://support.smartbear.com/testcomplete/docs/" }, +{ "name": "Mockito", "crawlerStart": "https://javadoc.io/doc/org.mockito/mockito-core/latest/index.html", "crawlerPrefix": "https://javadoc.io/doc/org.mockito/mockito-core/latest/" }, +{ "name": "PHPUnit", "crawlerStart": "https://docs.phpunit.de/en/10.2/", "crawlerPrefix": "https://docs.phpunit.de/en/10.2/" }, +{ "name": "JTest", "crawlerStart": "https://docs.parasoft.com/display/JTEST20231/About+this+User+Guide", "crawlerPrefix": "https://docs.parasoft.com/display/JTEST20231/" }, +{ "name": "GoogleTest", "crawlerStart": "https://google.github.io/googletest/", "crawlerPrefix": "https://google.github.io/googletest/" }, +{ "name": "Material UI", "crawlerStart": "https://mui.com/material-ui/getting-started/", "crawlerPrefix": "https://mui.com/material-ui/" }, +{ "name": "Chakra UI", "crawlerStart": "https://chakra-ui.com/getting-started", "crawlerPrefix": "https://chakra-ui.com/docs/" }, +{ "name": "Ant Design", "crawlerStart": "https://ant.design/docs/react/introduce", "crawlerPrefix": "https://ant.design/docs/react/" }, +{ "name": "React Suite", "crawlerStart": "https://rsuitejs.com/guide/introduction/", "crawlerPrefix": "https://rsuitejs.com/guide/" }, +{ "name": "React Bootstrap", "crawlerStart": "https://react-bootstrap.netlify.app/docs/getting-started/introduction", "crawlerPrefix": "https://react-bootstrap.netlify.app/docs/" }, +{ "name": "Semantic UI", "crawlerStart": "https://semantic-ui.com/behaviors/api.html", "crawlerPrefix": "https://semantic-ui.com/behaviors/api.html#" }, +{ "name": "Mantine React Table", "crawlerStart": "https://www.mantine-react-table.com/docs/getting-started", "crawlerPrefix": "https://www.mantine-react-table.com/docs/" }, +{ "name": "Mantine", "crawlerStart": "https://mantine.dev/pages/getting-started/", "crawlerPrefix": "https://mantine.dev/pages/" }, +{ "name": "Blueprint", "crawlerStart": "https://blueprintjs.com/docs/", "crawlerPrefix": "https://blueprintjs.com/docs/" }, +{ "name": "NextUI", "crawlerStart": "https://nextui.org/docs", "crawlerPrefix": "https://nextui.org/docs" }, +{ "name": "PrimeReact", "crawlerStart": "https://primereact.org/", "crawlerPrefix": "https://primereact.org/" }, +{ "name": "Grommet", "crawlerStart": "https://v2.grommet.io/docs", "crawlerPrefix": "https://v2.grommet.io/" }, +{ "name": "Onsen UI", "crawlerStart": "https://onsen.io/v2/api/css.html", "crawlerPrefix": "https://onsen.io/v2/api/" }, +{ "name": "Quasar", "crawlerStart": "hhttps://quasar.dev/docs", "crawlerPrefix": "https://quasar.dev/" }, +{ "name": "Vuetify", "crawlerStart": "https://vuetifyjs.com/en/getting-started/installation/", "crawlerPrefix": "https://vuetifyjs.com/en/" }, +{ "name": "Bootstrap Vue", "crawlerStart": "https://bootstrap-vue.org/docs", "crawlerPrefix": "https://bootstrap-vue.org/docs" }, +{ "name": "Element Plus", "crawlerStart": "https://element-plus.org/#/en-US", "crawlerPrefix": "https://element-plus.org/#/en-US" }, +{ "name": "Keen UI", "crawlerStart": "https://josephuspaye.github.io/Keen-UI/#/ui-alert", "crawlerPrefix": "https://josephuspaye.github.io/Keen-UI/#/" }, +{ "name": "Equal UI", "crawlerStart": "https://equal-ui.github.io/Equal/", "crawlerPrefix": "https://equal-ui.github.io/Equal/" }, +{ "name": "Anti-Design Vue", "crawlerStart": "https://antdv.com/components/overview/", "crawlerPrefix": "https://antdv.com/components/" }, +{ "name": "Prime Vue", "crawlerStart": "https://primevue.org/", "crawlerPrefix": "https://primevue.org/" }, +{ "name": "Fish UI", "crawlerStart": "https://myliang.github.io/fish-ui/#/components/index", "crawlerPrefix": "https://myliang.github.io/fish-ui/#/components/" }, +{ "name": "CoreUI Vue", "crawlerStart": "https://coreui.io/vue/docs/", "crawlerPrefix": "https://coreui.io/vue/docs/" }, +{ "name": "Vuesax", "crawlerStart": "https://vuesax.com/docs/", "crawlerPrefix": "https://vuesax.com/docs/" }, +{ "name": "Vue Material", "crawlerStart": "https://www.creative-tim.com/vuematerial/getting-started", "crawlerPrefix": "https://www.creative-tim.com/vuematerial/" }, +{ "name": "Buefy", "crawlerStart": "https://buefy.org/documentation/", "crawlerPrefix": "https://buefy.org/documentation/" }, +{ "name": "Meteor", "crawlerStart": "https://docs.meteor.com/", "crawlerPrefix": "https://docs.meteor.com/" }, +{ "name": "Koajs", "crawlerStart": "https://koajs.com/#", "crawlerPrefix": "https://koajs.com/#" }, +{ "name": "Browserify", "crawlerStart": "https://github.com/browserify/browserify#", "crawlerPrefix": "https://github.com/browserify/browserify#" }, +{ "name": "Underscore", "crawlerStart": "https://underscorejs.org/#", "crawlerPrefix": "https://underscorejs.org/#" }, +{ "name": "Mongoose", "crawlerStart": "https://mongoosejs.com/docs/guide.html", "crawlerPrefix": "https://mongoosejs.com/docs/" }, +{ "name": "Cors", "crawlerStart": "https://expressjs.com/en/resources/middleware/cors.html", "crawlerPrefix": "https://expressjs.com/en/resources/middleware/cors.html" }, +{ "name": "Dotenv", "crawlerStart": "https://github.com/motdotla/dotenv", "crawlerPrefix": "https://github.com/motdotla/dotenv" }, +{ "name": "Gulp", "crawlerStart": "https://gulpjs.com/docs/en/getting-started/quick-start", "crawlerPrefix": "https://gulpjs.com/docs/en/" }, +{ "name": "Async", "crawlerStart": "https://caolan.github.io/async/v3/docs.html", "crawlerPrefix": "https://caolan.github.io/async/v3/docs.html#" }, +{ "name": "Moment JS", "crawlerStart": "https://momentjs.com/docs/", "crawlerPrefix": "https://momentjs.com/docs/#/" }, +{ "name": "Cheerio", "crawlerStart": "https://cheerio.js.org/docs/intro", "crawlerPrefix": "https://cheerio.js.org/docs/" }, +{ "name": "Nodemailer", "crawlerStart": "https://nodemailer.com/", "crawlerPrefix": "https://nodemailer.com/" }, +{ "name": "Selenium", "crawlerStart": "https://www.selenium.dev/documentation/", "crawlerPrefix": "https://www.selenium.dev/documentation/" }, +{ "name": "Katalon", "crawlerStart": "https://docs.katalon.com/", "crawlerPrefix": "https://docs.katalon.com/docs/" }, +{ "name": "Katalon TestOps API", "crawlerStart": "https://developers.katalon.com/reference/api-reference", "crawlerPrefix": "https://developers.katalon.com/reference/" }, +{ "name": "Appium", "crawlerStart": "http://appium.io/docs/en/2.0/", "crawlerPrefix": "http://appium.io/docs/en/2.0/" }, +{ "name": "Ranorex", "crawlerStart": "https://support.ranorex.com/help/latest/ranorex-studio-fundamentals/", "crawlerPrefix": "https://support.ranorex.com/help/latest/" }, +{ "name": "Cucumber", "crawlerStart": "https://cucumber.io/docs/cucumber/", "crawlerPrefix": "https://cucumber.io/docs/cucumber/" }, +{ "name": "SoapUI", "crawlerStart": "https://www.soapui.org/docs/", "crawlerPrefix": "https://www.soapui.org/docs/" }, +{ "name": "ReadyAPI", "crawlerStart": "https://support.smartbear.com/readyapi/docs/", "crawlerPrefix": "https://support.smartbear.com/readyapi/docs/" }, +{ "name": "Watir", "crawlerStart": "http://watir.com/guides/", "crawlerPrefix": "http://watir.com/guides/" }, +{ "name": "BrowserStack", "crawlerStart": "https://www.browserstack.com/docs", "crawlerPrefix": "https://www.browserstack.com/docs/automate/" }, +{ "name": "TestProject", "crawlerStart": "https://docs.testproject.io/testproject-sdk/overview", "crawlerPrefix": "https://docs.testproject.io/testproject-sdk/" }, +{ "name": "Apache JMeter", "crawlerStart": "https://jmeter.apache.org/usermanual/index.html", "crawlerPrefix": "https://jmeter.apache.org/usermanual/" }, +{ "name": "SauceLabs", "crawlerStart": "https://docs.saucelabs.com/", "crawlerPrefix": "https://docs.saucelabs.com/" }, +{ "name": "Playwright", "crawlerStart": "https://playwright.dev/docs/intro", "crawlerPrefix": "https://playwright.dev/docs/" }, +{ "name": "Espresso", "crawlerStart": "https://developer.android.com/training/testing/espresso/", "crawlerPrefix": "https://developer.android.com/training/testing/espresso/" }, +{ "name": "EarlGrey", "crawlerStart": "https://google.github.io/EarlGrey/", "crawlerPrefix": "https://google.github.io/EarlGrey/" }, +{ "name": "Detox Android", "crawlerStart": "https://wix.github.io/Detox/docs/introduction/getting-started/", "crawlerPrefix": "https://wix.github.io/Detox/docs/" }, +{ "name": "Jenkins", "crawlerStart": "https://www.jenkins.io/doc/", "crawlerPrefix": "https://www.jenkins.io/doc/" }, +{ "name": "Bamboo", "crawlerStart": "https://docs.atlassian.com/bamboo-specs/8.1.12/", "crawlerPrefix": "https://docs.atlassian.com/bamboo-specs/8.1.12/" }, +{ "name": "TeamCity", "crawlerStart": "https://www.jetbrains.com/help/teamcity/teamcity-documentation.html", "crawlerPrefix": "https://www.jetbrains.com/help/teamcity/" }, +{ "name": "Azure Pipelines", "crawlerStart": "https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops", "crawlerPrefix": "https://docs.microsoft.com/en-us/azure/devops/pipelines/" }, +{ "name": "CircleCI", "crawlerStart": "https://circleci.com/docs/", "crawlerPrefix": "https://circleci.com/docs/" }, +{ "name": "Bitbucket Pipelines", "crawlerStart": "https://support.atlassian.com/bitbucket-cloud/docs/get-started-with-bitbucket-pipelines/", "crawlerPrefix": "https://support.atlassian.com/bitbucket-cloud/docs/" }, +{ "name": "Travis CI", "crawlerStart": "https://docs.travis-ci.com/", "crawlerPrefix": "https://docs.travis-ci.com/user/" }, +{ "name": "GitHub Actions", "crawlerStart": "https://docs.github.com/en/actions", "crawlerPrefix": "https://docs.github.com/en/actions" }, +{ "name": "GitLab CI", "crawlerStart": "https://docs.gitlab.com/ee/ci/", "crawlerPrefix": "https://docs.gitlab.com/ee/ci/" }, +{ "name": "AWS CodePipeline CI", "crawlerStart": "https://docs.aws.amazon.com/codepipeline/latest/userguide/welcome.html", "crawlerPrefix": "https://docs.aws.amazon.com/codepipeline/latest/userguide/" }, +{ "name": "Drone", "crawlerStart": "https://docs.drone.io/", "crawlerPrefix": "https://docs.drone.io/" }, +{ "name": "Java", "crawlerStart": "https://docs.oracle.com/javase/8/docs/api/", "crawlerPrefix": "https://docs.oracle.com/javase/8/docs/api/" }, +{ "name": "Gauge", "crawlerStart": "https://docs.gauge.org/index.html?os=windows&language=javascript&ide=vscode", "crawlerPrefix": "https://docs.gauge.org/" }, +{ "name": "JBehave", "crawlerStart": "http://jbehave.org/reference/stable/", "crawlerPrefix": "http://jbehave.org/reference/stable/" }, +{ "name": "JUnit 4", "crawlerStart": "https://junit.org/junit4/", "crawlerPrefix": "https://junit.org/junit4/" }, +{ "name": "Selenide", "crawlerStart": "https://selenide.gitbooks.io/user-guide/content/en/", "crawlerPrefix": "https://selenide.gitbooks.io/user-guide/content/en/" }, +{ "name": "Serenity", "crawlerStart": "https://serenity-bdd.github.io/", "crawlerPrefix": "https://serenity-bdd.github.io/docs/" }, +{ "name": "CodeceptJS", "crawlerStart": "https://codecept.io/basics/", "crawlerPrefix": "https://codecept.io/" }, +{ "name": "Intern", "crawlerStart": "https://theintern.io/docs.html#Intern/4/docs/README.md", "crawlerPrefix": "https://theintern.io/docs.html#Intern/4/docs/" }, +{ "name": "Nightwatch", "crawlerStart": "https://nightwatchjs.org/guide/", "crawlerPrefix": "https://nightwatchjs.org/guide/" }, +{ "name": "Protractor", "crawlerStart": "https://www.protractortest.org/#/", "crawlerPrefix": "https://www.protractortest.org/#/" }, +{ "name": "TestCafe", "crawlerStart": "https://testcafe.io/documentation/", "crawlerPrefix": "https://testcafe.io/documentation/" }, +{ "name": "WebdriverIO", "crawlerStart": "https://webdriver.io/docs/gettingstarted", "crawlerPrefix": "https://webdriver.io/docs/" }, +{ "name": "Passport", "crawlerStart": "http://www.passportjs.org/docs/", "crawlerPrefix": "http://www.passportjs.org/" }, +{ "name": "Socket.IO", "crawlerStart": "https://socket.io/docs/v4", "crawlerPrefix": "https://socket.io/docs/v4" }, +{ "name": "PM2", "crawlerStart": "https://pm2.keymetrics.io/docs/usage/quick-start/", "crawlerPrefix": "https://pm2.keymetrics.io/docs/usage/" }, +{ "name": "NestJS", "crawlerStart": "https://docs.nestjs.com/", "crawlerPrefix": "https://docs.nestjs.com/" }, +{ "name": "C#", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/csharp/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/csharp/" }, +{ "name": "SpecFlow", "crawlerStart": "https://docs.specflow.org/en/latest/", "crawlerPrefix": "https://docs.specflow.org/projects/" }, +{ "name": "XUnit", "crawlerStart": "https://xunit.net/#documentation", "crawlerPrefix": "https://xunit.net/docs/" }, +{ "name": "PHP", "crawlerStart": "https://www.php.net/manual/en/", "crawlerPrefix": "https://www.php.net/manual/en/" }, +{ "name": "Behat", "crawlerStart": "https://docs.behat.org/en/latest/guides.html", "crawlerPrefix": "https://docs.behat.org/en/latest/" }, +{ "name": "Codeception", "crawlerStart": "https://codeception.com/docs/GettingStarted", "crawlerPrefix": "https://codeception.com/docs/" }, +{ "name": "Symfony", "crawlerStart": "https://symfony.com/doc/current/index.html", "crawlerPrefix": "https://symfony.com/doc/current/" }, +{ "name": "CodeIgniter", "crawlerStart": "https://codeigniter.com/user_guide/", "crawlerPrefix": "https://codeigniter.com/user_guide/" }, +{ "name": "Yii", "crawlerStart": "https://www.yiiframework.com/doc/guide/2.0/en", "crawlerPrefix": "https://www.yiiframework.com/doc/" }, +{ "name": "Laminas", "crawlerStart": "https://docs.laminas.dev/", "crawlerPrefix": "https://docs.laminas.dev/" }, +{ "name": "Phalcon", "crawlerStart": "https://docs.phalcon.io/5.0/en/introduction", "crawlerPrefix": "https://docs.phalcon.io/5.0/en/" }, +{ "name": "Slim", "crawlerStart": "http://www.slimframework.com/docs/v4/", "crawlerPrefix": "http://www.slimframework.com/docs/v4/" }, +{ "name": "CakePHP", "crawlerStart": "https://book.cakephp.org/4/en/index.html", "crawlerPrefix": "https://book.cakephp.org/4/en/" }, +{ "name": "CakePHP API", "crawlerStart": "https://api.cakephp.org/4.4/", "crawlerPrefix": "https://api.cakephp.org/4.4/" }, +{ "name": "FuelPHP", "crawlerStart": "https://fuelphp.com/docs/", "crawlerPrefix": "https://fuelphp.com/docs/" }, +{ "name": "PHPixie", "crawlerStart": "https://phpixie.com/docs.html/", "crawlerPrefix": "https://phpixie.com/" }, +{ "name": "Python 3", "crawlerStart": "https://docs.python.org/3/", "crawlerPrefix": "https://docs.python.org/3/" }, +{ "name": "Behave", "crawlerStart": "https://behave.readthedocs.io/en/latest/", "crawlerPrefix": "https://behave.readthedocs.io/en/latest/" }, +{ "name": "Lettuce", "crawlerStart": "https://lettuce.readthedocs.io/en/latest/", "crawlerPrefix": "https://lettuce.readthedocs.io/en/latest/" }, +{ "name": "Robot Framework", "crawlerStart": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#", "crawlerPrefix": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#" }, +{ "name": "Pytest", "crawlerStart": "https://docs.pytest.org/en/latest/", "crawlerPrefix": "https://docs.pytest.org/en/latest/" }, +{ "name": "Python 2", "crawlerStart": "https://docs.python.org/2/", "crawlerPrefix": "https://docs.python.org/2/" }, +{ "name": "NumPy", "crawlerStart": "https://numpy.org/doc/stable/", "crawlerPrefix": "https://numpy.org/doc/stable/" }, +{ "name": "SciPy", "crawlerStart": "https://docs.scipy.org/doc/scipy/reference/", "crawlerPrefix": "https://docs.scipy.org/doc/scipy/reference/" }, +{ "name": "Scikit-learn", "crawlerStart": "https://scikit-learn.org/stable/user_guide.html", "crawlerPrefix": "https://scikit-learn.org/stable/" }, +{ "name": "Seaborn", "crawlerStart": "https://seaborn.pydata.org/api.html", "crawlerPrefix": "https://seaborn.pydata.org/" }, +{ "name": "Bokeh", "crawlerStart": "https://docs.bokeh.org/en/latest/index.html", "crawlerPrefix": "https://docs.bokeh.org/en/latest/docs/" }, +{ "name": "BeautifulSoup", "crawlerStart": "https://www.crummy.com/software/BeautifulSoup/bs4/doc/", "crawlerPrefix": "https://www.crummy.com/software/BeautifulSoup/bs4/doc/" }, +{ "name": "NLTK", "crawlerStart": "https://www.nltk.org/", "crawlerPrefix": "https://www.nltk.org/" }, +{ "name": "Spacy", "crawlerStart": "https://spacy.io/api", "crawlerPrefix": "https://spacy.io/api" }, +{ "name": "Pygame", "crawlerStart": "https://www.pygame.org/docs/", "crawlerPrefix": "https://www.pygame.org/docs/" }, +{ "name": "Spring Framework", "crawlerStart": "https://docs.spring.io/spring-framework/reference/", "crawlerPrefix": "https://docs.spring.io/spring-framework/reference/" }, +{ "name": "Vue 2", "crawlerStart": "https://v2.vuejs.org/v2/", "crawlerPrefix": "https://vuejs.org/v2/" }, +{ "name": "Sinatra", "crawlerStart": "http://sinatrarb.com/documentation.html", "crawlerPrefix": "http://sinatrarb.com/" }, +{ "name": "ASP.NET", "crawlerStart": "https://learn.microsoft.com/en-us/aspnet/core/?view=aspnetcore-7.0", "crawlerPrefix": "https://learn.microsoft.com/en-us/aspnet/core/" }, +{ "name": "Pubnub", "crawlerStart": "https://www.pubnub.com/docs", "crawlerPrefix": "https://www.pubnub.com/docs" }, +{ "name": "Twilio", "crawlerStart": "https://www.twilio.com/docs/quickstart", "crawlerPrefix": "https://www.twilio.com/docs/" }, +{ "name": "Bandwidth", "crawlerStart": "https://dev.bandwidth.com/docs/", "crawlerPrefix": "https://dev.bandwidth.com/docs/" }, +{ "name": "HubSpot Operations Hub", "crawlerStart": "https://developers.hubspot.com/docs/api/overview", "crawlerPrefix": "https://developers.hubspot.com/docs/api/" }, +{ "name": "Podium", "crawlerStart": "https://docs.podium.com/docs", "crawlerPrefix": "https://docs.podium.com/" }, +{ "name": "Whereby", "crawlerStart": "https://docs.whereby.com//", "crawlerPrefix": "https://docs.whereby.com/" }, +{ "name": "Plivo", "crawlerStart": "https://www.plivo.com/docs/", "crawlerPrefix": "https://www.plivo.com/docs/" }, +{ "name": "Dotdigital", "crawlerStart": "https://developer.dotdigital.com/docs", "crawlerPrefix": "https://developer.dotdigital.com/docs" }, +{ "name": "Vonage Communications APIs", "crawlerStart": "https://developer.vonage.com/en/documentation", "crawlerPrefix": "https://developer.vonage.com/en/" }, +{ "name": "Dailyco", "crawlerStart": "https://docs.daily.co/", "crawlerPrefix": "https://docs.daily.co/" }, +{ "name": "Zoom", "crawlerStart": "https://developers.zoom.us/docs/api/", "crawlerPrefix": "https://developers.zoom.us/docs/api/" }, +{ "name": "Webex", "crawlerStart": "https://developer.webex.com/docs", "crawlerPrefix": "https://developer.webex.com/docs" }, +{ "name": "Microsoft Teams", "crawlerStart": "https://learn.microsoft.com/en-us/microsoftteams/platform/", "crawlerPrefix": "https://learn.microsoft.com/en-us/microsoftteams/platform/" }, +{ "name": "Office Addin", "crawlerStart": "https://learn.microsoft.com/en-us/office/dev/add-ins/", "crawlerPrefix": "https://learn.microsoft.com/en-us/office/dev/add-ins/" }, +{ "name": "Steam", "crawlerStart": "https://partner.steamgames.com/doc/sdk/", "crawlerPrefix": "https://partner.steamgames.com/doc/sdk/" }, +{ "name": "Epic Games", "crawlerStart": "https://dev.epicgames.com/docs", "crawlerPrefix": "https://dev.epicgames.com/docs" }, +{ "name": "Unreal Engine", "crawlerStart": "https://docs.unrealengine.com/5.2/en-US/", "crawlerPrefix": "https://docs.unrealengine.com/5.2/en-US/" }, +{ "name": "Splunk", "crawlerStart": "https://docs.splunk.com/Documentation", "crawlerPrefix": "https://docs.splunk.com/Documentation" }, +{ "name": "Elasticsearch", "crawlerStart": "https://www.elastic.co/guide/en/enterprise-search/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/enterprise-search/current/" }, +{ "name": "Logstash", "crawlerStart": "https://www.elastic.co/guide/en/logstash/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/logstash/current" }, +{ "name": "Kibana", "crawlerStart": "https://www.elastic.co/guide/en/kibana/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/kibana/current/" }, +{ "name": "Graylog", "crawlerStart": "https://go2docs.graylog.org/5-1/home.htm/", "crawlerPrefix": "https://go2docs.graylog.org/5-1/" }, +{ "name": "Chart.js", "crawlerStart": "https://www.chartjs.org/docs/latest/", "crawlerPrefix": "https://www.chartjs.org/docs/latest/" }, +{ "name": "FusionCharts", "crawlerStart": "https://www.fusioncharts.com/dev/", "crawlerPrefix": "https://www.fusioncharts.com/dev/" }, +{ "name": "Dygraphs", "crawlerStart": "https://dygraphs.com/", "crawlerPrefix": "https://dygraphs.com/" }, +{ "name": "Victory", "crawlerStart": "https://formidable.com/open-source/victory/docs/", "crawlerPrefix": "https://formidable.com/open-source/victory/" }, +{ "name": "Chartist.js", "crawlerStart": "https://gionkunz.github.io/chartist-js/api-documentation.html", "crawlerPrefix": "https://gionkunz.github.io/chartist-js/api-documentation.html" }, +{ "name": "Recharts", "crawlerStart": "http://recharts.org/en-US/guide/", "crawlerPrefix": "http://recharts.org/en-US/" }, +{ "name": "AmCharts", "crawlerStart": "https://www.amcharts.com/docs/v5/", "crawlerPrefix": "https://www.amcharts.com/docs/v5/" }, +{ "name": "Google Charts", "crawlerStart": "https://developers.google.com/chart", "crawlerPrefix": "https://developers.google.com/chart" }, +{ "name": "AnyChart", "crawlerStart": "https://docs.anychart.com/Quick_Start/Quick_Start", "crawlerPrefix": "https://docs.anychart.com/" }, +{ "name": "Highcharts", "crawlerStart": "https://www.highcharts.com/docs/", "crawlerPrefix": "https://www.highcharts.com/docs/" }, +{ "name": "Highcharts API", "crawlerStart": "https://api.highcharts.com/highcharts/", "crawlerPrefix": "https://api.highcharts.com/highcharts/" }, +{ "name": "Billboard.js", "crawlerStart": "https://naver.github.io/billboard.js/release/latest/doc/", "crawlerPrefix": "https://naver.github.io/billboard.js/release/latest/doc/" }, +{ "name": "ApexCharts.js", "crawlerStart": "https://apexcharts.com/docs/installation/", "crawlerPrefix": "https://apexcharts.com/docs/" }, +{ "name": "NVD3", "crawlerStart": "http://nvd3.org/examples/index.html", "crawlerPrefix": "http://nvd3.org/examples/" }, +{ "name": "Vis.js", "crawlerStart": "https://visjs.github.io/vis-network/docs/network/", "crawlerPrefix": "https://visjs.github.io/vis-network/docs/network/" }, +{ "name": "Pica", "crawlerStart": "https://github.com/nodeca/pica", "crawlerPrefix": "https://github.com/nodeca/pica" }, +{ "name": "Lena.js", "crawlerStart": "https://github.com/davidsonfellipe/lena.js", "crawlerPrefix": "https://github.com/davidsonfellipe/lena.js" }, +{ "name": "Jimp", "crawlerStart": "https://www.npmjs.com/package/jimp", "crawlerPrefix": "https://www.npmjs.com/package/jimp" }, +{ "name": "Grade", "crawlerStart": "https://benhowdle89.github.io/grade/", "crawlerPrefix": "https://benhowdle89.github.io/grade/" }, +{ "name": "MarvinJ", "crawlerStart": "https://www.marvinj.org/en/", "crawlerPrefix": "https://www.marvinj.org/en/" }, +{ "name": "Compressor.js", "crawlerStart": "https://github.com/fengyuanchen/compressorjs", "crawlerPrefix": "https://github.com/fengyuanchen/compressorjs" }, +{ "name": "Fabric.js", "crawlerStart": "http://fabricjs.com/docs/", "crawlerPrefix": "http://fabricjs.com/docs/" }, +{ "name": "CamanJS", "crawlerStart": "http://camanjs.com/guides/", "crawlerPrefix": "http://camanjs.com/guides/" }, +{ "name": "Cropper.js", "crawlerStart": "https://fengyuanchen.github.io/cropperjs", "crawlerPrefix": "https://fengyuanchen.github.io/cropperjs" }, +{ "name": "Croppie", "crawlerStart": "https://foliotek.github.io/Croppie/", "crawlerPrefix": "https://foliotek.github.io/Croppie/" }, +{ "name": "Merge Images", "crawlerStart": "https://github.com/lukechilds/merge-images", "crawlerPrefix": "https://github.com/lukechilds/merge-images" }, +{ "name": "Blurify", "crawlerStart": "https://github.com/dabanlee/blurify", "crawlerPrefix": "https://github.com/dabanlee/blurify" }, +{ "name": "Pintura", "crawlerStart": "https://pqina.nl/pintura/docs/v8/", "crawlerPrefix": "https://pqina.nl/doka/docs/" }, +{ "name": "Gin", "crawlerStart": "https://gin-gonic.com/docs/", "crawlerPrefix": "https://gin-gonic.com/docs/" }, +{ "name": "Revel", "crawlerStart": "https://revel.github.io/manual/index.html", "crawlerPrefix": "https://revel.github.io/manual/" }, +{ "name": "Echo", "crawlerStart": "https://echo.labstack.com/docs", "crawlerPrefix": "https://echo.labstack.com/docs" }, +{ "name": "Martini", "crawlerStart": "https://github.com/go-martini/martini", "crawlerPrefix": "https://github.com/go-martini/martini" }, +{ "name": "Buffalo", "crawlerStart": "https://gobuffalo.io/documentation/", "crawlerPrefix": "https://gobuffalo.io/documentation/" }, +{ "name": "Gorm", "crawlerStart": "https://gorm.io/docs/index.html", "crawlerPrefix": "https://gorm.io/docs/" }, +{ "name": "Go-kit", "crawlerStart": "https://gokit.io/examples/stringsvc.html", "crawlerPrefix": "https://gokit.io/examples/" }, +{ "name": "Negroni", "crawlerStart": "https://github.com/urfave/negroni", "crawlerPrefix": "https://github.com/urfave/negroni" }, +{ "name": "Mux", "crawlerStart": "https://github.com/gorilla/mux", "crawlerPrefix": "https://github.com/gorilla/mux" }, +{ "name": "Devise", "crawlerStart": "https://github.com/heartcombo/devise", "crawlerPrefix": "https://github.com/heartcombo/devise" }, +{ "name": "CanCanCan", "crawlerStart": "https://github.com/CanCanCommunity/cancancan/blob/develop/docs/README.md", "crawlerPrefix": "https://github.com/CanCanCommunity/cancancan/blob/develop/docs/" }, +{ "name": "Active Storage", "crawlerStart": "https://guides.rubyonrails.org/active_storage_overview.html#", "crawlerPrefix": "https://guides.rubyonrails.org/active_storage_overview.html#" }, +{ "name": "Resque", "crawlerStart": "https://github.com/resque/resque", "crawlerPrefix": "https://github.com/resque/resque" }, +{ "name": "Sidekiq", "crawlerStart": "https://github.com/sidekiq/sidekiq/wiki/", "crawlerPrefix": "https://github.com/sidekiq/sidekiq/wiki/" }, +{ "name": "Carrierwave", "crawlerStart": "https://github.com/carrierwaveuploader/carrierwave", "crawlerPrefix": "https://github.com/carrierwaveuploader/carrierwave" }, +{ "name": "Pundit", "crawlerStart": "https://github.com/varvet/pundit", "crawlerPrefix": "https://github.com/varvet/pundit" }, +{ "name": "Active Admin", "crawlerStart": "https://activeadmin.info/documentation.html", "crawlerPrefix": "https://activeadmin.info/" }, +{ "name": "Delayed Job", "crawlerStart": "https://github.com/collectiveidea/delayed_job", "crawlerPrefix": "https://github.com/collectiveidea/delayed_job" }, +{ "name": "FriendlyId", "crawlerStart": "https://norman.github.io/friendly_id/file.Guide.html#", "crawlerPrefix": "https://norman.github.io/friendly_id/file.Guide.html#" }, +{ "name": "Scrapy", "crawlerStart": "https://docs.scrapy.org/en/latest/", "crawlerPrefix": "https://docs.scrapy.org/en/latest/" }, +{ "name": "PyQT", "crawlerStart": "https://www.riverbankcomputing.com/static/Docs/PyQt5/", "crawlerPrefix": "https://www.riverbankcomputing.com/static/Docs/PyQt5/" }, +{ "name": "Tkinter", "crawlerStart": "https://docs.python.org/3/library/tkinter.html", "crawlerPrefix": "https://docs.python.org/3/library/" }, +{ "name": "Entity Framework", "crawlerStart": "https://learn.microsoft.com/en-us/ef/", "crawlerPrefix": "https://learn.microsoft.com/en-us/ef/" }, +{ "name": "Xamarin", "crawlerStart": "https://learn.microsoft.com/en-us/xamarin/", "crawlerPrefix": "https://learn.microsoft.com/en-us/xamarin/" }, +{ "name": "ML.NET", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/machine-learning/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/machine-learning/" }, +{ "name": "SignalR", "crawlerStart": "https://learn.microsoft.com/en-us/aspnet/core/signalr/introduction?view=aspnetcore-7.0", "crawlerPrefix": "https://learn.microsoft.com/en-us/aspnet/core/signalr/" }, +{ "name": "Boost", "crawlerStart": "https://www.boost.org/doc/libs/", "crawlerPrefix": "https://www.boost.org/doc/libs/" }, +{ "name": "Qt", "crawlerStart": "https://doc.qt.io/", "crawlerPrefix": "https://doc.qt.io/qt-6/" }, +{ "name": "Eigen", "crawlerStart": "https://eigen.tuxfamily.org/dox/", "crawlerPrefix": "https://eigen.tuxfamily.org/dox/" }, +{ "name": "SFML", "crawlerStart": "https://www.sfml-dev.org/documentation/2.6.0/", "crawlerPrefix": "https://www.sfml-dev.org/documentation/2.6.0/" }, +{ "name": "Hibernate", "crawlerStart": "https://hibernate.org/orm/documentation/6.2/", "crawlerPrefix": "https://hibernate.org/orm/documentation/6.2/" }, +{ "name": "Apache Struts", "crawlerStart": "https://struts.apache.org/getting-started/", "crawlerPrefix": "https://struts.apache.org/" }, +{ "name": "Thymeleaf", "crawlerStart": "https://www.thymeleaf.org/documentation.html", "crawlerPrefix": "https://www.thymeleaf.org/doc/tutorials/3.1/" }, +{ "name": "Apache Ant", "crawlerStart": "https://ant.apache.org/manual/", "crawlerPrefix": "https://ant.apache.org/manual/" }, +{ "name": "Gradle", "crawlerStart": "https://docs.gradle.org/current/userguide/userguide.html", "crawlerPrefix": "https://docs.gradle.org/current/userguide/" }, +{ "name": "Quartz Scheduler", "crawlerStart": "http://www.quartz-scheduler.org/api/2.3.0/index.html", "crawlerPrefix": "http://www.quartz-scheduler.org/api/2.3.0/index.html" }, +{ "name": "Apache POI", "crawlerStart": "https://poi.apache.org/apidocs/dev/index.html", "crawlerPrefix": "https://poi.apache.org/apidocs/dev/index.html" }, +{ "name": "JHipster", "crawlerStart": "https://www.jhipster.tech/development/", "crawlerPrefix": "https://www.jhipster.tech/" }, +{ "name": "Vaadin", "crawlerStart": "https://vaadin.com/docs/latest/", "crawlerPrefix": "https://vaadin.com/docs/latest/" }, +{ "name": "Hadoop", "crawlerStart": "https://hadoop.apache.org/docs/current/", "crawlerPrefix": "https://hadoop.apache.org/docs/current/" }, +{ "name": "Kafka", "crawlerStart": "https://kafka.apache.org/documentation/", "crawlerPrefix": "https://kafka.apache.org/documentation/" }, +{ "name": "Guava", "crawlerStart": "https://github.com/google/guava/wiki/", "crawlerPrefix": "https://github.com/google/guava/wiki/" }, +{ "name": "Jackson", "crawlerStart": "https://github.com/FasterXML/jackson-core/wiki", "crawlerPrefix": "https://github.com/FasterXML/jackson-core/wiki" }, +{ "name": "Log4j", "crawlerStart": "https://logging.apache.org/log4j/2.x/manual/", "crawlerPrefix": "https://logging.apache.org/log4j/2.x/manual/" }, +{ "name": "SLF4J", "crawlerStart": "http://www.slf4j.org/manual.html", "crawlerPrefix": "http://www.slf4j.org/manual.html" }, +{ "name": "Chargebee", "crawlerStart": "https://www.chargebee.com/docs/2.0/", "crawlerPrefix": "https://www.chargebee.com/docs/2.0/" }, +{ "name": "Chargebee API", "crawlerStart": "https://apidocs.chargebee.com/docs/api/", "crawlerPrefix": "https://apidocs.chargebee.com/docs/api/" }, +{ "name": "Paypal", "crawlerStart": "https://developer.paypal.com/docs/", "crawlerPrefix": "https://developer.paypal.com/docs/" }, +{ "name": "FullCalendar", "crawlerStart": "https://fullcalendar.io/docs", "crawlerPrefix": "https://fullcalendar.io/docs" }, +{ "name": "Day.js", "crawlerStart": "https://day.js.org/docs/en/installation/installation", "crawlerPrefix": "https://day.js.org/docs/en/" }, +{ "name": "Date-fns", "crawlerStart": "https://date-fns.org/docs/Getting-Started", "crawlerPrefix": "https://date-fns.org/v2.30.0/docs/" }, +{ "name": "Luxon", "crawlerStart": "https://moment.github.io/luxon/docs/manual/", "crawlerPrefix": "https://moment.github.io/luxon/docs/manual/" }, +{ "name": "Chrono", "crawlerStart": "https://docs.rs/chrono/0.4.26/chrono/index.html", "crawlerPrefix": "https://docs.rs/chrono/0.4.26/chrono/" }, +{ "name": "WordPress.com", "crawlerStart": "https://developer.wordpress.com/docs/", "crawlerPrefix": "https://developer.wordpress.com/docs/" }, +{ "name": "WordPress.org", "crawlerStart": "https://developer.wordpress.org/", "crawlerPrefix": "https://developer.wordpress.org/" }, +{ "name": "Swiper", "crawlerStart": "https://swiperjs.com/get-started", "crawlerPrefix": "https://swiperjs.com/swiper-api" }, +{ "name": "Flickity", "crawlerStart": "https://flickity.metafizzy.co/", "crawlerPrefix": "https://flickity.metafizzy.co/" }, +{ "name": "Glide.js", "crawlerStart": "https://glidejs.com/docs/", "crawlerPrefix": "https://glidejs.com/" }, +{ "name": "Owl Carousel", "crawlerStart": "https://owlcarousel2.github.io/OwlCarousel2/docs/started-welcome.html", "crawlerPrefix": "https://owlcarousel2.github.io/OwlCarousel2/docs/" }, +{ "name": "Slick", "crawlerStart": "https://kenwheeler.github.io/slick/", "crawlerPrefix": "https://kenwheeler.github.io/slick/" }, +{ "name": "Docker", "crawlerStart": "https://docs.docker.com/", "crawlerPrefix": "https://docs.docker.com/" }, +{ "name": "Ansible", "crawlerStart": "https://docs.ansible.com/ansible/latest/index.html", "crawlerPrefix": "https://docs.ansible.com/ansible/latest/" }, +{ "name": "Terraform", "crawlerStart": "https://developer.hashicorp.com/terraform/docs", "crawlerPrefix": "https://developer.hashicorp.com/terraform/" }, +{ "name": "Puppet", "crawlerStart": "https://www.puppet.com/docs/puppet/8/puppet_index.html", "crawlerPrefix": "https://www.puppet.com/docs/puppet/8/" }, +{ "name": "Chef", "crawlerStart": "https://docs.chef.io/", "crawlerPrefix": "https://docs.chef.io/" }, +{ "name": "SaltStack", "crawlerStart": "https://docs.saltproject.io/en/latest/contents.html", "crawlerPrefix": "https://docs.saltproject.io/en/latest/topics/" }, +{ "name": "Nagios", "crawlerStart": "https://library.nagios.com/", "crawlerPrefix": "https://library.nagios.com/library/" }, +{ "name": "Prometheus", "crawlerStart": "https://prometheus.io/docs/introduction/overview/", "crawlerPrefix": "https://prometheus.io/docs/" }, +{ "name": "Grafana", "crawlerStart": "https://grafana.com/docs/grafana/latest/", "crawlerPrefix": "https://grafana.com/docs/grafana/latest/" }, +{ "name": "ELK Stack", "crawlerStart": "https://www.elastic.co/guide/en/elastic-stack/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/elastic-stack/current/" }, +{ "name": "Sequelize", "crawlerStart": "https://sequelize.org/docs/v6/", "crawlerPrefix": "https://sequelize.org/docs/v6/" }, +{ "name": "SQLAlchemy", "crawlerStart": "https://docs.sqlalchemy.org/en/20/", "crawlerPrefix": "https://docs.sqlalchemy.org/en/20/" }, +{ "name": "ActiveRecord", "crawlerStart": "https://guides.rubyonrails.org/active_record_basics.html", "crawlerPrefix": "https://guides.rubyonrails.org/active_record_basics.html" }, +{ "name": "MySQL", "crawlerStart": "https://dev.mysql.com/doc/", "crawlerPrefix": "https://dev.mysql.com/doc/" }, +{ "name": "Gensim", "crawlerStart": "https://radimrehurek.com/gensim/auto_examples/index.html#documentation", "crawlerPrefix": "https://radimrehurek.com/gensim/auto_examples/" }, +{ "name": "Gymnasium", "crawlerStart": "https://gymnasium.farama.org/", "crawlerPrefix": "https://gymnasium.farama.org/" }, +{ "name": "OpenSSL", "crawlerStart": "https://www.openssl.org/docs/", "crawlerPrefix": "https://www.openssl.org/docs/" }, +{ "name": "Catch2", "crawlerStart": "https://github.com/catchorg/Catch2/blob/master/docs/tutorial.md", "crawlerPrefix": "https://github.com/catchorg/Catch2/blob/master/docs/" }, +{ "name": "Netty", "crawlerStart": "https://netty.io/5.0/api/index.html", "crawlerPrefix": "https://netty.io/5.0/api/" }, +{ "name": "RestSharp", "crawlerStart": "https://restsharp.dev/intro.html#introduction", "crawlerPrefix": "https://restsharp.dev/" }, +{ "name": "Serilog", "crawlerStart": "https://github.com/serilog/serilog/wiki/", "crawlerPrefix": "https://github.com/serilog/serilog/wiki/" }, +{ "name": "spdlog", "crawlerStart": "https://github.com/gabime/spdlog/blob/v2.x/README.md", "crawlerPrefix": "https://github.com/gabime/spdlog/blob/v2.x/" }, +{ "name": "Winston", "crawlerStart": "https://github.com/winstonjs/winston/blob/master/README.md", "crawlerPrefix": "https://github.com/winstonjs/winston/blob/master/" }, +{ "name": "Python Logging", "crawlerStart": "https://docs.python.org/3/library/logging.html", "crawlerPrefix": "https://docs.python.org/3/library/logging.html" }, +{ "name": "Bouncy Castle", "crawlerStart": "https://www.bouncycastle.org/docs/", "crawlerPrefix": "https://www.bouncycastle.org/docs/" }, +{ "name": "IdentityServer", "crawlerStart": "https://identityserver4.readthedocs.io/en/latest/", "crawlerPrefix": "https://identityserver4.readthedocs.io/en/latest/" }, +{ "name": "Unity", "crawlerStart": "https://docs.unity3d.com/Manual/index.html", "crawlerPrefix": "https://docs.unity3d.com/" }, +{ "name": "LibGDX", "crawlerStart": "https://libgdx.com/wiki/", "crawlerPrefix": "https://libgdx.com/wiki/" }, +{ "name": "Phaser", "crawlerStart": "https://newdocs.phaser.io/docs/3.60.0", "crawlerPrefix": "https://newdocs.phaser.io/docs/3.60.0" }, +{ "name": "Ionic", "crawlerStart": "https://ionicframework.com/docs", "crawlerPrefix": "https://ionicframework.com/docs" }, +{ "name": "NativeScript", "crawlerStart": "https://docs.nativescript.org/", "crawlerPrefix": "https://docs.nativescript.org/" }, +{ "name": "LinQ", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/csharp/linq/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/csharp/linq/" }, +{ "name": "Brain.js", "crawlerStart": "https://github.com/BrainJS/brain.js#brainjs", "crawlerPrefix": "https://github.com/BrainJS/brain.js#brainjs" }, +{ "name": "Deeplearning4J", "crawlerStart": "https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn/latest/index.html", "crawlerPrefix": "https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn/latest/" }, +{ "name": "Accord.NET", "crawlerStart": "http://accord-framework.net/docs/html/N_Accord.htm", "crawlerPrefix": "http://accord-framework.net/docs/html/" }, +{ "name": "Giphy SDK", "crawlerStart": "https://developers.giphy.com/docs/sdk", "crawlerPrefix": "https://developers.giphy.com/docs/sdk/" }, +{ "name": "Giphy API", "crawlerStart": "https://developers.giphy.com/docs/api", "crawlerPrefix": "https://developers.giphy.com/docs/api/" }, +{ "name": "Amazon S3", "crawlerStart": "https://docs.aws.amazon.com/s3/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/" }, +{ "name": "Amazon EC2", "crawlerStart": "https://docs.aws.amazon.com/ec2/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/" }, +{ "name": "Aws Lambda", "crawlerStart": "https://docs.aws.amazon.com/lambda/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/lambda/latest/dg/" }, +{ "name": "Mantis", "crawlerStart": "https://www.mantisbt.org/documentation.php", "crawlerPrefix": "https://www.mantisbt.org/docs/master/en-US/Developers_Guide/" }, +{ "name": "React Admin", "crawlerStart": "https://marmelab.com/react-admin/documentation.html", "crawlerPrefix": "https://marmelab.com/react-admin/" }, +{ "name": "SQLFluff", "crawlerStart": "https://docs.sqlfluff.com/en/stable/", "crawlerPrefix": "https://docs.sqlfluff.com/en/stable/" }, +{ "name": "Ruff", "crawlerStart": "https://beta.ruff.rs/docs/", "crawlerPrefix": "https://beta.ruff.rs/docs/" }, +{ "name": "FusionAuth API", "crawlerStart": "https://fusionauth.io/docs/v1/tech/apis/", "crawlerPrefix": "https://fusionauth.io/docs/v1/tech/apis/" }, +{ "name": "Serp Google Scholar API", "crawlerStart": "https://serpapi.com/google-scholar-api", "crawlerPrefix": "https://serpapi.com/google-scholar-api" }, +{ "name": "Novu", "crawlerStart": "https://docs.novu.co/", "crawlerPrefix": "https://docs.novu.co/" }, +{ "name": "Drizzle", "crawlerStart": "https://orm.drizzle.team/docs/overview", "crawlerPrefix": "https://orm.drizzle.team/docs/overview" }, +{ "name": "Autogen", "crawlerStart": "https://microsoft.github.io/autogen/docs/", "crawlerPrefix": "https://microsoft.github.io/autogen/docs/" }, +{ "name": "Domo", "crawlerStart": "https://developer.domo.com/", "crawlerPrefix": "https://developer.domo.com/" }, +{ "name": "Crystal", "crawlerStart": "https://crystal-lang.org/api", "crawlerPrefix": "https://crystal-lang.org/api" }, +{ "name": "Amber", "crawlerStart": "https://docs.amberframework.org/amber", "crawlerPrefix": "https://docs.amberframework.org/amber" }, +{ "name": "TLDraw", "crawlerStart": "https://tldraw.dev/", "crawlerPrefix": "https://tldraw.dev/" }, +{ "name": "Swift", "crawlerStart": "https://developer.apple.com/documentation/technologies", "crawlerPrefix": "https://developer.apple.com/documentation/technologies" }]"#; diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs new file mode 100644 index 00000000000..a9614b76bf2 --- /dev/null +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -0,0 +1,278 @@ +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use anyhow::anyhow; +use async_trait::async_trait; +use juniper::ID; +use tabby_db::{DbConn, WebDocumentDAO}; +use tabby_schema::{ + job::{JobInfo, JobService}, + web_documents::{CustomWebDocument, PresetWebDocument, WebDocumentService}, + AsID, AsRowid, CoreError, Result, +}; + +use super::{ + background_job::BackgroundJobEvent, graphql_pagination_to_filter, + preset_web_documents_data::PRESET_WEB_DOCUMENTS_DATA, +}; + +pub fn create(db: DbConn, job_service: Arc) -> impl WebDocumentService { + let data: serde_json::Value = serde_json::from_str(PRESET_WEB_DOCUMENTS_DATA).unwrap(); + let mut preset_web_documents = HashMap::new(); + for doc in data.as_array().unwrap() { + let name = doc.get("name").unwrap().to_string(); + let url = doc.get("crawlerStart").unwrap().to_string(); + preset_web_documents.insert( + String::from(&name[1..name.len() - 1]), + String::from(&url[1..url.len() - 1]), + ); + } + WebDocumentServiceImpl { + db, + job_service, + preset_web_documents, + } +} + +struct WebDocumentServiceImpl { + db: DbConn, + job_service: Arc, + preset_web_documents: HashMap, +} + +#[async_trait] +impl WebDocumentService for WebDocumentServiceImpl { + async fn list_custom_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + let urls = self + .db + .list_web_documents(limit, skip_id, backwards, false) + .await?; + + let mut converted_urls = vec![]; + + for url in urls { + let event = BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&url.id.as_id()), + url.url.clone(), + ); + + let job_info = self.job_service.get_job_info(event.to_command()).await?; + converted_urls.push(to_custom_web_document(url, job_info)); + } + Ok(converted_urls) + } + + async fn create_custom_web_document(&self, name: String, url: String) -> Result { + if self.preset_web_documents.contains_key(&name) { + return Err(CoreError::Other(anyhow!(format!( + "name: {} is conflicts with preset document", + name + )))); + } + let id = self + .db + .create_web_document(name, url.clone(), false) + .await?; + let _ = self + .job_service + .trigger( + BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&id.as_id()), + url, + ) + .to_command(), + ) + .await; + Ok(id.as_id()) + } + + async fn delete_custom_web_document(&self, id: ID) -> Result<()> { + self.db.delete_web_document(id.as_rowid()?).await?; + self.job_service + .trigger(BackgroundJobEvent::IndexGarbageCollection.to_command()) + .await?; + Ok(()) + } + + async fn list_preset_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + is_active: bool, + ) -> Result> { + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + let urls = self + .db + .list_web_documents(limit, skip_id, backwards, true) + .await?; + + let mut converted_urls = vec![]; + let mut active_urls: HashSet = HashSet::default(); + + for url in urls { + active_urls.insert(url.name.clone()); + + if is_active { + let event = BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&url.id.as_id()), + url.url.clone(), + ); + + let job_info = self.job_service.get_job_info(event.to_command()).await?; + converted_urls.push(to_preset_web_document(url, job_info)); + } + } + + if !is_active { + for name in self.preset_web_documents.keys() { + if active_urls.contains(name) { + continue; + } + + converted_urls.push(PresetWebDocument { + id: ID::from(name.clone()), + name: name.clone(), + job_info: None, + updated_at: None, + }); + } + } + + Ok(converted_urls) + } + + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<()> { + if !active { + self.db.deactivate_preset_web_document(name).await?; + self.job_service + .trigger(BackgroundJobEvent::IndexGarbageCollection.to_command()) + .await?; + Ok(()) + } else { + let Some(url) = self.preset_web_documents.get(&name) else { + return Err(CoreError::Other(anyhow!(format!( + "name {} does not exist", + name + )))); + }; + let id = self.db.create_web_document(name, url.clone(), true).await?; + let _ = self + .job_service + .trigger( + BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&id.as_id()), + url.clone(), + ) + .to_command(), + ) + .await; + Ok(()) + } + } +} + +fn to_custom_web_document(value: WebDocumentDAO, job_info: JobInfo) -> CustomWebDocument { + CustomWebDocument { + id: value.id.as_id(), + url: value.url.clone(), + created_at: value.created_at, + updated_at: value.updated_at, + job_info, + name: value.name, + } +} + +fn to_preset_web_document(value: WebDocumentDAO, job_info: JobInfo) -> PresetWebDocument { + PresetWebDocument { + id: ID::from(value.name.clone()), + name: value.name, + job_info: Some(job_info), + updated_at: Some(value.updated_at), + } +} +#[cfg(test)] +mod tests { + use tabby_db::DbConn; + + use super::*; + use crate::background_job::BackgroundJobEvent; + + #[tokio::test] + async fn test_list_web_documents() { + let db = DbConn::new_in_memory().await.unwrap(); + let job = Arc::new(crate::service::job::create(db.clone()).await); + let service = create(db.clone(), job.clone()); + + let url = "https://example.com".to_string(); + let id = service + .create_custom_web_document("example".to_string(), url.clone()) + .await + .unwrap(); + + let command = BackgroundJobEvent::WebCrawler(id.to_string(), "https://example.com".into()) + .to_command(); + + db.create_job_run("web".into(), command).await.unwrap(); + + let urls = service + .list_custom_web_documents(None, None, None, None) + .await + .unwrap(); + + assert_eq!(1, urls.len()); + assert_eq!(id, urls[0].id); + assert!(urls[0].job_info.last_job_run.is_some()); + + service + .set_preset_web_documents_active("React".to_string(), true) + .await + .unwrap(); + + let command = + BackgroundJobEvent::WebCrawler("id".into(), "https://react.dev/reference/".into()) + .to_command(); + + db.create_job_run("preset".into(), command).await.unwrap(); + let urls = service + .list_preset_web_documents(None, None, None, None, true) + .await + .unwrap(); + + assert_eq!(1, urls.len()); + assert!(urls[0].updated_at.is_some()); + let urls = service + .list_preset_web_documents(None, None, None, None, false) + .await + .unwrap(); + + assert_eq!(383, urls.len()); + service + .set_preset_web_documents_active("React".to_string(), false) + .await + .unwrap(); + let urls = service + .list_preset_web_documents(None, None, None, None, false) + .await + .unwrap(); + assert_eq!(384, urls.len()); + + service.delete_custom_web_document(id).await.unwrap(); + let urls = service + .list_custom_web_documents(None, None, None, None) + .await + .unwrap(); + + assert_eq!(0, urls.len()); + } +} diff --git a/ee/tabby-webserver/src/webserver.rs b/ee/tabby-webserver/src/webserver.rs index 98a53520bdb..e9d358d2939 100644 --- a/ee/tabby-webserver/src/webserver.rs +++ b/ee/tabby-webserver/src/webserver.rs @@ -13,7 +13,7 @@ use tabby_db::DbConn; use tabby_inference::{ChatCompletionStream, Embedding}; use tabby_schema::{ integration::IntegrationService, job::JobService, repository::RepositoryService, - web_crawler::WebCrawlerService, + web_crawler::WebCrawlerService, web_documents::WebDocumentService, }; use crate::{ @@ -21,7 +21,7 @@ use crate::{ routes, service::{ background_job, create_service_locator, event_logger::create_event_logger, integration, - job, repository, web_crawler, + job, repository, web_crawler, web_documents, }, }; @@ -31,6 +31,7 @@ pub struct Webserver { repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, } @@ -66,6 +67,7 @@ impl Webserver { let repository = repository::create(db.clone(), integration.clone(), job.clone()); let web_crawler = Arc::new(web_crawler::create(db.clone(), job.clone())); + let web_documents = Arc::new(web_documents::create(db.clone(), job.clone())); let logger2 = create_event_logger(db.clone()); let logger = Arc::new(ComposedLogger::new(logger1, logger2)); @@ -75,6 +77,7 @@ impl Webserver { repository: repository.clone(), integration: integration.clone(), web_crawler: web_crawler.clone(), + web_documents: web_documents.clone(), job: job.clone(), }); @@ -126,6 +129,7 @@ impl Webserver { self.repository.clone(), self.integration.clone(), self.web_crawler.clone(), + self.web_documents.clone(), self.job.clone(), answer.clone(), self.db.clone(),