From 7972e0d8aa07650c6a0a9bda5a5f78a78c4147dc Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Fri, 16 Aug 2024 15:52:26 +0800 Subject: [PATCH 01/12] add web document change Signed-off-by: xxs-wallace --- .../migrations/0036_web_document.down.sql | 2 + .../migrations/0036_web_document.up.sql | 11 ++++ ee/tabby-db/schema.sqlite | Bin 188416 -> 188416 bytes ee/tabby-db/src/web_crawler.rs | 53 ++++++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 ee/tabby-db/migrations/0036_web_document.down.sql create mode 100644 ee/tabby-db/migrations/0036_web_document.up.sql diff --git a/ee/tabby-db/migrations/0036_web_document.down.sql b/ee/tabby-db/migrations/0036_web_document.down.sql new file mode 100644 index 00000000000..5963737320e --- /dev/null +++ b/ee/tabby-db/migrations/0036_web_document.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE web_documents; diff --git a/ee/tabby-db/migrations/0036_web_document.up.sql b/ee/tabby-db/migrations/0036_web_document.up.sql new file mode 100644 index 00000000000..62a42b1950b --- /dev/null +++ b/ee/tabby-db/migrations/0036_web_document.up.sql @@ -0,0 +1,11 @@ +-- Add up migration script here +CREATE TABLE web_documents( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + is_preset BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + CONSTRAINT `unique_name` UNIQUE(name), + CONSTRAINT `unique_url` UNIQUE(url) +); diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 6f19779ffb8394bc5af574489d298245b4b9309a..3699c748649f338b8ec5cc933d83be7a971a1ea4 100644 GIT binary patch delta 1900 zcmZvce{2(F7{~8@@7imxKVj{Ht+q69&r2tzP9qEU!uFrr2!GZGCD^}WF&J@@ABlK1(% z&-eM>d+$BnXH?y1RK1otkRXUWe9w%MzFdzC5_#MA4hQ>nH69#KHf!)R19lR+0o@Yq z4ed^Cy5>WTUwvQwrn*oyt_rCX%u%LXd0V+lX;qw61nGb1K{{W4QQjuk%XryB>MHdj zWhUPzm%^`M=V+DQ34NO~XpreqhROLmLJe%NtuY=6MLXP_yTHjUbh-;z&RzUuL9xfv z4}T7Pvw!uuXaCyp`tY6CJ}PgW2*Y_B3e*4iAnAJB6Yu;mG|;(P<88X0{;pR&#@;2p zub^vlbC~}W5R=4_ZIY^SDV!4)Og(3yLAGex(Aw7EY>!8sEzw}; zh1o3cOTk)CzlYAGSS0m-#DrW`fh=a}c2=4>at^$YUN+B{;p&})2V3v? z7#cXlyk1Yqq4!d65ivpnFzn8Alp^9fuM5(xj&M zU;@ua(8;uH_2Yj4`X$Zfgr~po$sReAJ^E^~hez%I&d@5GizKxI+G(>TN*%!fe%_02 z*wPaLW5*eLuAwD%OCktiKz$NM19pk4OLi9{^PObGex&0N&6YtOH(JF`Xyq~zfcY7_ z3-g7-d@#nA;3ad3#e{ZTi!^>siuyCg&$t;{v0t%K!OCN@CRrYJm*l||UL+lYJ_t?# zp{oY$qJ-vk;21!|%Bb+?CU6A^#%j<4(P?$Hkm3hF!5o~dgZl{G$C~3Bv-*U(N`;uU z$_Zsyp`gd)Bl0-)J;rzdf8W99g;(nU35BZ7uo?=#HN$=Y7v=I@5oi@k!|)U-oNET9 zK$wgG3lvV*gK9X$u_MbU{UM5DM=9=seQ25f;Uz{#sqP2%kzSQvDf*bc1;aY{tIh#R z2_Bg~uJ7JpTT{N>U zJplw<3DX=Zmrw}h_3#$d$uAItrlKOScky(O;?FiiBOl)kUs}04znC^t(8C$dbsWoM60$RYZysHHkp-Uhgu>65AFfnXPEU>jbg#RJFz6oZc6Pjdx zX(Q0{iy|XA!B4tB-b%JntX%PT5t*t&SMWm`NH5W9LETU%LCRa>?|oH%lE uy;EDDgFn;^S5xXN&@cSn4Br4kZV+Zdp{ErN0g52VZ~5*BOr8Fuv3~*acKa*< delta 1979 zcmaKteQXnD9LMjuyYAZS+i>05_4R%{5giY5693@s55f{rtmcB6F(bh@A_Q4X)F~u_K>|dB5|oHfI|A-R+y0)kzt8i0 zzrVZZcTf9A)cqssS1pqoFbu0fQI3K@aeT1aqiDctQu~jE_vo8*>5>vH8mHi+n0~K* zwr)(f_CsmhJn^hL&h_aLVi`qxk6ns_?FVHDoQ`C`X z$&I9uI7-aIN3%`Lb{u~FNEOX9ZjYDaz1$3_DdGs)a~RGk@~C;9;W%$?t+%!wZbEr| z-$M9ykyBg8y4?&{@Ab^|@*KR3h<54?SZZJp;XMpL!#jP3mvh4g1Ls^d)ZAjHfWxDT z684_7kYeMZnBeBzwJbMr_C5ngvcrI6=h9M+=>D|>4wRP3)?^WBJ)D7;OQ*<4 zv6&;*eR&ivFw4nCMEtr@IApfV){G+3a%v1JEjAf(TaeE;7olKr$;bgj+Aj{l^A@=e zVMS}itFX>0_X!~qR9WC|YlTehQ$zx7C*hw~x!R*PvfLp4qRmCNo?D9Rj|))pQv@j&J+GkC*YWU@09&&y&dbKZc-mm0|jMd@8E&iQhFtL6G*`}5W%Hi zn!o@_Vi?$-7UxHCyV|U}jomal zonVVij79Oupj3P%iaW&jf}k3*lu5iE#ns|1l!6YG7LI6Xv9JrDBHd^LXMwb|8_WlK zbcTL!4EVt2CHY53d_DwhbWbuI$_U~1P$u`=swiS({44n66AFnfAI|;Y4S;W_)c;XCmej z;;xjC&Y%J5jE{3d$;e6}m5wChvrzxJS)OOz^;4sfbcVfe7ZK9z+C+*CvAOe?X3*<_ zquA65o)KFTU_k>@`Co+!x#FBmv#-I|>}#Codz$(0nPM^wD&Sm=V!?frdzYJ6S;MSo zT-db2$5h_O)&HZG^OXkI0~>yjW8Rrd79JuwCQEX690%tXj(>95EZKj@HMoq@;;74} zp{PPpCK;CD*Ys1-R~=Z4>7m|+{w!b6bZW|#*9)4Ihv>`dOY~N<8Z6E>6;=XiaR|SF rOYL3wG+cVQ12p4e?^=A3^g|fz0n*lP{2;&|s}_4>xK%=Z`eFKS@sKq- diff --git a/ee/tabby-db/src/web_crawler.rs b/ee/tabby-db/src/web_crawler.rs index 717d747d653..e0e8114724e 100644 --- a/ee/tabby-db/src/web_crawler.rs +++ b/ee/tabby-db/src/web_crawler.rs @@ -12,6 +12,17 @@ pub struct WebCrawlerUrlDAO { pub created_at: DateTime, } +#[allow(unused)] +#[derive(FromRow)] +pub struct WebDocumentDAO { + pub id: i64, + pub name: String, + pub url: String, + pub is_preset: bool, + pub created_at: DateTime, + pub updated_at: DateTime, +} + impl DbConn { pub async fn list_web_crawler_urls( &self, @@ -47,4 +58,46 @@ impl DbConn { .await?; Ok(()) } + + pub async fn list_web_document( + &self, + limit: Option, + skip_id: Option, + backwards: bool, + is_preset: Option, + ) -> Result> { + let condition = if let Some(is_preset) = is_preset { + Some(format!("is_preset={}", is_preset)) + } else { + None + }; + + let urls = query_paged_as!( + WebDocumentDAO, + "web_documents", + ["id", "name", "url", "is_preset", "created_at" as "created_at!: DateTime", "updated_at" as "updated_at!: DateTime"], + limit, + skip_id, + backwards, + condition + ).fetch_all(&self.pool) + .await?; + + Ok(urls) + } + + pub async fn create_web_document(&self, name: String, url: String, is_preset: bool) -> Result { + let res = query!("INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", name, url, is_preset) + .execute(&self.pool) + .await?; + + Ok(res.last_insert_rowid()) + } + + pub async fn delete_web_document(&self, id: i64) -> Result<()> { + query!("DELETE FROM web_documents WHERE id = ?;", id) + .execute(&self.pool) + .await?; + Ok(()) + } } From 20311bfca45ec3e255f33f2450450990b6c74c01 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:04:46 +0000 Subject: [PATCH 02/12] [autofix.ci] apply automated fixes --- ee/tabby-db/src/web_crawler.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ee/tabby-db/src/web_crawler.rs b/ee/tabby-db/src/web_crawler.rs index e0e8114724e..56caff55cbc 100644 --- a/ee/tabby-db/src/web_crawler.rs +++ b/ee/tabby-db/src/web_crawler.rs @@ -66,11 +66,7 @@ impl DbConn { backwards: bool, is_preset: Option, ) -> Result> { - let condition = if let Some(is_preset) = is_preset { - Some(format!("is_preset={}", is_preset)) - } else { - None - }; + let condition = is_preset.map(|is_preset| format!("is_preset={}", is_preset)); let urls = query_paged_as!( WebDocumentDAO, @@ -86,10 +82,20 @@ impl DbConn { Ok(urls) } - pub async fn create_web_document(&self, name: String, url: String, is_preset: bool) -> Result { - let res = query!("INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", name, url, is_preset) - .execute(&self.pool) - .await?; + pub async fn create_web_document( + &self, + name: String, + url: String, + is_preset: bool, + ) -> Result { + let res = query!( + "INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", + name, + url, + is_preset + ) + .execute(&self.pool) + .await?; Ok(res.last_insert_rowid()) } From 0b36e304aba8fef73cebf26a63e01706f173d593 Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Fri, 16 Aug 2024 17:06:28 +0800 Subject: [PATCH 03/12] add to schema.sql Signed-off-by: xxs-wallace --- ee/tabby-db/schema/schema.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql index 34a20b3acba..803126d9abc 100644 --- a/ee/tabby-db/schema/schema.sql +++ b/ee/tabby-db/schema/schema.sql @@ -190,3 +190,13 @@ CREATE TABLE thread_messages( updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), FOREIGN KEY(thread_id) REFERENCES threads(id) ON DELETE CASCADE ); +CREATE TABLE web_documents( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + is_preset BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + CONSTRAINT `unique_name` UNIQUE(name), + CONSTRAINT `unique_url` UNIQUE(url) +); From d4d1480e318c49b3f3c05adeaef19d2982261da4 Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Sat, 17 Aug 2024 00:56:02 +0800 Subject: [PATCH 04/12] move to new file Signed-off-by: xxs-wallace --- ee/tabby-db/src/lib.rs | 1 + ee/tabby-db/src/web_crawler.rs | 59 ----------------------------- ee/tabby-db/src/web_document.rs | 67 +++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 59 deletions(-) create mode 100644 ee/tabby-db/src/web_document.rs diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs index cd239e49e7f..cf5a5a9cf8a 100644 --- a/ee/tabby-db/src/lib.rs +++ b/ee/tabby-db/src/lib.rs @@ -41,6 +41,7 @@ mod user_completions; mod user_events; mod users; mod web_crawler; +mod web_document; use anyhow::Result; use sql_query_builder as sql; diff --git a/ee/tabby-db/src/web_crawler.rs b/ee/tabby-db/src/web_crawler.rs index 56caff55cbc..717d747d653 100644 --- a/ee/tabby-db/src/web_crawler.rs +++ b/ee/tabby-db/src/web_crawler.rs @@ -12,17 +12,6 @@ pub struct WebCrawlerUrlDAO { pub created_at: DateTime, } -#[allow(unused)] -#[derive(FromRow)] -pub struct WebDocumentDAO { - pub id: i64, - pub name: String, - pub url: String, - pub is_preset: bool, - pub created_at: DateTime, - pub updated_at: DateTime, -} - impl DbConn { pub async fn list_web_crawler_urls( &self, @@ -58,52 +47,4 @@ impl DbConn { .await?; Ok(()) } - - pub async fn list_web_document( - &self, - limit: Option, - skip_id: Option, - backwards: bool, - is_preset: Option, - ) -> Result> { - let condition = is_preset.map(|is_preset| format!("is_preset={}", is_preset)); - - let urls = query_paged_as!( - WebDocumentDAO, - "web_documents", - ["id", "name", "url", "is_preset", "created_at" as "created_at!: DateTime", "updated_at" as "updated_at!: DateTime"], - limit, - skip_id, - backwards, - condition - ).fetch_all(&self.pool) - .await?; - - Ok(urls) - } - - pub async fn create_web_document( - &self, - name: String, - url: String, - is_preset: bool, - ) -> Result { - let res = query!( - "INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", - name, - url, - is_preset - ) - .execute(&self.pool) - .await?; - - Ok(res.last_insert_rowid()) - } - - pub async fn delete_web_document(&self, id: i64) -> Result<()> { - query!("DELETE FROM web_documents WHERE id = ?;", id) - .execute(&self.pool) - .await?; - Ok(()) - } } diff --git a/ee/tabby-db/src/web_document.rs b/ee/tabby-db/src/web_document.rs new file mode 100644 index 00000000000..966f7e6cff0 --- /dev/null +++ b/ee/tabby-db/src/web_document.rs @@ -0,0 +1,67 @@ +use anyhow::Result; +use chrono::{DateTime, Utc}; +use sqlx::{prelude::FromRow, query}; +use tabby_db_macros::query_paged_as; + +use crate::DbConn; + +#[allow(unused)] +#[derive(FromRow)] +pub struct WebDocumentDAO { + pub id: i64, + pub name: String, + pub url: String, + pub is_preset: bool, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl DbConn { + pub async fn list_web_document( + &self, + limit: Option, + skip_id: Option, + backwards: bool, + is_preset: Option, + ) -> Result> { + let condition = is_preset.map(|is_preset| format!("is_preset={}", is_preset)); + + let urls = query_paged_as!( + WebDocumentDAO, + "web_documents", + ["id", "name", "url", "is_preset", "created_at" as "created_at!: DateTime", "updated_at" as "updated_at!: DateTime"], + limit, + skip_id, + backwards, + condition + ).fetch_all(&self.pool) + .await?; + + Ok(urls) + } + + pub async fn create_web_document( + &self, + name: String, + url: String, + is_preset: bool, + ) -> Result { + let res = query!( + "INSERT INTO web_documents(name, url, is_preset) VALUES (?,?,?);", + name, + url, + is_preset + ) + .execute(&self.pool) + .await?; + + Ok(res.last_insert_rowid()) + } + + pub async fn delete_web_document(&self, id: i64) -> Result<()> { + query!("DELETE FROM web_documents WHERE id = ?;", id) + .execute(&self.pool) + .await?; + Ok(()) + } +} From 88c47bd949d73223acb6847ed5cfedca956417d8 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 16 Aug 2024 13:09:22 -0700 Subject: [PATCH 05/12] update schema.sql --- .../migrations/0036_web_document.up.sql | 4 ++-- ee/tabby-db/schema.sqlite | Bin 188416 -> 188416 bytes ee/tabby-db/schema/schema.sql | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ee/tabby-db/migrations/0036_web_document.up.sql b/ee/tabby-db/migrations/0036_web_document.up.sql index 62a42b1950b..76248f8e164 100644 --- a/ee/tabby-db/migrations/0036_web_document.up.sql +++ b/ee/tabby-db/migrations/0036_web_document.up.sql @@ -6,6 +6,6 @@ CREATE TABLE web_documents( is_preset BOOLEAN NOT NULL DEFAULT FALSE, created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), - CONSTRAINT `unique_name` UNIQUE(name), - CONSTRAINT `unique_url` UNIQUE(url) + CONSTRAINT idx_name UNIQUE(name), + CONSTRAINT idx_url UNIQUE(url) ); diff --git a/ee/tabby-db/schema.sqlite b/ee/tabby-db/schema.sqlite index 3699c748649f338b8ec5cc933d83be7a971a1ea4..a38c2d52929dc0b39bbb0807ca18f1feefaa53a6 100644 GIT binary patch delta 696 zcmXw$T}V@57{@v9IpS*efoJDy=j=PHgr<#TxG(OAUzvn)F+jz-h|x?0G{;fO^$v`#W9V~~QGQ8yxE{i733&x>iYrTb?dz@roWd+hsKOI;k1I=-;>1<^{$H*Fk zRy|0+kLf0FJ1N3~&Kh_Fqm6G+W3Yk<;l|tW%rI#nYmgtYX@^aN1u9%*<*(;(!^Kt% zx-j~hhxaa)_)XZ-z~^R-O>UDfn!N}Qc14;9A-o?}-4%3)-9svKd8qZUhp`GGpMV!0 z*7=DDp=@%%oAh$;yhsXjOBwkV(gZG@8gAiU?|pbURIK>@TyjrlDTZ1&TYIec=AB5q UHFe%Xp5g-C>*~mKT4<>6KVbsWaR2}S delta 698 zcmXw#dq`7J9LKrmPI_$i@b2F2Zg(#Umm9U+tEd}M(H}B3@Q+Xkj7`}VshbQEk*H+V z`X{XTg+mBh(O>JO1+52!6_NQMkd697K?;QWBM}TDgZjC1`t$QW-^V#;WJ*6WrJu-E zHP#2~>c+_T@aNICx5s~VUY%UM@-o!>rJq!HHn`TG35x@sd&_sm&$LwrPA)z5ExJO)K8Yv2`?^!|nhg%xxoH|~TRik}9i6@0|xY1mL$z+=P7S`kjz z*u-U>;0N-nd2rdwjT#)aV_-)C&i@zkD3IxEuw?gaZfKx)pd(X)gO2jY zXb?w1_og4l9jtR6g^I4{X=ldCJ#`ZCFgI6_B1G14x$N~G?&kIjzTg&<+s$Qch2<#e h;VL78$y2GpSUB08h()rkapK6%_7O|rT%4qb{{hR<(Om!l diff --git a/ee/tabby-db/schema/schema.sql b/ee/tabby-db/schema/schema.sql index 803126d9abc..15bead2fad4 100644 --- a/ee/tabby-db/schema/schema.sql +++ b/ee/tabby-db/schema/schema.sql @@ -191,12 +191,12 @@ CREATE TABLE thread_messages( FOREIGN KEY(thread_id) REFERENCES threads(id) ON DELETE CASCADE ); CREATE TABLE web_documents( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name VARCHAR(255) NOT NULL, - url TEXT NOT NULL, - is_preset BOOLEAN NOT NULL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), - updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), - CONSTRAINT `unique_name` UNIQUE(name), - CONSTRAINT `unique_url` UNIQUE(url) + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR(255) NOT NULL, + url TEXT NOT NULL, + is_preset BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + updated_at TIMESTAMP NOT NULL DEFAULT(DATETIME('now')), + CONSTRAINT idx_name UNIQUE(name), + CONSTRAINT idx_url UNIQUE(url) ); From 0a52e7d6585183341266efa7534e22fc8e8af54a Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Fri, 16 Aug 2024 13:12:31 -0700 Subject: [PATCH 06/12] update --- ee/tabby-db/migrations/0036_web_document.down.sql | 1 - ee/tabby-db/migrations/0036_web_document.up.sql | 1 - 2 files changed, 2 deletions(-) diff --git a/ee/tabby-db/migrations/0036_web_document.down.sql b/ee/tabby-db/migrations/0036_web_document.down.sql index 5963737320e..cfdbee70bfd 100644 --- a/ee/tabby-db/migrations/0036_web_document.down.sql +++ b/ee/tabby-db/migrations/0036_web_document.down.sql @@ -1,2 +1 @@ --- Add down migration script here DROP TABLE web_documents; diff --git a/ee/tabby-db/migrations/0036_web_document.up.sql b/ee/tabby-db/migrations/0036_web_document.up.sql index 76248f8e164..0a2f0d948f3 100644 --- a/ee/tabby-db/migrations/0036_web_document.up.sql +++ b/ee/tabby-db/migrations/0036_web_document.up.sql @@ -1,4 +1,3 @@ --- Add up migration script here CREATE TABLE web_documents( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, name VARCHAR(255) NOT NULL, From 4584202dfc084cde65ce2df4781420e45ca49d9c Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Sat, 17 Aug 2024 15:12:37 +0800 Subject: [PATCH 07/12] commit suggestion Signed-off-by: xxs-wallace --- ee/tabby-db/src/lib.rs | 2 +- ee/tabby-db/src/{web_document.rs => web_documents.rs} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename ee/tabby-db/src/{web_document.rs => web_documents.rs} (97%) diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs index cf5a5a9cf8a..28f6fd69b8b 100644 --- a/ee/tabby-db/src/lib.rs +++ b/ee/tabby-db/src/lib.rs @@ -41,7 +41,7 @@ mod user_completions; mod user_events; mod users; mod web_crawler; -mod web_document; +mod web_documents; use anyhow::Result; use sql_query_builder as sql; diff --git a/ee/tabby-db/src/web_document.rs b/ee/tabby-db/src/web_documents.rs similarity index 97% rename from ee/tabby-db/src/web_document.rs rename to ee/tabby-db/src/web_documents.rs index 966f7e6cff0..02326cac806 100644 --- a/ee/tabby-db/src/web_document.rs +++ b/ee/tabby-db/src/web_documents.rs @@ -17,7 +17,7 @@ pub struct WebDocumentDAO { } impl DbConn { - pub async fn list_web_document( + pub async fn list_web_documents( &self, limit: Option, skip_id: Option, From fc8ff71b3a4d612f6226c7cf0609fe03fe26bebc Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Mon, 19 Aug 2024 14:07:57 +0800 Subject: [PATCH 08/12] refactor(document): add web document service api (#2904) * add api Signed-off-by: xxs-wallace * fmt Signed-off-by: xxs-wallace * refactor some name Signed-off-by: xxs-wallace * fix generate example Signed-off-by: xxs-wallace * fmt Signed-off-by: xxs-wallace * rename Signed-off-by: xxs-wallace * fix method Signed-off-by: xxs-wallace * add updated at field Signed-off-by: xxs-wallace * add regex Signed-off-by: xxs-wallace * Update ee/tabby-schema/src/schema/web_documents.rs * fix doc Signed-off-by: xxs-wallace * add test Signed-off-by: xxs-wallace * fix space Signed-off-by: xxs-wallace --------- Signed-off-by: xxs-wallace Co-authored-by: Meng Zhang --- ee/tabby-db/src/lib.rs | 1 + ee/tabby-schema/graphql/schema.graphql | 52 ++++++++ ee/tabby-schema/src/schema/constants.rs | 19 +++ ee/tabby-schema/src/schema/mod.rs | 79 ++++++++++++ ee/tabby-schema/src/schema/web_documents.rs | 115 ++++++++++++++++++ ee/tabby-webserver/src/service/mod.rs | 11 ++ .../src/service/web_documents.rs | 57 +++++++++ ee/tabby-webserver/src/webserver.rs | 22 ++-- 8 files changed, 347 insertions(+), 9 deletions(-) create mode 100644 ee/tabby-schema/src/schema/web_documents.rs create mode 100644 ee/tabby-webserver/src/service/web_documents.rs diff --git a/ee/tabby-db/src/lib.rs b/ee/tabby-db/src/lib.rs index 28f6fd69b8b..ef6d06d8eb1 100644 --- a/ee/tabby-db/src/lib.rs +++ b/ee/tabby-db/src/lib.rs @@ -22,6 +22,7 @@ use user_completions::UserCompletionDailyStatsDAO; pub use user_events::UserEventDAO; pub use users::UserDAO; pub use web_crawler::WebCrawlerUrlDAO; +pub use web_documents::WebDocumentDAO; pub mod cache; mod email_setting; diff --git a/ee/tabby-schema/graphql/schema.graphql b/ee/tabby-schema/graphql/schema.graphql index 918cf99c869..09f706db746 100644 --- a/ee/tabby-schema/graphql/schema.graphql +++ b/ee/tabby-schema/graphql/schema.graphql @@ -99,6 +99,11 @@ input CodeSearchParamsOverrideInput { numToScore: Int } +input CreateCustomDocumentInput { + name: String! + url: String! +} + input CreateIntegrationInput { displayName: String! accessToken: String! @@ -188,6 +193,11 @@ input SecuritySettingInput { disableClientSideTelemetry: Boolean! } +input SetPresetDocumentActiveInput { + name: String! + active: Boolean! +} + input ThreadRunDebugOptionsInput { codeSearchParamsOverride: CodeSearchParamsOverrideInput = null } @@ -238,6 +248,24 @@ type CompletionStats { selects: Int! } +type CustomDocumentConnection { + edges: [CustomDocumentEdge!]! + pageInfo: PageInfo! +} + +type CustomDocumentEdge { + node: CustomWebDocument! + cursor: String! +} + +type CustomWebDocument { + url: String! + name: String! + id: ID! + createdAt: DateTime! + jobInfo: JobInfo! +} + type DiskUsage { filepath: [String!]! "Size in kilobytes." @@ -490,6 +518,9 @@ type Mutation { deleteWebCrawlerUrl(id: ID!): Boolean! "Delete pair of user message and bot response in a thread." deleteThreadMessagePair(threadId: ID!, userMessageId: ID!, assistantMessageId: ID!): Boolean! + createCustomDocument(input: CreateCustomDocumentInput!): ID! + deleteCustomDocument(id: ID!): Boolean! + setPresetDocumentActive(input: SetPresetDocumentActiveInput!): ID! } type NetworkSetting { @@ -510,6 +541,25 @@ type PageInfo { endCursor: String } +type PresetDocumentConnection { + edges: [PresetDocumentEdge!]! + pageInfo: PageInfo! +} + +type PresetDocumentEdge { + node: PresetWebDocument! + cursor: String! +} + +type PresetWebDocument { + name: String! + id: ID! + active: Boolean! + "`updated_at` is only filled when the preset is active." + updatedAt: DateTime + jobInfo: JobInfo +} + type ProvidedRepository { id: ID! integrationId: ID! @@ -581,6 +631,8 @@ type Query { Thread is public within an instance, so no need to check for ownership. """ threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection! + customWebDocuments(after: String, before: String, first: Int, last: Int): CustomDocumentConnection! + presetWebDocuments(after: String, before: String, first: Int, last: Int, active: Boolean!): PresetDocumentConnection! } type RefreshTokenResponse { diff --git a/ee/tabby-schema/src/schema/constants.rs b/ee/tabby-schema/src/schema/constants.rs index 66270f823e2..98c9f227edb 100644 --- a/ee/tabby-schema/src/schema/constants.rs +++ b/ee/tabby-schema/src/schema/constants.rs @@ -5,6 +5,7 @@ lazy_static! { pub static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[a-zA-Z][\\w.-]+$").unwrap(); pub static ref USERNAME_REGEX: Regex = Regex::new(r"^[^0-9±!@£$%^&*_+§¡€#¢¶•ªº«\\/<>?:;|=.,]{2,20}$").unwrap(); + pub static ref WEB_DOCUMENT_NAME_REGEX: Regex = Regex::new(r"^[A-Za-z][A-Za-z0-9\ ]*$").unwrap(); } #[cfg(test)] @@ -40,4 +41,22 @@ mod tests { assert_eq!(result, expected, "Failed for name: {}", name); } } + + #[test] + fn test_web_document_name_regex() { + let test_cases = vec![ + ("John", true), // English name + ("Müller", false), // German name + ("abc123", true), + ("Abc 123", true), + (" abc 123", false), + ("abc123*", false), + ("abc123_", false), + ]; + + for (name, expected) in test_cases { + let result = WEB_DOCUMENT_NAME_REGEX.is_match(name); + assert_eq!(result, expected, "Failed for name: {}", name); + } + } } diff --git a/ee/tabby-schema/src/schema/mod.rs b/ee/tabby-schema/src/schema/mod.rs index c0bcf954787..0e75f0cb49c 100644 --- a/ee/tabby-schema/src/schema/mod.rs +++ b/ee/tabby-schema/src/schema/mod.rs @@ -10,6 +10,7 @@ pub mod setting; pub mod thread; pub mod user_event; pub mod web_crawler; +pub mod web_documents; pub mod worker; use std::sync::Arc; @@ -51,7 +52,9 @@ use self::{ }, user_event::{UserEvent, UserEventService}, web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl}, + web_documents::{CreateCustomDocumentInput, CustomWebDocument, WebDocumentService}, }; +use crate::web_documents::{PresetWebDocument, SetPresetDocumentActiveInput}; use crate::{ env, juniper::relay::{self, query_async, Connection}, @@ -71,6 +74,7 @@ pub trait ServiceLocator: Send + Sync { fn analytic(&self) -> Arc; fn user_event(&self) -> Arc; fn web_crawler(&self) -> Arc; + fn web_documents(&self) -> Arc; fn thread(&self) -> Arc; } @@ -580,6 +584,50 @@ impl Query { ) .await } + + async fn custom_web_documents( + ctx: &Context, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + query_async( + after, + before, + first, + last, + |after, before, first, last| async move { + ctx.locator + .web_documents() + .list_custom_web_documents(after, before, first, last) + .await + }, + ) + .await + } + async fn preset_web_documents( + ctx: &Context, + after: Option, + before: Option, + first: Option, + last: Option, + active: bool, + ) -> Result> { + query_async( + after, + before, + first, + last, + |after, before, first, last| async move { + ctx.locator + .web_documents() + .list_preset_web_documents(after, before, first, last, active) + .await + }, + ) + .await + } } #[derive(GraphQLObject)] @@ -959,6 +1007,37 @@ impl Mutation { .await?; Ok(true) } + + async fn create_custom_document(ctx: &Context, input: CreateCustomDocumentInput) -> Result { + input.validate()?; + let id = ctx + .locator + .web_documents() + .create_custom_web_document(input.name, input.url) + .await?; + Ok(id) + } + + async fn delete_custom_document(ctx: &Context, id: ID) -> Result { + ctx.locator + .web_documents() + .delete_custom_web_document(id) + .await?; + Ok(true) + } + + async fn set_preset_document_active( + ctx: &Context, + input: SetPresetDocumentActiveInput, + ) -> Result { + input.validate()?; + let id = ctx + .locator + .web_documents() + .set_preset_web_documents_active(input.name, input.active) + .await?; + Ok(id) + } } async fn check_analytic_access(ctx: &Context, users: &[ID]) -> Result<(), CoreError> { diff --git a/ee/tabby-schema/src/schema/web_documents.rs b/ee/tabby-schema/src/schema/web_documents.rs new file mode 100644 index 00000000000..523016595d7 --- /dev/null +++ b/ee/tabby-schema/src/schema/web_documents.rs @@ -0,0 +1,115 @@ +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use juniper::{GraphQLInputObject, GraphQLObject, ID}; +use validator::Validate; + +use crate::{job::JobInfo, juniper::relay::NodeType, Context, Result}; + +#[derive(GraphQLObject)] +#[graphql(context = Context)] +pub struct CustomWebDocument { + pub url: String, + pub name: String, + pub id: ID, + pub created_at: DateTime, + pub job_info: JobInfo, +} + +#[derive(GraphQLObject)] +#[graphql(context = Context)] +pub struct PresetWebDocument { + pub name: String, + pub id: ID, + pub active: bool, + /// `updated_at` is only filled when the preset is active. + pub updated_at: Option>, + pub job_info: Option, +} + +impl CustomWebDocument { + pub fn source_id(&self) -> String { + Self::format_source_id(&self.id) + } + + pub fn format_source_id(id: &ID) -> String { + format!("web_document:{}", id) + } +} + +#[derive(Validate, GraphQLInputObject)] +pub struct CreateCustomDocumentInput { + #[validate(regex( + code = "name", + path = "*crate::schema::constants::WEB_DOCUMENT_NAME_REGEX", + message = "Invalid document name" + ))] + pub name: String, + #[validate(url(code = "url", message = "Invalid URL"))] + pub url: String, +} + +#[derive(Validate, GraphQLInputObject)] +pub struct SetPresetDocumentActiveInput { + #[validate(regex( + code = "name", + path = "*crate::schema::constants::WEB_DOCUMENT_NAME_REGEX", + message = "Invalid document name" + ))] + pub name: String, + pub active: bool, +} + +impl NodeType for CustomWebDocument { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.id.to_string() + } + + fn connection_type_name() -> &'static str { + "CustomDocumentConnection" + } + + fn edge_type_name() -> &'static str { + "CustomDocumentEdge" + } +} + +impl NodeType for PresetWebDocument { + type Cursor = String; + + fn cursor(&self) -> Self::Cursor { + self.name.clone() + } + + fn connection_type_name() -> &'static str { + "PresetDocumentConnection" + } + + fn edge_type_name() -> &'static str { + "PresetDocumentEdge" + } +} + +#[async_trait] +pub trait WebDocumentService: Send + Sync { + async fn list_custom_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result>; + + async fn create_custom_web_document(&self, name: String, url: String) -> Result; + async fn delete_custom_web_document(&self, id: ID) -> Result<()>; + async fn list_preset_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + active: bool, + ) -> Result>; + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result; +} diff --git a/ee/tabby-webserver/src/service/mod.rs b/ee/tabby-webserver/src/service/mod.rs index f9e5c10def3..e36a8c33ca0 100644 --- a/ee/tabby-webserver/src/service/mod.rs +++ b/ee/tabby-webserver/src/service/mod.rs @@ -12,6 +12,7 @@ mod setting; mod thread; mod user_event; pub mod web_crawler; +pub mod web_documents; use std::sync::Arc; @@ -43,6 +44,7 @@ use tabby_schema::{ thread::ThreadService, user_event::UserEventService, web_crawler::WebCrawlerService, + web_documents::WebDocumentService, worker::WorkerService, AsID, AsRowid, CoreError, Result, ServiceLocator, }; @@ -60,6 +62,7 @@ struct ServerContext { user_event: Arc, job: Arc, web_crawler: Arc, + web_documents: Arc, thread: Arc, logger: Arc, @@ -77,6 +80,7 @@ impl ServerContext { repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, answer: Option>, db_conn: DbConn, @@ -105,6 +109,7 @@ impl ServerContext { setting.clone(), )), web_crawler, + web_documents, thread, license, repository, @@ -260,6 +265,10 @@ impl ServiceLocator for ArcServerContext { self.0.web_crawler.clone() } + fn web_documents(&self) -> Arc { + self.0.web_documents.clone() + } + fn thread(&self) -> Arc { self.0.thread.clone() } @@ -271,6 +280,7 @@ pub async fn create_service_locator( repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, answer: Option>, db: DbConn, @@ -283,6 +293,7 @@ pub async fn create_service_locator( repository, integration, web_crawler, + web_documents, job, answer, db, diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs new file mode 100644 index 00000000000..8e1086e0f13 --- /dev/null +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -0,0 +1,57 @@ +use std::sync::Arc; + +use super::{background_job::BackgroundJobEvent, graphql_pagination_to_filter}; +use async_trait::async_trait; +use juniper::ID; +use tabby_db::{DbConn, WebDocumentDAO}; +use tabby_schema::web_documents::PresetWebDocument; +use tabby_schema::{ + job::{JobInfo, JobService}, + web_documents::{CustomWebDocument, WebDocumentService}, + AsID, AsRowid, Result, +}; + +pub fn create(db: DbConn, job_service: Arc) -> impl WebDocumentService { + WebDocumentServiceImpl { db, job_service } +} + +struct WebDocumentServiceImpl { + db: DbConn, + job_service: Arc, +} + +#[async_trait] +impl WebDocumentService for WebDocumentServiceImpl { + async fn list_custom_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + ) -> Result> { + Ok(vec![]) + } + + async fn create_custom_web_document(&self, name: String, url: String) -> Result { + Ok(ID::new("0")) + } + + async fn delete_custom_web_document(&self, id: ID) -> Result<()> { + Ok(()) + } + + async fn list_preset_web_documents( + &self, + after: Option, + before: Option, + first: Option, + last: Option, + active: bool, + ) -> Result> { + Ok(vec![]) + } + + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result { + Ok(ID::new("0")) + } +} diff --git a/ee/tabby-webserver/src/webserver.rs b/ee/tabby-webserver/src/webserver.rs index 98a53520bdb..cf857a8613c 100644 --- a/ee/tabby-webserver/src/webserver.rs +++ b/ee/tabby-webserver/src/webserver.rs @@ -1,5 +1,13 @@ use std::sync::Arc; +use crate::{ + path::db_file, + routes, + service::{ + background_job, create_service_locator, event_logger::create_event_logger, integration, + job, repository, web_crawler, web_documents, + }, +}; use axum::Router; use tabby_common::{ api::{ @@ -11,26 +19,19 @@ use tabby_common::{ }; use tabby_db::DbConn; use tabby_inference::{ChatCompletionStream, Embedding}; +use tabby_schema::web_documents::WebDocumentService; use tabby_schema::{ integration::IntegrationService, job::JobService, repository::RepositoryService, web_crawler::WebCrawlerService, }; -use crate::{ - path::db_file, - routes, - service::{ - background_job, create_service_locator, event_logger::create_event_logger, integration, - job, repository, web_crawler, - }, -}; - pub struct Webserver { db: DbConn, logger: Arc, repository: Arc, integration: Arc, web_crawler: Arc, + web_documents: Arc, job: Arc, } @@ -66,6 +67,7 @@ impl Webserver { let repository = repository::create(db.clone(), integration.clone(), job.clone()); let web_crawler = Arc::new(web_crawler::create(db.clone(), job.clone())); + let web_documents = Arc::new(web_documents::create(db.clone(), job.clone())); let logger2 = create_event_logger(db.clone()); let logger = Arc::new(ComposedLogger::new(logger1, logger2)); @@ -75,6 +77,7 @@ impl Webserver { repository: repository.clone(), integration: integration.clone(), web_crawler: web_crawler.clone(), + web_documents: web_documents.clone(), job: job.clone(), }); @@ -126,6 +129,7 @@ impl Webserver { self.repository.clone(), self.integration.clone(), self.web_crawler.clone(), + self.web_documents.clone(), self.job.clone(), answer.clone(), self.db.clone(), From 37784fa161933ea7102c63ffda25114bc06ca294 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:21:46 +0000 Subject: [PATCH 09/12] [autofix.ci] apply automated fixes --- ee/tabby-schema/src/schema/constants.rs | 3 ++- ee/tabby-schema/src/schema/mod.rs | 2 +- .../src/service/web_documents.rs | 10 ++++------ ee/tabby-webserver/src/webserver.rs | 20 +++++++++---------- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/ee/tabby-schema/src/schema/constants.rs b/ee/tabby-schema/src/schema/constants.rs index 98c9f227edb..75a87acb557 100644 --- a/ee/tabby-schema/src/schema/constants.rs +++ b/ee/tabby-schema/src/schema/constants.rs @@ -5,7 +5,8 @@ lazy_static! { pub static ref REPOSITORY_NAME_REGEX: Regex = Regex::new("^[a-zA-Z][\\w.-]+$").unwrap(); pub static ref USERNAME_REGEX: Regex = Regex::new(r"^[^0-9±!@£$%^&*_+§¡€#¢¶•ªº«\\/<>?:;|=.,]{2,20}$").unwrap(); - pub static ref WEB_DOCUMENT_NAME_REGEX: Regex = Regex::new(r"^[A-Za-z][A-Za-z0-9\ ]*$").unwrap(); + pub static ref WEB_DOCUMENT_NAME_REGEX: Regex = + Regex::new(r"^[A-Za-z][A-Za-z0-9\ ]*$").unwrap(); } #[cfg(test)] diff --git a/ee/tabby-schema/src/schema/mod.rs b/ee/tabby-schema/src/schema/mod.rs index 0e75f0cb49c..b184698d7b0 100644 --- a/ee/tabby-schema/src/schema/mod.rs +++ b/ee/tabby-schema/src/schema/mod.rs @@ -54,10 +54,10 @@ use self::{ web_crawler::{CreateWebCrawlerUrlInput, WebCrawlerService, WebCrawlerUrl}, web_documents::{CreateCustomDocumentInput, CustomWebDocument, WebDocumentService}, }; -use crate::web_documents::{PresetWebDocument, SetPresetDocumentActiveInput}; use crate::{ env, juniper::relay::{self, query_async, Connection}, + web_documents::{PresetWebDocument, SetPresetDocumentActiveInput}, }; pub trait ServiceLocator: Send + Sync { diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs index 8e1086e0f13..561871ba593 100644 --- a/ee/tabby-webserver/src/service/web_documents.rs +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -1,16 +1,14 @@ use std::sync::Arc; -use super::{background_job::BackgroundJobEvent, graphql_pagination_to_filter}; use async_trait::async_trait; use juniper::ID; -use tabby_db::{DbConn, WebDocumentDAO}; -use tabby_schema::web_documents::PresetWebDocument; +use tabby_db::DbConn; use tabby_schema::{ - job::{JobInfo, JobService}, - web_documents::{CustomWebDocument, WebDocumentService}, - AsID, AsRowid, Result, + job::JobService, + web_documents::{CustomWebDocument, PresetWebDocument, WebDocumentService}, Result, }; + pub fn create(db: DbConn, job_service: Arc) -> impl WebDocumentService { WebDocumentServiceImpl { db, job_service } } diff --git a/ee/tabby-webserver/src/webserver.rs b/ee/tabby-webserver/src/webserver.rs index cf857a8613c..e9d358d2939 100644 --- a/ee/tabby-webserver/src/webserver.rs +++ b/ee/tabby-webserver/src/webserver.rs @@ -1,13 +1,5 @@ use std::sync::Arc; -use crate::{ - path::db_file, - routes, - service::{ - background_job, create_service_locator, event_logger::create_event_logger, integration, - job, repository, web_crawler, web_documents, - }, -}; use axum::Router; use tabby_common::{ api::{ @@ -19,10 +11,18 @@ use tabby_common::{ }; use tabby_db::DbConn; use tabby_inference::{ChatCompletionStream, Embedding}; -use tabby_schema::web_documents::WebDocumentService; use tabby_schema::{ integration::IntegrationService, job::JobService, repository::RepositoryService, - web_crawler::WebCrawlerService, + web_crawler::WebCrawlerService, web_documents::WebDocumentService, +}; + +use crate::{ + path::db_file, + routes, + service::{ + background_job, create_service_locator, event_logger::create_event_logger, integration, + job, repository, web_crawler, web_documents, + }, }; pub struct Webserver { From bd47f52b85b74ff356cce43a01d548626b303d8e Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 06:28:32 +0000 Subject: [PATCH 10/12] [autofix.ci] apply automated fixes (attempt 2/3) --- ee/tabby-webserver/src/service/web_documents.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs index 561871ba593..4aa5cbbe382 100644 --- a/ee/tabby-webserver/src/service/web_documents.rs +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -5,10 +5,10 @@ use juniper::ID; use tabby_db::DbConn; use tabby_schema::{ job::JobService, - web_documents::{CustomWebDocument, PresetWebDocument, WebDocumentService}, Result, + web_documents::{CustomWebDocument, PresetWebDocument, WebDocumentService}, + Result, }; - pub fn create(db: DbConn, job_service: Arc) -> impl WebDocumentService { WebDocumentServiceImpl { db, job_service } } From 06cc963fd55c579097f8b53ec707dc89146cc39b Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Tue, 20 Aug 2024 12:46:21 +0800 Subject: [PATCH 11/12] feat(document): support crawl preset document (#2907) * finish api Signed-off-by: xxs-wallace * fix delete by id Signed-off-by: xxs-wallace * fix active Signed-off-by: xxs-wallace * fix name Signed-off-by: xxs-wallace * fix api Signed-off-by: xxs-wallace * add ut Signed-off-by: xxs-wallace --------- Signed-off-by: xxs-wallace --- ee/tabby-db/src/web_documents.rs | 16 +- ee/tabby-schema/graphql/schema.graphql | 9 +- ee/tabby-schema/src/schema/constants.rs | 3 +- ee/tabby-schema/src/schema/mod.rs | 11 +- ee/tabby-schema/src/schema/web_documents.rs | 12 +- ee/tabby-webserver/src/service/mod.rs | 1 + .../src/service/preset_web_documents_data.rs | 384 ++++++++++++++++++ .../src/service/web_documents.rs | 228 ++++++++++- 8 files changed, 634 insertions(+), 30 deletions(-) create mode 100644 ee/tabby-webserver/src/service/preset_web_documents_data.rs diff --git a/ee/tabby-db/src/web_documents.rs b/ee/tabby-db/src/web_documents.rs index 02326cac806..c361cc485be 100644 --- a/ee/tabby-db/src/web_documents.rs +++ b/ee/tabby-db/src/web_documents.rs @@ -1,4 +1,4 @@ -use anyhow::Result; +use anyhow::{anyhow, Result}; use chrono::{DateTime, Utc}; use sqlx::{prelude::FromRow, query}; use tabby_db_macros::query_paged_as; @@ -22,9 +22,9 @@ impl DbConn { limit: Option, skip_id: Option, backwards: bool, - is_preset: Option, + is_preset: bool, ) -> Result> { - let condition = is_preset.map(|is_preset| format!("is_preset={}", is_preset)); + let condition = Some(format!("is_preset={}", is_preset)); let urls = query_paged_as!( WebDocumentDAO, @@ -58,6 +58,16 @@ impl DbConn { Ok(res.last_insert_rowid()) } + pub async fn deactivate_preset_web_document(&self, name: String) -> Result<()> { + let res = query!("DELETE FROM web_documents WHERE name = ?;", name) + .execute(&self.pool) + .await?; + if res.rows_affected() != 1 { + return Err(anyhow!("No preset web document to deactivate")); + } + Ok(()) + } + pub async fn delete_web_document(&self, id: i64) -> Result<()> { query!("DELETE FROM web_documents WHERE id = ?;", id) .execute(&self.pool) diff --git a/ee/tabby-schema/graphql/schema.graphql b/ee/tabby-schema/graphql/schema.graphql index 09f706db746..f1b9b72b5ba 100644 --- a/ee/tabby-schema/graphql/schema.graphql +++ b/ee/tabby-schema/graphql/schema.graphql @@ -263,6 +263,7 @@ type CustomWebDocument { name: String! id: ID! createdAt: DateTime! + updatedAt: DateTime! jobInfo: JobInfo! } @@ -520,7 +521,7 @@ type Mutation { deleteThreadMessagePair(threadId: ID!, userMessageId: ID!, assistantMessageId: ID!): Boolean! createCustomDocument(input: CreateCustomDocumentInput!): ID! deleteCustomDocument(id: ID!): Boolean! - setPresetDocumentActive(input: SetPresetDocumentActiveInput!): ID! + setPresetDocumentActive(input: SetPresetDocumentActiveInput!): Boolean! } type NetworkSetting { @@ -552,11 +553,11 @@ type PresetDocumentEdge { } type PresetWebDocument { - name: String! id: ID! - active: Boolean! + name: String! "`updated_at` is only filled when the preset is active." updatedAt: DateTime + "`job_info` is only filled when the preset is active." jobInfo: JobInfo } @@ -632,7 +633,7 @@ type Query { """ threadMessages(threadId: ID!, after: String, before: String, first: Int, last: Int): MessageConnection! customWebDocuments(after: String, before: String, first: Int, last: Int): CustomDocumentConnection! - presetWebDocuments(after: String, before: String, first: Int, last: Int, active: Boolean!): PresetDocumentConnection! + presetWebDocuments(after: String, before: String, first: Int, last: Int, isActive: Boolean!): PresetDocumentConnection! } type RefreshTokenResponse { diff --git a/ee/tabby-schema/src/schema/constants.rs b/ee/tabby-schema/src/schema/constants.rs index 75a87acb557..ca2931effdc 100644 --- a/ee/tabby-schema/src/schema/constants.rs +++ b/ee/tabby-schema/src/schema/constants.rs @@ -6,7 +6,7 @@ lazy_static! { pub static ref USERNAME_REGEX: Regex = Regex::new(r"^[^0-9±!@£$%^&*_+§¡€#¢¶•ªº«\\/<>?:;|=.,]{2,20}$").unwrap(); pub static ref WEB_DOCUMENT_NAME_REGEX: Regex = - Regex::new(r"^[A-Za-z][A-Za-z0-9\ ]*$").unwrap(); + Regex::new(r"^[A-Za-z][A-Za-z0-9]*(\ [A-Za-z0-9]+)*$").unwrap(); } #[cfg(test)] @@ -53,6 +53,7 @@ mod tests { (" abc 123", false), ("abc123*", false), ("abc123_", false), + ("abc 123", false), // two space ]; for (name, expected) in test_cases { diff --git a/ee/tabby-schema/src/schema/mod.rs b/ee/tabby-schema/src/schema/mod.rs index b184698d7b0..35411cff312 100644 --- a/ee/tabby-schema/src/schema/mod.rs +++ b/ee/tabby-schema/src/schema/mod.rs @@ -612,7 +612,7 @@ impl Query { before: Option, first: Option, last: Option, - active: bool, + is_active: bool, ) -> Result> { query_async( after, @@ -622,7 +622,7 @@ impl Query { |after, before, first, last| async move { ctx.locator .web_documents() - .list_preset_web_documents(after, before, first, last, active) + .list_preset_web_documents(after, before, first, last, is_active) .await }, ) @@ -1029,14 +1029,13 @@ impl Mutation { async fn set_preset_document_active( ctx: &Context, input: SetPresetDocumentActiveInput, - ) -> Result { + ) -> Result { input.validate()?; - let id = ctx - .locator + ctx.locator .web_documents() .set_preset_web_documents_active(input.name, input.active) .await?; - Ok(id) + Ok(true) } } diff --git a/ee/tabby-schema/src/schema/web_documents.rs b/ee/tabby-schema/src/schema/web_documents.rs index 523016595d7..eb958a2561b 100644 --- a/ee/tabby-schema/src/schema/web_documents.rs +++ b/ee/tabby-schema/src/schema/web_documents.rs @@ -12,17 +12,19 @@ pub struct CustomWebDocument { pub name: String, pub id: ID, pub created_at: DateTime, + pub updated_at: DateTime, pub job_info: JobInfo, } #[derive(GraphQLObject)] #[graphql(context = Context)] pub struct PresetWebDocument { - pub name: String, pub id: ID, - pub active: bool, + + pub name: String, /// `updated_at` is only filled when the preset is active. pub updated_at: Option>, + /// `job_info` is only filled when the preset is active. pub job_info: Option, } @@ -79,7 +81,7 @@ impl NodeType for PresetWebDocument { type Cursor = String; fn cursor(&self) -> Self::Cursor { - self.name.clone() + self.id.to_string() } fn connection_type_name() -> &'static str { @@ -109,7 +111,7 @@ pub trait WebDocumentService: Send + Sync { before: Option, first: Option, last: Option, - active: bool, + is_active: bool, ) -> Result>; - async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result; + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<()>; } diff --git a/ee/tabby-webserver/src/service/mod.rs b/ee/tabby-webserver/src/service/mod.rs index e36a8c33ca0..af0bb3c5a59 100644 --- a/ee/tabby-webserver/src/service/mod.rs +++ b/ee/tabby-webserver/src/service/mod.rs @@ -7,6 +7,7 @@ pub mod event_logger; pub mod integration; pub mod job; mod license; +mod preset_web_documents_data; pub mod repository; mod setting; mod thread; diff --git a/ee/tabby-webserver/src/service/preset_web_documents_data.rs b/ee/tabby-webserver/src/service/preset_web_documents_data.rs new file mode 100644 index 00000000000..02d83fe36c7 --- /dev/null +++ b/ee/tabby-webserver/src/service/preset_web_documents_data.rs @@ -0,0 +1,384 @@ +pub const PRESET_WEB_DOCUMENTS_DATA: &str = r#"[{ "name": "CodeMirror", "crawlerStart": "https://codemirror.net/docs/", "crawlerPrefix": "https://codemirror.net/docs/" }, +{ "name": "React", "crawlerStart": "https://react.dev/reference/", "crawlerPrefix": "https://react.dev/reference/" }, +{ "name": "Tailwind CSS", "crawlerStart": "https://tailwindcss.com/docs/", "crawlerPrefix": "https://tailwindcss.com/docs/" }, +{ "name": "Qwik", "crawlerStart": "https://qwik.builder.io/docs/", "crawlerPrefix": "https://qwik.builder.io/docs/" }, +{ "name": "Stripe", "crawlerStart": "https://stripe.com/docs/api", "crawlerPrefix": "https://stripe.com/docs/api" }, +{ "name": "PostHog", "crawlerStart": "https://posthog.com/docs", "crawlerPrefix": "https://posthog.com/docs" }, +{ "name": "Express", "crawlerStart": "https://expressjs.com/en/5x/api.html", "crawlerPrefix": "https://expressjs.com/en/5x/" }, +{ "name": "Boto3", "crawlerStart": "https://boto3.amazonaws.com/v1/documentation/api/latest/index.html", "crawlerPrefix": "https://boto3.amazonaws.com/v1/documentation/api/latest/" }, +{ "name": "Redux", "crawlerStart": "https://redux.js.org/api/", "crawlerPrefix": "https://redux.js.org/api/" }, +{ "name": "Electron", "crawlerStart": "https://www.electronjs.org/docs/latest/", "crawlerPrefix": "https://www.electronjs.org/docs/latest/" }, +{ "name": "Tauri", "crawlerStart": "https://tauri.app/v1/api/js/", "crawlerPrefix": "https://tauri.app/v1/api/js/" }, +{ "name": "selenium-python", "crawlerStart": "https://selenium-python.readthedocs.io/", "crawlerPrefix": "https://selenium-python.readthedocs.io/" }, +{ "name": "SolidJS", "crawlerStart": "https://www.solidjs.com/docs/", "crawlerPrefix": "https://www.solidjs.com/docs/" }, +{ "name": "Lexical", "crawlerStart": "https://lexical.dev/docs/concepts/editor-state", "crawlerPrefix": "https://lexical.dev/docs/concepts/editor-state" }, +{ "name": "Lexical Packages", "crawlerStart": "https://lexical.dev/docs/packages/lexical", "crawlerPrefix": "https://lexical.dev/docs/packages/lexical" }, +{ "name": "Prisma", "crawlerStart": "https://www.prisma.io/docs", "crawlerPrefix": "https://www.prisma.io/docs" }, +{ "name": "Kubernetes", "crawlerStart": "https://kubernetes.io/docs/", "crawlerPrefix": "https://kubernetes.io/docs/" }, +{ "name": "Sentry-Python", "crawlerStart": "https://docs.sentry.io/platforms/python/", "crawlerPrefix": "https://docs.sentry.io/platforms/python/" }, +{ "name": "Pytorch", "crawlerStart": "https://pytorch.org/docs/stable/", "crawlerPrefix": "https://pytorch.org/docs/stable/" }, +{ "name": "Transformers", "crawlerStart": "https://huggingface.co/docs/transformers/", "crawlerPrefix": "https://huggingface.co/docs/transformers/" }, +{ "name": "Hugging-Face-Transformers", "crawlerStart": "https://huggingface.co/docs/transformers/", "crawlerPrefix": "https://huggingface.co/docs/transformers/" }, +{ "name": "Langchain", "crawlerStart": "https://python.langchain.com/docs/", "crawlerPrefix": "https://python.langchain.com/docs/" }, +{ "name": "Vue", "crawlerStart": "https://vuejs.org/guide/", "crawlerPrefix": "https://vuejs.org/guide/" }, +{ "name": "Angular", "crawlerStart": "https://angular.io/docs", "crawlerPrefix": "https://angular.io/docs" }, +{ "name": "Astro", "crawlerStart": "https://docs.astro.build/en/", "crawlerPrefix": "https://docs.astro.build/en/" }, +{ "name": "Elixir", "crawlerStart": "https://elixir-lang.org/docs.html", "crawlerPrefix": "https://elixir-lang.org/docs.html" }, +{ "name": "Svelte", "crawlerStart": "https://svelte.dev/docs", "crawlerPrefix": "https://svelte.dev/docs" }, +{ "name": "SvelteKit", "crawlerStart": "https://kit.svelte.dev/docs/introduction", "crawlerPrefix": "https://kit.svelte.dev/docs" }, +{ "name": "PineconeDB", "crawlerStart": "https://docs.pinecone.io/docs/", "crawlerPrefix": "https://docs.pinecone.io/docs/" }, +{ "name": "Redis", "crawlerStart": "https://redis.io/docs/", "crawlerPrefix": "https://redis.io/docs/" }, +{ "name": "KeyDB", "crawlerStart": "https://docs.keydb.dev/docs/", "crawlerPrefix": "https://docs.keydb.dev/docs/" }, +{ "name": "Yugabyte", "crawlerStart": "https://docs.yugabyte.com/", "crawlerPrefix": "https://docs.yugabyte.com/" }, +{ "name": "Tensorflow", "crawlerStart": "https://www.tensorflow.org/api_docs/python/tf/", "crawlerPrefix": "https://www.tensorflow.org/api_docs/python/tf/" }, +{ "name": "Tmux", "crawlerStart": "https://tmuxguide.readthedocs.io/en/latest/tmux/tmux.html", "crawlerPrefix": "https://tmuxguide.readthedocs.io/en/latest/tmux/tmux.html" }, +{ "name": "FFmpeg", "crawlerStart": "https://ffmpeg.org/ffmpeg.html", "crawlerPrefix": "https://ffmpeg.org/ffmpeg.html" }, +{ "name": "Flutter", "crawlerStart": "https://docs.flutter.dev/", "crawlerPrefix": "https://docs.flutter.dev/" }, +{ "name": "Pandas", "crawlerStart": "https://pandas.pydata.org/docs/", "crawlerPrefix": "https://pandas.pydata.org/docs/" }, +{ "name": "Julia", "crawlerStart": "https://docs.julialang.org/en/v1/", "crawlerPrefix": "https://docs.julialang.org/en/v1/" }, +{ "name": "Django", "crawlerStart": "https://docs.djangoproject.com/en/4.2/", "crawlerPrefix": "https://docs.djangoproject.com/en/4.2/" }, +{ "name": "Flask", "crawlerStart": "https://flask.palletsprojects.com/en/2.3.x/", "crawlerPrefix": "https://flask.palletsprojects.com/en/2.3.x/" }, +{ "name": "Keras", "crawlerStart": "https://keras.io/api/", "crawlerPrefix": "https://keras.io/api/" }, +{ "name": "Langchain-JS", "crawlerStart": "https://js.langchain.com/docs/", "crawlerPrefix": "https://js.langchain.com/docs/" }, +{ "name": "requests", "crawlerStart": "https://requests.readthedocs.io/en/latest/api/", "crawlerPrefix": "https://requests.readthedocs.io/en/latest/api/" }, +{ "name": "ggplot2", "crawlerStart": "https://ggplot2.tidyverse.org/reference/", "crawlerPrefix": "https://ggplot2.tidyverse.org/reference/" }, +{ "name": "RubyOnRails", "crawlerStart": "https://api.rubyonrails.org/", "crawlerPrefix": "https://api.rubyonrails.org/" }, +{ "name": "Spark", "crawlerStart": "https://spark.apache.org/docs/latest/", "crawlerPrefix": "https://spark.apache.org/docs/latest/" }, +{ "name": "PySpark", "crawlerStart": "https://spark.apache.org/docs/latest/api/python/", "crawlerPrefix": "https://spark.apache.org/docs/latest/api/python/" }, +{ "name": "LanguageServerProtocol", "crawlerStart": "https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/", "crawlerPrefix": "https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/" }, +{ "name": "axios", "crawlerStart": "https://axios-http.com/docs/api_intro", "crawlerPrefix": "https://axios-http.com/docs/api_intro" }, +{ "name": "Jquery", "crawlerStart": "https://api.jqueryui.com/1.12/", "crawlerPrefix": "https://api.jqueryui.com/1.12/" }, +{ "name": "lodash", "crawlerStart": "https://lodash.com/docs/", "crawlerPrefix": "https://lodash.com/docs/" }, +{ "name": "java.swing", "crawlerStart": "https://docs.oracle.com/javase%2F7%2Fdocs%2Fapi%2F%2F/javax/swing/package-summary.html", "crawlerPrefix": "https://docs.oracle.com/javase%2F7%2Fdocs%2Fapi%2F%2F/javax/swing/" }, +{ "name": "Matplotlib", "crawlerStart": "https://matplotlib.org/stable/api/", "crawlerPrefix": "https://matplotlib.org/stable/api/" }, +{ "name": "NodeJS", "crawlerStart": "https://nodejs.org/api/", "crawlerPrefix": "https://nodejs.org/api/" }, +{ "name": "Tomcat", "crawlerStart": "https://tomcat.apache.org/tomcat-8.5-doc/", "crawlerPrefix": "https://tomcat.apache.org/tomcat-8.5-doc/" }, +{ "name": "redb", "crawlerStart": "https://docs.rs/redb/latest/redb/", "crawlerPrefix": "https://docs.rs/redb/latest/redb/" }, +{ "name": "OpenAI", "crawlerStart": "https://platform.openai.com/docs/", "crawlerPrefix": "https://platform.openai.com/docs/" }, +{ "name": "Notion", "crawlerStart": "https://developers.notion.com/reference/", "crawlerPrefix": "https://developers.notion.com/reference/" }, +{ "name": "GCP CLI", "crawlerStart": "https://cloud.google.com/sdk/docs", "crawlerPrefix": "https://cloud.google.com/sdk/docs" }, +{ "name": "AWS CLI", "crawlerStart": "https://docs.aws.amazon.com/cli/latest/reference/", "crawlerPrefix": "https://docs.aws.amazon.com/cli/latest/reference/" }, +{ "name": "React Native", "crawlerStart": "https://reactnative.dev/docs/appstate", "crawlerPrefix": "https://reactnative.dev/docs/" }, +{ "name": "Lottie", "crawlerStart": "http://airbnb.io/lottie/#/README", "crawlerPrefix": "http://airbnb.io/lottie/#/" }, +{ "name": "Expo", "crawlerStart": "https://docs.expo.dev/", "crawlerPrefix": "https://docs.expo.dev/" }, +{ "name": "Goreplay", "crawlerStart": "https://github.com/buger/goreplay/wiki", "crawlerPrefix": "https://github.com/buger/goreplay/wiki" }, +{ "name": "NextJS", "crawlerStart": "https://nextjs.org/docs", "crawlerPrefix": "https://nextjs.org/docs" }, +{ "name": "Jest", "crawlerStart": "https://jestjs.io/docs/getting-started", "crawlerPrefix": "https://jestjs.io/docs/getting-started" }, +{ "name": "Vite", "crawlerStart": "https://vitejs.dev/guide/", "crawlerPrefix": "https://vitejs.dev/guide/" }, +{ "name": "Webpack", "crawlerStart": "https://webpack.js.org/concepts/", "crawlerPrefix": "https://webpack.js.org/concepts/" }, +{ "name": "ESBuild", "crawlerStart": "https://esbuild.github.io/api/", "crawlerPrefix": "https://esbuild.github.io/api/" }, +{ "name": "D3", "crawlerStart": "https://d3js.org/getting-started", "crawlerPrefix": "https://d3js.org/" }, +{ "name": "Deno API", "crawlerStart": "https://deno.land/api@v1.35.0", "crawlerPrefix": "https://deno.land/api@v1.35.0" }, +{ "name": "Deno", "crawlerStart": "https://deno.land/manual@v1.35.0/introduction", "crawlerPrefix": "https://deno.land/manual@v1.35.0/" }, +{ "name": "Puppeteer", "crawlerStart": "https://pptr.dev/", "crawlerPrefix": "https://pptr.dev/" }, +{ "name": "Laravel", "crawlerStart": "https://laravel.com/docs/10.x", "crawlerPrefix": "https://laravel.com/docs/10.x" }, +{ "name": "FontAwesome", "crawlerStart": "https://fontawesome.com/docs/web/", "crawlerPrefix": "https://fontawesome.com/docs/web/" }, +{ "name": "OpenCV", "crawlerStart": "https://docs.opencv.org/4.x/", "crawlerPrefix": "https://docs.opencv.org/4.x/" }, +{ "name": "Nginx", "crawlerStart": "http://nginx.org/en/docs/", "crawlerPrefix": "http://nginx.org/en/docs/" }, +{ "name": "Hugo", "crawlerStart": "https://gohugo.io/documentation/", "crawlerPrefix": "https://gohugo.io/documentation/" }, +{ "name": "MongoDB", "crawlerStart": "https://www.mongodb.com/docs/manual/", "crawlerPrefix": "https://www.mongodb.com/docs/manual/" }, +{ "name": "Phoenix", "crawlerStart": "https://hexdocs.pm/phoenix/", "crawlerPrefix": "https://hexdocs.pm/phoenix/" }, +{ "name": "Spring Boot", "crawlerStart": "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/", "crawlerPrefix": "https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/" }, +{ "name": "FastAPI", "crawlerStart": "https://fastapi.tiangolo.com/tutorial/", "crawlerPrefix": "https://fastapi.tiangolo.com/tutorial/" }, +{ "name": "Nuxt", "crawlerStart": "https://nuxt.com/docs", "crawlerPrefix": "https://nuxt.com/docs" }, +{ "name": "Postgres", "crawlerStart": "https://www.postgresql.org/docs/current/index.html", "crawlerPrefix": "https://www.postgresql.org/docs/current/" }, +{ "name": "Streamlit", "crawlerStart": "https://docs.streamlit.io/", "crawlerPrefix": "https://docs.streamlit.io/" }, +{ "name": "AngularJS", "crawlerStart": "https://docs.angularjs.org/guide/", "crawlerPrefix": "https://docs.angularjs.org/guide/" }, +{ "name": "Fontello", "crawlerStart": "https://github.com/fontello/fontello/wiki/", "crawlerPrefix": "https://github.com/fontello/fontello/wiki/" }, +{ "name": "Go", "crawlerStart": "https://go.dev/doc", "crawlerPrefix": "https://go.dev/doc" }, +{ "name": "Maven", "crawlerStart": "https://maven.apache.org/guides/", "crawlerPrefix": "https://maven.apache.org/guides/" }, +{ "name": "Vagrant", "crawlerStart": "https://developer.hashicorp.com/vagrant/docs", "crawlerPrefix": "https://developer.hashicorp.com/vagrant/docs" }, +{ "name": "ESLint", "crawlerStart": "https://eslint.org/docs/latest/", "crawlerPrefix": "https://eslint.org/docs/latest/" }, +{ "name": "Swagger", "crawlerStart": "https://swagger.io/docs/", "crawlerPrefix": "https://swagger.io/docs/" }, +{ "name": "JUnit 5", "crawlerStart": "https://junit.org/junit5/docs/current/user-guide/", "crawlerPrefix": "https://junit.org/junit5/docs/current/user-guide/" }, +{ "name": "NUnit", "crawlerStart": "https://docs.nunit.org/", "crawlerPrefix": "https://docs.nunit.org/articles/" }, +{ "name": "TestNG", "crawlerStart": "https://testng.org/doc/documentation-main.html", "crawlerPrefix": "https://testng.org/doc/" }, +{ "name": "Jasmine", "crawlerStart": "https://jasmine.github.io/pages/docs_home.html/", "crawlerPrefix": "https://jasmine.github.io/api/" }, +{ "name": "Mocha", "crawlerStart": "https://mochajs.org/api/", "crawlerPrefix": "https://mochajs.org/api/" }, +{ "name": "Cypress", "crawlerStart": "https://docs.cypress.io/guides/overview/why-cypress", "crawlerPrefix": "https://docs.cypress.io/guides/" }, +{ "name": "TestComplete", "crawlerStart": "https://support.smartbear.com/testcomplete/docs/", "crawlerPrefix": "https://support.smartbear.com/testcomplete/docs/" }, +{ "name": "Mockito", "crawlerStart": "https://javadoc.io/doc/org.mockito/mockito-core/latest/index.html", "crawlerPrefix": "https://javadoc.io/doc/org.mockito/mockito-core/latest/" }, +{ "name": "PHPUnit", "crawlerStart": "https://docs.phpunit.de/en/10.2/", "crawlerPrefix": "https://docs.phpunit.de/en/10.2/" }, +{ "name": "JTest", "crawlerStart": "https://docs.parasoft.com/display/JTEST20231/About+this+User+Guide", "crawlerPrefix": "https://docs.parasoft.com/display/JTEST20231/" }, +{ "name": "GoogleTest", "crawlerStart": "https://google.github.io/googletest/", "crawlerPrefix": "https://google.github.io/googletest/" }, +{ "name": "Material UI", "crawlerStart": "https://mui.com/material-ui/getting-started/", "crawlerPrefix": "https://mui.com/material-ui/" }, +{ "name": "Chakra UI", "crawlerStart": "https://chakra-ui.com/getting-started", "crawlerPrefix": "https://chakra-ui.com/docs/" }, +{ "name": "Ant Design", "crawlerStart": "https://ant.design/docs/react/introduce", "crawlerPrefix": "https://ant.design/docs/react/" }, +{ "name": "React Suite", "crawlerStart": "https://rsuitejs.com/guide/introduction/", "crawlerPrefix": "https://rsuitejs.com/guide/" }, +{ "name": "React Bootstrap", "crawlerStart": "https://react-bootstrap.netlify.app/docs/getting-started/introduction", "crawlerPrefix": "https://react-bootstrap.netlify.app/docs/" }, +{ "name": "Semantic UI", "crawlerStart": "https://semantic-ui.com/behaviors/api.html", "crawlerPrefix": "https://semantic-ui.com/behaviors/api.html#" }, +{ "name": "Mantine React Table", "crawlerStart": "https://www.mantine-react-table.com/docs/getting-started", "crawlerPrefix": "https://www.mantine-react-table.com/docs/" }, +{ "name": "Mantine", "crawlerStart": "https://mantine.dev/pages/getting-started/", "crawlerPrefix": "https://mantine.dev/pages/" }, +{ "name": "Blueprint", "crawlerStart": "https://blueprintjs.com/docs/", "crawlerPrefix": "https://blueprintjs.com/docs/" }, +{ "name": "NextUI", "crawlerStart": "https://nextui.org/docs", "crawlerPrefix": "https://nextui.org/docs" }, +{ "name": "PrimeReact", "crawlerStart": "https://primereact.org/", "crawlerPrefix": "https://primereact.org/" }, +{ "name": "Grommet", "crawlerStart": "https://v2.grommet.io/docs", "crawlerPrefix": "https://v2.grommet.io/" }, +{ "name": "Onsen UI", "crawlerStart": "https://onsen.io/v2/api/css.html", "crawlerPrefix": "https://onsen.io/v2/api/" }, +{ "name": "Quasar", "crawlerStart": "hhttps://quasar.dev/docs", "crawlerPrefix": "https://quasar.dev/" }, +{ "name": "Vuetify", "crawlerStart": "https://vuetifyjs.com/en/getting-started/installation/", "crawlerPrefix": "https://vuetifyjs.com/en/" }, +{ "name": "Bootstrap Vue", "crawlerStart": "https://bootstrap-vue.org/docs", "crawlerPrefix": "https://bootstrap-vue.org/docs" }, +{ "name": "Element Plus", "crawlerStart": "https://element-plus.org/#/en-US", "crawlerPrefix": "https://element-plus.org/#/en-US" }, +{ "name": "Keen UI", "crawlerStart": "https://josephuspaye.github.io/Keen-UI/#/ui-alert", "crawlerPrefix": "https://josephuspaye.github.io/Keen-UI/#/" }, +{ "name": "Equal UI", "crawlerStart": "https://equal-ui.github.io/Equal/", "crawlerPrefix": "https://equal-ui.github.io/Equal/" }, +{ "name": "Anti-Design Vue", "crawlerStart": "https://antdv.com/components/overview/", "crawlerPrefix": "https://antdv.com/components/" }, +{ "name": "Prime Vue", "crawlerStart": "https://primevue.org/", "crawlerPrefix": "https://primevue.org/" }, +{ "name": "Fish UI", "crawlerStart": "https://myliang.github.io/fish-ui/#/components/index", "crawlerPrefix": "https://myliang.github.io/fish-ui/#/components/" }, +{ "name": "CoreUI Vue", "crawlerStart": "https://coreui.io/vue/docs/", "crawlerPrefix": "https://coreui.io/vue/docs/" }, +{ "name": "Vuesax", "crawlerStart": "https://vuesax.com/docs/", "crawlerPrefix": "https://vuesax.com/docs/" }, +{ "name": "Vue Material", "crawlerStart": "https://www.creative-tim.com/vuematerial/getting-started", "crawlerPrefix": "https://www.creative-tim.com/vuematerial/" }, +{ "name": "Buefy", "crawlerStart": "https://buefy.org/documentation/", "crawlerPrefix": "https://buefy.org/documentation/" }, +{ "name": "Meteor", "crawlerStart": "https://docs.meteor.com/", "crawlerPrefix": "https://docs.meteor.com/" }, +{ "name": "Koajs", "crawlerStart": "https://koajs.com/#", "crawlerPrefix": "https://koajs.com/#" }, +{ "name": "Browserify", "crawlerStart": "https://github.com/browserify/browserify#", "crawlerPrefix": "https://github.com/browserify/browserify#" }, +{ "name": "Underscore", "crawlerStart": "https://underscorejs.org/#", "crawlerPrefix": "https://underscorejs.org/#" }, +{ "name": "Mongoose", "crawlerStart": "https://mongoosejs.com/docs/guide.html", "crawlerPrefix": "https://mongoosejs.com/docs/" }, +{ "name": "Cors", "crawlerStart": "https://expressjs.com/en/resources/middleware/cors.html", "crawlerPrefix": "https://expressjs.com/en/resources/middleware/cors.html" }, +{ "name": "Dotenv", "crawlerStart": "https://github.com/motdotla/dotenv", "crawlerPrefix": "https://github.com/motdotla/dotenv" }, +{ "name": "Gulp", "crawlerStart": "https://gulpjs.com/docs/en/getting-started/quick-start", "crawlerPrefix": "https://gulpjs.com/docs/en/" }, +{ "name": "Async", "crawlerStart": "https://caolan.github.io/async/v3/docs.html", "crawlerPrefix": "https://caolan.github.io/async/v3/docs.html#" }, +{ "name": "Moment JS", "crawlerStart": "https://momentjs.com/docs/", "crawlerPrefix": "https://momentjs.com/docs/#/" }, +{ "name": "Cheerio", "crawlerStart": "https://cheerio.js.org/docs/intro", "crawlerPrefix": "https://cheerio.js.org/docs/" }, +{ "name": "Nodemailer", "crawlerStart": "https://nodemailer.com/", "crawlerPrefix": "https://nodemailer.com/" }, +{ "name": "Selenium", "crawlerStart": "https://www.selenium.dev/documentation/", "crawlerPrefix": "https://www.selenium.dev/documentation/" }, +{ "name": "Katalon", "crawlerStart": "https://docs.katalon.com/", "crawlerPrefix": "https://docs.katalon.com/docs/" }, +{ "name": "Katalon TestOps API", "crawlerStart": "https://developers.katalon.com/reference/api-reference", "crawlerPrefix": "https://developers.katalon.com/reference/" }, +{ "name": "Appium", "crawlerStart": "http://appium.io/docs/en/2.0/", "crawlerPrefix": "http://appium.io/docs/en/2.0/" }, +{ "name": "Ranorex", "crawlerStart": "https://support.ranorex.com/help/latest/ranorex-studio-fundamentals/", "crawlerPrefix": "https://support.ranorex.com/help/latest/" }, +{ "name": "Cucumber", "crawlerStart": "https://cucumber.io/docs/cucumber/", "crawlerPrefix": "https://cucumber.io/docs/cucumber/" }, +{ "name": "SoapUI", "crawlerStart": "https://www.soapui.org/docs/", "crawlerPrefix": "https://www.soapui.org/docs/" }, +{ "name": "ReadyAPI", "crawlerStart": "https://support.smartbear.com/readyapi/docs/", "crawlerPrefix": "https://support.smartbear.com/readyapi/docs/" }, +{ "name": "Watir", "crawlerStart": "http://watir.com/guides/", "crawlerPrefix": "http://watir.com/guides/" }, +{ "name": "BrowserStack", "crawlerStart": "https://www.browserstack.com/docs", "crawlerPrefix": "https://www.browserstack.com/docs/automate/" }, +{ "name": "TestProject", "crawlerStart": "https://docs.testproject.io/testproject-sdk/overview", "crawlerPrefix": "https://docs.testproject.io/testproject-sdk/" }, +{ "name": "Apache JMeter", "crawlerStart": "https://jmeter.apache.org/usermanual/index.html", "crawlerPrefix": "https://jmeter.apache.org/usermanual/" }, +{ "name": "SauceLabs", "crawlerStart": "https://docs.saucelabs.com/", "crawlerPrefix": "https://docs.saucelabs.com/" }, +{ "name": "Playwright", "crawlerStart": "https://playwright.dev/docs/intro", "crawlerPrefix": "https://playwright.dev/docs/" }, +{ "name": "Espresso", "crawlerStart": "https://developer.android.com/training/testing/espresso/", "crawlerPrefix": "https://developer.android.com/training/testing/espresso/" }, +{ "name": "EarlGrey", "crawlerStart": "https://google.github.io/EarlGrey/", "crawlerPrefix": "https://google.github.io/EarlGrey/" }, +{ "name": "Detox Android", "crawlerStart": "https://wix.github.io/Detox/docs/introduction/getting-started/", "crawlerPrefix": "https://wix.github.io/Detox/docs/" }, +{ "name": "Jenkins", "crawlerStart": "https://www.jenkins.io/doc/", "crawlerPrefix": "https://www.jenkins.io/doc/" }, +{ "name": "Bamboo", "crawlerStart": "https://docs.atlassian.com/bamboo-specs/8.1.12/", "crawlerPrefix": "https://docs.atlassian.com/bamboo-specs/8.1.12/" }, +{ "name": "TeamCity", "crawlerStart": "https://www.jetbrains.com/help/teamcity/teamcity-documentation.html", "crawlerPrefix": "https://www.jetbrains.com/help/teamcity/" }, +{ "name": "Azure Pipelines", "crawlerStart": "https://docs.microsoft.com/en-us/azure/devops/pipelines/?view=azure-devops", "crawlerPrefix": "https://docs.microsoft.com/en-us/azure/devops/pipelines/" }, +{ "name": "CircleCI", "crawlerStart": "https://circleci.com/docs/", "crawlerPrefix": "https://circleci.com/docs/" }, +{ "name": "Bitbucket Pipelines", "crawlerStart": "https://support.atlassian.com/bitbucket-cloud/docs/get-started-with-bitbucket-pipelines/", "crawlerPrefix": "https://support.atlassian.com/bitbucket-cloud/docs/" }, +{ "name": "Travis CI", "crawlerStart": "https://docs.travis-ci.com/", "crawlerPrefix": "https://docs.travis-ci.com/user/" }, +{ "name": "GitHub Actions", "crawlerStart": "https://docs.github.com/en/actions", "crawlerPrefix": "https://docs.github.com/en/actions" }, +{ "name": "GitLab CI", "crawlerStart": "https://docs.gitlab.com/ee/ci/", "crawlerPrefix": "https://docs.gitlab.com/ee/ci/" }, +{ "name": "AWS CodePipeline CI", "crawlerStart": "https://docs.aws.amazon.com/codepipeline/latest/userguide/welcome.html", "crawlerPrefix": "https://docs.aws.amazon.com/codepipeline/latest/userguide/" }, +{ "name": "Drone", "crawlerStart": "https://docs.drone.io/", "crawlerPrefix": "https://docs.drone.io/" }, +{ "name": "Java", "crawlerStart": "https://docs.oracle.com/javase/8/docs/api/", "crawlerPrefix": "https://docs.oracle.com/javase/8/docs/api/" }, +{ "name": "Gauge", "crawlerStart": "https://docs.gauge.org/index.html?os=windows&language=javascript&ide=vscode", "crawlerPrefix": "https://docs.gauge.org/" }, +{ "name": "JBehave", "crawlerStart": "http://jbehave.org/reference/stable/", "crawlerPrefix": "http://jbehave.org/reference/stable/" }, +{ "name": "JUnit 4", "crawlerStart": "https://junit.org/junit4/", "crawlerPrefix": "https://junit.org/junit4/" }, +{ "name": "Selenide", "crawlerStart": "https://selenide.gitbooks.io/user-guide/content/en/", "crawlerPrefix": "https://selenide.gitbooks.io/user-guide/content/en/" }, +{ "name": "Serenity", "crawlerStart": "https://serenity-bdd.github.io/", "crawlerPrefix": "https://serenity-bdd.github.io/docs/" }, +{ "name": "CodeceptJS", "crawlerStart": "https://codecept.io/basics/", "crawlerPrefix": "https://codecept.io/" }, +{ "name": "Intern", "crawlerStart": "https://theintern.io/docs.html#Intern/4/docs/README.md", "crawlerPrefix": "https://theintern.io/docs.html#Intern/4/docs/" }, +{ "name": "Nightwatch", "crawlerStart": "https://nightwatchjs.org/guide/", "crawlerPrefix": "https://nightwatchjs.org/guide/" }, +{ "name": "Protractor", "crawlerStart": "https://www.protractortest.org/#/", "crawlerPrefix": "https://www.protractortest.org/#/" }, +{ "name": "TestCafe", "crawlerStart": "https://testcafe.io/documentation/", "crawlerPrefix": "https://testcafe.io/documentation/" }, +{ "name": "WebdriverIO", "crawlerStart": "https://webdriver.io/docs/gettingstarted", "crawlerPrefix": "https://webdriver.io/docs/" }, +{ "name": "Passport", "crawlerStart": "http://www.passportjs.org/docs/", "crawlerPrefix": "http://www.passportjs.org/" }, +{ "name": "Socket.IO", "crawlerStart": "https://socket.io/docs/v4", "crawlerPrefix": "https://socket.io/docs/v4" }, +{ "name": "PM2", "crawlerStart": "https://pm2.keymetrics.io/docs/usage/quick-start/", "crawlerPrefix": "https://pm2.keymetrics.io/docs/usage/" }, +{ "name": "NestJS", "crawlerStart": "https://docs.nestjs.com/", "crawlerPrefix": "https://docs.nestjs.com/" }, +{ "name": "C#", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/csharp/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/csharp/" }, +{ "name": "SpecFlow", "crawlerStart": "https://docs.specflow.org/en/latest/", "crawlerPrefix": "https://docs.specflow.org/projects/" }, +{ "name": "XUnit", "crawlerStart": "https://xunit.net/#documentation", "crawlerPrefix": "https://xunit.net/docs/" }, +{ "name": "PHP", "crawlerStart": "https://www.php.net/manual/en/", "crawlerPrefix": "https://www.php.net/manual/en/" }, +{ "name": "Behat", "crawlerStart": "https://docs.behat.org/en/latest/guides.html", "crawlerPrefix": "https://docs.behat.org/en/latest/" }, +{ "name": "Codeception", "crawlerStart": "https://codeception.com/docs/GettingStarted", "crawlerPrefix": "https://codeception.com/docs/" }, +{ "name": "Symfony", "crawlerStart": "https://symfony.com/doc/current/index.html", "crawlerPrefix": "https://symfony.com/doc/current/" }, +{ "name": "CodeIgniter", "crawlerStart": "https://codeigniter.com/user_guide/", "crawlerPrefix": "https://codeigniter.com/user_guide/" }, +{ "name": "Yii", "crawlerStart": "https://www.yiiframework.com/doc/guide/2.0/en", "crawlerPrefix": "https://www.yiiframework.com/doc/" }, +{ "name": "Laminas", "crawlerStart": "https://docs.laminas.dev/", "crawlerPrefix": "https://docs.laminas.dev/" }, +{ "name": "Phalcon", "crawlerStart": "https://docs.phalcon.io/5.0/en/introduction", "crawlerPrefix": "https://docs.phalcon.io/5.0/en/" }, +{ "name": "Slim", "crawlerStart": "http://www.slimframework.com/docs/v4/", "crawlerPrefix": "http://www.slimframework.com/docs/v4/" }, +{ "name": "CakePHP", "crawlerStart": "https://book.cakephp.org/4/en/index.html", "crawlerPrefix": "https://book.cakephp.org/4/en/" }, +{ "name": "CakePHP API", "crawlerStart": "https://api.cakephp.org/4.4/", "crawlerPrefix": "https://api.cakephp.org/4.4/" }, +{ "name": "FuelPHP", "crawlerStart": "https://fuelphp.com/docs/", "crawlerPrefix": "https://fuelphp.com/docs/" }, +{ "name": "PHPixie", "crawlerStart": "https://phpixie.com/docs.html/", "crawlerPrefix": "https://phpixie.com/" }, +{ "name": "Python 3", "crawlerStart": "https://docs.python.org/3/", "crawlerPrefix": "https://docs.python.org/3/" }, +{ "name": "Behave", "crawlerStart": "https://behave.readthedocs.io/en/latest/", "crawlerPrefix": "https://behave.readthedocs.io/en/latest/" }, +{ "name": "Lettuce", "crawlerStart": "https://lettuce.readthedocs.io/en/latest/", "crawlerPrefix": "https://lettuce.readthedocs.io/en/latest/" }, +{ "name": "Robot Framework", "crawlerStart": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#", "crawlerPrefix": "https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#" }, +{ "name": "Pytest", "crawlerStart": "https://docs.pytest.org/en/latest/", "crawlerPrefix": "https://docs.pytest.org/en/latest/" }, +{ "name": "Python 2", "crawlerStart": "https://docs.python.org/2/", "crawlerPrefix": "https://docs.python.org/2/" }, +{ "name": "NumPy", "crawlerStart": "https://numpy.org/doc/stable/", "crawlerPrefix": "https://numpy.org/doc/stable/" }, +{ "name": "SciPy", "crawlerStart": "https://docs.scipy.org/doc/scipy/reference/", "crawlerPrefix": "https://docs.scipy.org/doc/scipy/reference/" }, +{ "name": "Scikit-learn", "crawlerStart": "https://scikit-learn.org/stable/user_guide.html", "crawlerPrefix": "https://scikit-learn.org/stable/" }, +{ "name": "Seaborn", "crawlerStart": "https://seaborn.pydata.org/api.html", "crawlerPrefix": "https://seaborn.pydata.org/" }, +{ "name": "Bokeh", "crawlerStart": "https://docs.bokeh.org/en/latest/index.html", "crawlerPrefix": "https://docs.bokeh.org/en/latest/docs/" }, +{ "name": "BeautifulSoup", "crawlerStart": "https://www.crummy.com/software/BeautifulSoup/bs4/doc/", "crawlerPrefix": "https://www.crummy.com/software/BeautifulSoup/bs4/doc/" }, +{ "name": "NLTK", "crawlerStart": "https://www.nltk.org/", "crawlerPrefix": "https://www.nltk.org/" }, +{ "name": "Spacy", "crawlerStart": "https://spacy.io/api", "crawlerPrefix": "https://spacy.io/api" }, +{ "name": "Pygame", "crawlerStart": "https://www.pygame.org/docs/", "crawlerPrefix": "https://www.pygame.org/docs/" }, +{ "name": "Spring Framework", "crawlerStart": "https://docs.spring.io/spring-framework/reference/", "crawlerPrefix": "https://docs.spring.io/spring-framework/reference/" }, +{ "name": "Vue 2", "crawlerStart": "https://v2.vuejs.org/v2/", "crawlerPrefix": "https://vuejs.org/v2/" }, +{ "name": "Sinatra", "crawlerStart": "http://sinatrarb.com/documentation.html", "crawlerPrefix": "http://sinatrarb.com/" }, +{ "name": "ASP.NET", "crawlerStart": "https://learn.microsoft.com/en-us/aspnet/core/?view=aspnetcore-7.0", "crawlerPrefix": "https://learn.microsoft.com/en-us/aspnet/core/" }, +{ "name": "Pubnub", "crawlerStart": "https://www.pubnub.com/docs", "crawlerPrefix": "https://www.pubnub.com/docs" }, +{ "name": "Twilio", "crawlerStart": "https://www.twilio.com/docs/quickstart", "crawlerPrefix": "https://www.twilio.com/docs/" }, +{ "name": "Bandwidth", "crawlerStart": "https://dev.bandwidth.com/docs/", "crawlerPrefix": "https://dev.bandwidth.com/docs/" }, +{ "name": "HubSpot Operations Hub", "crawlerStart": "https://developers.hubspot.com/docs/api/overview", "crawlerPrefix": "https://developers.hubspot.com/docs/api/" }, +{ "name": "Podium", "crawlerStart": "https://docs.podium.com/docs", "crawlerPrefix": "https://docs.podium.com/" }, +{ "name": "Whereby", "crawlerStart": "https://docs.whereby.com//", "crawlerPrefix": "https://docs.whereby.com/" }, +{ "name": "Plivo", "crawlerStart": "https://www.plivo.com/docs/", "crawlerPrefix": "https://www.plivo.com/docs/" }, +{ "name": "Dotdigital", "crawlerStart": "https://developer.dotdigital.com/docs", "crawlerPrefix": "https://developer.dotdigital.com/docs" }, +{ "name": "Vonage Communications APIs", "crawlerStart": "https://developer.vonage.com/en/documentation", "crawlerPrefix": "https://developer.vonage.com/en/" }, +{ "name": "Dailyco", "crawlerStart": "https://docs.daily.co/", "crawlerPrefix": "https://docs.daily.co/" }, +{ "name": "Zoom", "crawlerStart": "https://developers.zoom.us/docs/api/", "crawlerPrefix": "https://developers.zoom.us/docs/api/" }, +{ "name": "Webex", "crawlerStart": "https://developer.webex.com/docs", "crawlerPrefix": "https://developer.webex.com/docs" }, +{ "name": "Microsoft Teams", "crawlerStart": "https://learn.microsoft.com/en-us/microsoftteams/platform/", "crawlerPrefix": "https://learn.microsoft.com/en-us/microsoftteams/platform/" }, +{ "name": "Office Addin", "crawlerStart": "https://learn.microsoft.com/en-us/office/dev/add-ins/", "crawlerPrefix": "https://learn.microsoft.com/en-us/office/dev/add-ins/" }, +{ "name": "Steam", "crawlerStart": "https://partner.steamgames.com/doc/sdk/", "crawlerPrefix": "https://partner.steamgames.com/doc/sdk/" }, +{ "name": "Epic Games", "crawlerStart": "https://dev.epicgames.com/docs", "crawlerPrefix": "https://dev.epicgames.com/docs" }, +{ "name": "Unreal Engine", "crawlerStart": "https://docs.unrealengine.com/5.2/en-US/", "crawlerPrefix": "https://docs.unrealengine.com/5.2/en-US/" }, +{ "name": "Splunk", "crawlerStart": "https://docs.splunk.com/Documentation", "crawlerPrefix": "https://docs.splunk.com/Documentation" }, +{ "name": "Elasticsearch", "crawlerStart": "https://www.elastic.co/guide/en/enterprise-search/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/enterprise-search/current/" }, +{ "name": "Logstash", "crawlerStart": "https://www.elastic.co/guide/en/logstash/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/logstash/current" }, +{ "name": "Kibana", "crawlerStart": "https://www.elastic.co/guide/en/kibana/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/kibana/current/" }, +{ "name": "Graylog", "crawlerStart": "https://go2docs.graylog.org/5-1/home.htm/", "crawlerPrefix": "https://go2docs.graylog.org/5-1/" }, +{ "name": "Chart.js", "crawlerStart": "https://www.chartjs.org/docs/latest/", "crawlerPrefix": "https://www.chartjs.org/docs/latest/" }, +{ "name": "FusionCharts", "crawlerStart": "https://www.fusioncharts.com/dev/", "crawlerPrefix": "https://www.fusioncharts.com/dev/" }, +{ "name": "Dygraphs", "crawlerStart": "https://dygraphs.com/", "crawlerPrefix": "https://dygraphs.com/" }, +{ "name": "Victory", "crawlerStart": "https://formidable.com/open-source/victory/docs/", "crawlerPrefix": "https://formidable.com/open-source/victory/" }, +{ "name": "Chartist.js", "crawlerStart": "https://gionkunz.github.io/chartist-js/api-documentation.html", "crawlerPrefix": "https://gionkunz.github.io/chartist-js/api-documentation.html" }, +{ "name": "Recharts", "crawlerStart": "http://recharts.org/en-US/guide/", "crawlerPrefix": "http://recharts.org/en-US/" }, +{ "name": "AmCharts", "crawlerStart": "https://www.amcharts.com/docs/v5/", "crawlerPrefix": "https://www.amcharts.com/docs/v5/" }, +{ "name": "Google Charts", "crawlerStart": "https://developers.google.com/chart", "crawlerPrefix": "https://developers.google.com/chart" }, +{ "name": "AnyChart", "crawlerStart": "https://docs.anychart.com/Quick_Start/Quick_Start", "crawlerPrefix": "https://docs.anychart.com/" }, +{ "name": "Highcharts", "crawlerStart": "https://www.highcharts.com/docs/", "crawlerPrefix": "https://www.highcharts.com/docs/" }, +{ "name": "Highcharts API", "crawlerStart": "https://api.highcharts.com/highcharts/", "crawlerPrefix": "https://api.highcharts.com/highcharts/" }, +{ "name": "Billboard.js", "crawlerStart": "https://naver.github.io/billboard.js/release/latest/doc/", "crawlerPrefix": "https://naver.github.io/billboard.js/release/latest/doc/" }, +{ "name": "ApexCharts.js", "crawlerStart": "https://apexcharts.com/docs/installation/", "crawlerPrefix": "https://apexcharts.com/docs/" }, +{ "name": "NVD3", "crawlerStart": "http://nvd3.org/examples/index.html", "crawlerPrefix": "http://nvd3.org/examples/" }, +{ "name": "Vis.js", "crawlerStart": "https://visjs.github.io/vis-network/docs/network/", "crawlerPrefix": "https://visjs.github.io/vis-network/docs/network/" }, +{ "name": "Pica", "crawlerStart": "https://github.com/nodeca/pica", "crawlerPrefix": "https://github.com/nodeca/pica" }, +{ "name": "Lena.js", "crawlerStart": "https://github.com/davidsonfellipe/lena.js", "crawlerPrefix": "https://github.com/davidsonfellipe/lena.js" }, +{ "name": "Jimp", "crawlerStart": "https://www.npmjs.com/package/jimp", "crawlerPrefix": "https://www.npmjs.com/package/jimp" }, +{ "name": "Grade", "crawlerStart": "https://benhowdle89.github.io/grade/", "crawlerPrefix": "https://benhowdle89.github.io/grade/" }, +{ "name": "MarvinJ", "crawlerStart": "https://www.marvinj.org/en/", "crawlerPrefix": "https://www.marvinj.org/en/" }, +{ "name": "Compressor.js", "crawlerStart": "https://github.com/fengyuanchen/compressorjs", "crawlerPrefix": "https://github.com/fengyuanchen/compressorjs" }, +{ "name": "Fabric.js", "crawlerStart": "http://fabricjs.com/docs/", "crawlerPrefix": "http://fabricjs.com/docs/" }, +{ "name": "CamanJS", "crawlerStart": "http://camanjs.com/guides/", "crawlerPrefix": "http://camanjs.com/guides/" }, +{ "name": "Cropper.js", "crawlerStart": "https://fengyuanchen.github.io/cropperjs", "crawlerPrefix": "https://fengyuanchen.github.io/cropperjs" }, +{ "name": "Croppie", "crawlerStart": "https://foliotek.github.io/Croppie/", "crawlerPrefix": "https://foliotek.github.io/Croppie/" }, +{ "name": "Merge Images", "crawlerStart": "https://github.com/lukechilds/merge-images", "crawlerPrefix": "https://github.com/lukechilds/merge-images" }, +{ "name": "Blurify", "crawlerStart": "https://github.com/dabanlee/blurify", "crawlerPrefix": "https://github.com/dabanlee/blurify" }, +{ "name": "Pintura", "crawlerStart": "https://pqina.nl/pintura/docs/v8/", "crawlerPrefix": "https://pqina.nl/doka/docs/" }, +{ "name": "Gin", "crawlerStart": "https://gin-gonic.com/docs/", "crawlerPrefix": "https://gin-gonic.com/docs/" }, +{ "name": "Revel", "crawlerStart": "https://revel.github.io/manual/index.html", "crawlerPrefix": "https://revel.github.io/manual/" }, +{ "name": "Echo", "crawlerStart": "https://echo.labstack.com/docs", "crawlerPrefix": "https://echo.labstack.com/docs" }, +{ "name": "Martini", "crawlerStart": "https://github.com/go-martini/martini", "crawlerPrefix": "https://github.com/go-martini/martini" }, +{ "name": "Buffalo", "crawlerStart": "https://gobuffalo.io/documentation/", "crawlerPrefix": "https://gobuffalo.io/documentation/" }, +{ "name": "Gorm", "crawlerStart": "https://gorm.io/docs/index.html", "crawlerPrefix": "https://gorm.io/docs/" }, +{ "name": "Go-kit", "crawlerStart": "https://gokit.io/examples/stringsvc.html", "crawlerPrefix": "https://gokit.io/examples/" }, +{ "name": "Negroni", "crawlerStart": "https://github.com/urfave/negroni", "crawlerPrefix": "https://github.com/urfave/negroni" }, +{ "name": "Mux", "crawlerStart": "https://github.com/gorilla/mux", "crawlerPrefix": "https://github.com/gorilla/mux" }, +{ "name": "Devise", "crawlerStart": "https://github.com/heartcombo/devise", "crawlerPrefix": "https://github.com/heartcombo/devise" }, +{ "name": "CanCanCan", "crawlerStart": "https://github.com/CanCanCommunity/cancancan/blob/develop/docs/README.md", "crawlerPrefix": "https://github.com/CanCanCommunity/cancancan/blob/develop/docs/" }, +{ "name": "Active Storage", "crawlerStart": "https://guides.rubyonrails.org/active_storage_overview.html#", "crawlerPrefix": "https://guides.rubyonrails.org/active_storage_overview.html#" }, +{ "name": "Resque", "crawlerStart": "https://github.com/resque/resque", "crawlerPrefix": "https://github.com/resque/resque" }, +{ "name": "Sidekiq", "crawlerStart": "https://github.com/sidekiq/sidekiq/wiki/", "crawlerPrefix": "https://github.com/sidekiq/sidekiq/wiki/" }, +{ "name": "Carrierwave", "crawlerStart": "https://github.com/carrierwaveuploader/carrierwave", "crawlerPrefix": "https://github.com/carrierwaveuploader/carrierwave" }, +{ "name": "Pundit", "crawlerStart": "https://github.com/varvet/pundit", "crawlerPrefix": "https://github.com/varvet/pundit" }, +{ "name": "Active Admin", "crawlerStart": "https://activeadmin.info/documentation.html", "crawlerPrefix": "https://activeadmin.info/" }, +{ "name": "Delayed Job", "crawlerStart": "https://github.com/collectiveidea/delayed_job", "crawlerPrefix": "https://github.com/collectiveidea/delayed_job" }, +{ "name": "FriendlyId", "crawlerStart": "https://norman.github.io/friendly_id/file.Guide.html#", "crawlerPrefix": "https://norman.github.io/friendly_id/file.Guide.html#" }, +{ "name": "Scrapy", "crawlerStart": "https://docs.scrapy.org/en/latest/", "crawlerPrefix": "https://docs.scrapy.org/en/latest/" }, +{ "name": "PyQT", "crawlerStart": "https://www.riverbankcomputing.com/static/Docs/PyQt5/", "crawlerPrefix": "https://www.riverbankcomputing.com/static/Docs/PyQt5/" }, +{ "name": "Tkinter", "crawlerStart": "https://docs.python.org/3/library/tkinter.html", "crawlerPrefix": "https://docs.python.org/3/library/" }, +{ "name": "Entity Framework", "crawlerStart": "https://learn.microsoft.com/en-us/ef/", "crawlerPrefix": "https://learn.microsoft.com/en-us/ef/" }, +{ "name": "Xamarin", "crawlerStart": "https://learn.microsoft.com/en-us/xamarin/", "crawlerPrefix": "https://learn.microsoft.com/en-us/xamarin/" }, +{ "name": "ML.NET", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/machine-learning/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/machine-learning/" }, +{ "name": "SignalR", "crawlerStart": "https://learn.microsoft.com/en-us/aspnet/core/signalr/introduction?view=aspnetcore-7.0", "crawlerPrefix": "https://learn.microsoft.com/en-us/aspnet/core/signalr/" }, +{ "name": "Boost", "crawlerStart": "https://www.boost.org/doc/libs/", "crawlerPrefix": "https://www.boost.org/doc/libs/" }, +{ "name": "Qt", "crawlerStart": "https://doc.qt.io/", "crawlerPrefix": "https://doc.qt.io/qt-6/" }, +{ "name": "Eigen", "crawlerStart": "https://eigen.tuxfamily.org/dox/", "crawlerPrefix": "https://eigen.tuxfamily.org/dox/" }, +{ "name": "SFML", "crawlerStart": "https://www.sfml-dev.org/documentation/2.6.0/", "crawlerPrefix": "https://www.sfml-dev.org/documentation/2.6.0/" }, +{ "name": "Hibernate", "crawlerStart": "https://hibernate.org/orm/documentation/6.2/", "crawlerPrefix": "https://hibernate.org/orm/documentation/6.2/" }, +{ "name": "Apache Struts", "crawlerStart": "https://struts.apache.org/getting-started/", "crawlerPrefix": "https://struts.apache.org/" }, +{ "name": "Thymeleaf", "crawlerStart": "https://www.thymeleaf.org/documentation.html", "crawlerPrefix": "https://www.thymeleaf.org/doc/tutorials/3.1/" }, +{ "name": "Apache Ant", "crawlerStart": "https://ant.apache.org/manual/", "crawlerPrefix": "https://ant.apache.org/manual/" }, +{ "name": "Gradle", "crawlerStart": "https://docs.gradle.org/current/userguide/userguide.html", "crawlerPrefix": "https://docs.gradle.org/current/userguide/" }, +{ "name": "Quartz Scheduler", "crawlerStart": "http://www.quartz-scheduler.org/api/2.3.0/index.html", "crawlerPrefix": "http://www.quartz-scheduler.org/api/2.3.0/index.html" }, +{ "name": "Apache POI", "crawlerStart": "https://poi.apache.org/apidocs/dev/index.html", "crawlerPrefix": "https://poi.apache.org/apidocs/dev/index.html" }, +{ "name": "JHipster", "crawlerStart": "https://www.jhipster.tech/development/", "crawlerPrefix": "https://www.jhipster.tech/" }, +{ "name": "Vaadin", "crawlerStart": "https://vaadin.com/docs/latest/", "crawlerPrefix": "https://vaadin.com/docs/latest/" }, +{ "name": "Hadoop", "crawlerStart": "https://hadoop.apache.org/docs/current/", "crawlerPrefix": "https://hadoop.apache.org/docs/current/" }, +{ "name": "Kafka", "crawlerStart": "https://kafka.apache.org/documentation/", "crawlerPrefix": "https://kafka.apache.org/documentation/" }, +{ "name": "Guava", "crawlerStart": "https://github.com/google/guava/wiki/", "crawlerPrefix": "https://github.com/google/guava/wiki/" }, +{ "name": "Jackson", "crawlerStart": "https://github.com/FasterXML/jackson-core/wiki", "crawlerPrefix": "https://github.com/FasterXML/jackson-core/wiki" }, +{ "name": "Log4j", "crawlerStart": "https://logging.apache.org/log4j/2.x/manual/", "crawlerPrefix": "https://logging.apache.org/log4j/2.x/manual/" }, +{ "name": "SLF4J", "crawlerStart": "http://www.slf4j.org/manual.html", "crawlerPrefix": "http://www.slf4j.org/manual.html" }, +{ "name": "Chargebee", "crawlerStart": "https://www.chargebee.com/docs/2.0/", "crawlerPrefix": "https://www.chargebee.com/docs/2.0/" }, +{ "name": "Chargebee API", "crawlerStart": "https://apidocs.chargebee.com/docs/api/", "crawlerPrefix": "https://apidocs.chargebee.com/docs/api/" }, +{ "name": "Paypal", "crawlerStart": "https://developer.paypal.com/docs/", "crawlerPrefix": "https://developer.paypal.com/docs/" }, +{ "name": "FullCalendar", "crawlerStart": "https://fullcalendar.io/docs", "crawlerPrefix": "https://fullcalendar.io/docs" }, +{ "name": "Day.js", "crawlerStart": "https://day.js.org/docs/en/installation/installation", "crawlerPrefix": "https://day.js.org/docs/en/" }, +{ "name": "Date-fns", "crawlerStart": "https://date-fns.org/docs/Getting-Started", "crawlerPrefix": "https://date-fns.org/v2.30.0/docs/" }, +{ "name": "Luxon", "crawlerStart": "https://moment.github.io/luxon/docs/manual/", "crawlerPrefix": "https://moment.github.io/luxon/docs/manual/" }, +{ "name": "Chrono", "crawlerStart": "https://docs.rs/chrono/0.4.26/chrono/index.html", "crawlerPrefix": "https://docs.rs/chrono/0.4.26/chrono/" }, +{ "name": "WordPress.com", "crawlerStart": "https://developer.wordpress.com/docs/", "crawlerPrefix": "https://developer.wordpress.com/docs/" }, +{ "name": "WordPress.org", "crawlerStart": "https://developer.wordpress.org/", "crawlerPrefix": "https://developer.wordpress.org/" }, +{ "name": "Swiper", "crawlerStart": "https://swiperjs.com/get-started", "crawlerPrefix": "https://swiperjs.com/swiper-api" }, +{ "name": "Flickity", "crawlerStart": "https://flickity.metafizzy.co/", "crawlerPrefix": "https://flickity.metafizzy.co/" }, +{ "name": "Glide.js", "crawlerStart": "https://glidejs.com/docs/", "crawlerPrefix": "https://glidejs.com/" }, +{ "name": "Owl Carousel", "crawlerStart": "https://owlcarousel2.github.io/OwlCarousel2/docs/started-welcome.html", "crawlerPrefix": "https://owlcarousel2.github.io/OwlCarousel2/docs/" }, +{ "name": "Slick", "crawlerStart": "https://kenwheeler.github.io/slick/", "crawlerPrefix": "https://kenwheeler.github.io/slick/" }, +{ "name": "Docker", "crawlerStart": "https://docs.docker.com/", "crawlerPrefix": "https://docs.docker.com/" }, +{ "name": "Ansible", "crawlerStart": "https://docs.ansible.com/ansible/latest/index.html", "crawlerPrefix": "https://docs.ansible.com/ansible/latest/" }, +{ "name": "Terraform", "crawlerStart": "https://developer.hashicorp.com/terraform/docs", "crawlerPrefix": "https://developer.hashicorp.com/terraform/" }, +{ "name": "Puppet", "crawlerStart": "https://www.puppet.com/docs/puppet/8/puppet_index.html", "crawlerPrefix": "https://www.puppet.com/docs/puppet/8/" }, +{ "name": "Chef", "crawlerStart": "https://docs.chef.io/", "crawlerPrefix": "https://docs.chef.io/" }, +{ "name": "SaltStack", "crawlerStart": "https://docs.saltproject.io/en/latest/contents.html", "crawlerPrefix": "https://docs.saltproject.io/en/latest/topics/" }, +{ "name": "Nagios", "crawlerStart": "https://library.nagios.com/", "crawlerPrefix": "https://library.nagios.com/library/" }, +{ "name": "Prometheus", "crawlerStart": "https://prometheus.io/docs/introduction/overview/", "crawlerPrefix": "https://prometheus.io/docs/" }, +{ "name": "Grafana", "crawlerStart": "https://grafana.com/docs/grafana/latest/", "crawlerPrefix": "https://grafana.com/docs/grafana/latest/" }, +{ "name": "ELK Stack", "crawlerStart": "https://www.elastic.co/guide/en/elastic-stack/current/index.html", "crawlerPrefix": "https://www.elastic.co/guide/en/elastic-stack/current/" }, +{ "name": "Sequelize", "crawlerStart": "https://sequelize.org/docs/v6/", "crawlerPrefix": "https://sequelize.org/docs/v6/" }, +{ "name": "SQLAlchemy", "crawlerStart": "https://docs.sqlalchemy.org/en/20/", "crawlerPrefix": "https://docs.sqlalchemy.org/en/20/" }, +{ "name": "ActiveRecord", "crawlerStart": "https://guides.rubyonrails.org/active_record_basics.html", "crawlerPrefix": "https://guides.rubyonrails.org/active_record_basics.html" }, +{ "name": "MySQL", "crawlerStart": "https://dev.mysql.com/doc/", "crawlerPrefix": "https://dev.mysql.com/doc/" }, +{ "name": "Gensim", "crawlerStart": "https://radimrehurek.com/gensim/auto_examples/index.html#documentation", "crawlerPrefix": "https://radimrehurek.com/gensim/auto_examples/" }, +{ "name": "Gymnasium", "crawlerStart": "https://gymnasium.farama.org/", "crawlerPrefix": "https://gymnasium.farama.org/" }, +{ "name": "OpenSSL", "crawlerStart": "https://www.openssl.org/docs/", "crawlerPrefix": "https://www.openssl.org/docs/" }, +{ "name": "Catch2", "crawlerStart": "https://github.com/catchorg/Catch2/blob/master/docs/tutorial.md", "crawlerPrefix": "https://github.com/catchorg/Catch2/blob/master/docs/" }, +{ "name": "Netty", "crawlerStart": "https://netty.io/5.0/api/index.html", "crawlerPrefix": "https://netty.io/5.0/api/" }, +{ "name": "RestSharp", "crawlerStart": "https://restsharp.dev/intro.html#introduction", "crawlerPrefix": "https://restsharp.dev/" }, +{ "name": "Serilog", "crawlerStart": "https://github.com/serilog/serilog/wiki/", "crawlerPrefix": "https://github.com/serilog/serilog/wiki/" }, +{ "name": "spdlog", "crawlerStart": "https://github.com/gabime/spdlog/blob/v2.x/README.md", "crawlerPrefix": "https://github.com/gabime/spdlog/blob/v2.x/" }, +{ "name": "Winston", "crawlerStart": "https://github.com/winstonjs/winston/blob/master/README.md", "crawlerPrefix": "https://github.com/winstonjs/winston/blob/master/" }, +{ "name": "Python Logging", "crawlerStart": "https://docs.python.org/3/library/logging.html", "crawlerPrefix": "https://docs.python.org/3/library/logging.html" }, +{ "name": "Bouncy Castle", "crawlerStart": "https://www.bouncycastle.org/docs/", "crawlerPrefix": "https://www.bouncycastle.org/docs/" }, +{ "name": "IdentityServer", "crawlerStart": "https://identityserver4.readthedocs.io/en/latest/", "crawlerPrefix": "https://identityserver4.readthedocs.io/en/latest/" }, +{ "name": "Unity", "crawlerStart": "https://docs.unity3d.com/Manual/index.html", "crawlerPrefix": "https://docs.unity3d.com/" }, +{ "name": "LibGDX", "crawlerStart": "https://libgdx.com/wiki/", "crawlerPrefix": "https://libgdx.com/wiki/" }, +{ "name": "Phaser", "crawlerStart": "https://newdocs.phaser.io/docs/3.60.0", "crawlerPrefix": "https://newdocs.phaser.io/docs/3.60.0" }, +{ "name": "Ionic", "crawlerStart": "https://ionicframework.com/docs", "crawlerPrefix": "https://ionicframework.com/docs" }, +{ "name": "NativeScript", "crawlerStart": "https://docs.nativescript.org/", "crawlerPrefix": "https://docs.nativescript.org/" }, +{ "name": "LinQ", "crawlerStart": "https://learn.microsoft.com/en-us/dotnet/csharp/linq/", "crawlerPrefix": "https://learn.microsoft.com/en-us/dotnet/csharp/linq/" }, +{ "name": "Brain.js", "crawlerStart": "https://github.com/BrainJS/brain.js#brainjs", "crawlerPrefix": "https://github.com/BrainJS/brain.js#brainjs" }, +{ "name": "Deeplearning4J", "crawlerStart": "https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn/latest/index.html", "crawlerPrefix": "https://javadoc.io/doc/org.deeplearning4j/deeplearning4j-nn/latest/" }, +{ "name": "Accord.NET", "crawlerStart": "http://accord-framework.net/docs/html/N_Accord.htm", "crawlerPrefix": "http://accord-framework.net/docs/html/" }, +{ "name": "Giphy SDK", "crawlerStart": "https://developers.giphy.com/docs/sdk", "crawlerPrefix": "https://developers.giphy.com/docs/sdk/" }, +{ "name": "Giphy API", "crawlerStart": "https://developers.giphy.com/docs/api", "crawlerPrefix": "https://developers.giphy.com/docs/api/" }, +{ "name": "Amazon S3", "crawlerStart": "https://docs.aws.amazon.com/s3/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/" }, +{ "name": "Amazon EC2", "crawlerStart": "https://docs.aws.amazon.com/ec2/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/" }, +{ "name": "Aws Lambda", "crawlerStart": "https://docs.aws.amazon.com/lambda/index.html", "crawlerPrefix": "https://docs.aws.amazon.com/lambda/latest/dg/" }, +{ "name": "Mantis", "crawlerStart": "https://www.mantisbt.org/documentation.php", "crawlerPrefix": "https://www.mantisbt.org/docs/master/en-US/Developers_Guide/" }, +{ "name": "React Admin", "crawlerStart": "https://marmelab.com/react-admin/documentation.html", "crawlerPrefix": "https://marmelab.com/react-admin/" }, +{ "name": "SQLFluff", "crawlerStart": "https://docs.sqlfluff.com/en/stable/", "crawlerPrefix": "https://docs.sqlfluff.com/en/stable/" }, +{ "name": "Ruff", "crawlerStart": "https://beta.ruff.rs/docs/", "crawlerPrefix": "https://beta.ruff.rs/docs/" }, +{ "name": "FusionAuth API", "crawlerStart": "https://fusionauth.io/docs/v1/tech/apis/", "crawlerPrefix": "https://fusionauth.io/docs/v1/tech/apis/" }, +{ "name": "Serp Google Scholar API", "crawlerStart": "https://serpapi.com/google-scholar-api", "crawlerPrefix": "https://serpapi.com/google-scholar-api" }, +{ "name": "Novu", "crawlerStart": "https://docs.novu.co/", "crawlerPrefix": "https://docs.novu.co/" }, +{ "name": "Drizzle", "crawlerStart": "https://orm.drizzle.team/docs/overview", "crawlerPrefix": "https://orm.drizzle.team/docs/overview" }, +{ "name": "Autogen", "crawlerStart": "https://microsoft.github.io/autogen/docs/", "crawlerPrefix": "https://microsoft.github.io/autogen/docs/" }, +{ "name": "Domo", "crawlerStart": "https://developer.domo.com/", "crawlerPrefix": "https://developer.domo.com/" }, +{ "name": "Crystal", "crawlerStart": "https://crystal-lang.org/api", "crawlerPrefix": "https://crystal-lang.org/api" }, +{ "name": "Amber", "crawlerStart": "https://docs.amberframework.org/amber", "crawlerPrefix": "https://docs.amberframework.org/amber" }, +{ "name": "TLDraw", "crawlerStart": "https://tldraw.dev/", "crawlerPrefix": "https://tldraw.dev/" }, +{ "name": "Swift", "crawlerStart": "https://developer.apple.com/documentation/technologies", "crawlerPrefix": "https://developer.apple.com/documentation/technologies" }]"#; diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs index 4aa5cbbe382..a31e6e294c2 100644 --- a/ee/tabby-webserver/src/service/web_documents.rs +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -1,21 +1,45 @@ -use std::sync::Arc; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use anyhow::anyhow; use async_trait::async_trait; use juniper::ID; -use tabby_db::DbConn; +use tabby_db::{DbConn, WebDocumentDAO}; use tabby_schema::{ - job::JobService, + job::{JobInfo, JobService}, web_documents::{CustomWebDocument, PresetWebDocument, WebDocumentService}, - Result, + AsID, AsRowid, CoreError, Result, +}; + +use super::{ + background_job::BackgroundJobEvent, graphql_pagination_to_filter, + preset_web_documents_data::PRESET_WEB_DOCUMENTS_DATA, }; pub fn create(db: DbConn, job_service: Arc) -> impl WebDocumentService { - WebDocumentServiceImpl { db, job_service } + let data: serde_json::Value = serde_json::from_str(PRESET_WEB_DOCUMENTS_DATA).unwrap(); + let mut preset_web_documents = HashMap::new(); + for doc in data.as_array().unwrap() { + let name = doc.get("name").unwrap().to_string(); + let url = doc.get("crawlerStart").unwrap().to_string(); + preset_web_documents.insert( + String::from(&name[1..name.len() - 1]), + String::from(&url[1..url.len() - 1]), + ); + } + WebDocumentServiceImpl { + db, + job_service, + preset_web_documents, + } } struct WebDocumentServiceImpl { db: DbConn, job_service: Arc, + preset_web_documents: HashMap, } #[async_trait] @@ -27,14 +51,55 @@ impl WebDocumentService for WebDocumentServiceImpl { first: Option, last: Option, ) -> Result> { - Ok(vec![]) + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + let urls = self + .db + .list_web_documents(limit, skip_id, backwards, false) + .await?; + + let mut converted_urls = vec![]; + + for url in urls { + let event = BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&url.id.as_id()), + url.url.clone(), + ); + + let job_info = self.job_service.get_job_info(event.to_command()).await?; + converted_urls.push(to_custom_web_document(url, job_info)); + } + Ok(converted_urls) } async fn create_custom_web_document(&self, name: String, url: String) -> Result { - Ok(ID::new("0")) + if self.preset_web_documents.contains_key(&name) { + return Err(CoreError::Other(anyhow!(format!( + "name: {} is conflicts with preset document", + name + )))); + } + let id = self + .db + .create_web_document(name, url.clone(), false) + .await?; + let _ = self + .job_service + .trigger( + BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&id.as_id()), + url, + ) + .to_command(), + ) + .await; + Ok(id.as_id()) } async fn delete_custom_web_document(&self, id: ID) -> Result<()> { + self.db.delete_web_document(id.as_rowid()?).await?; + self.job_service + .trigger(BackgroundJobEvent::IndexGarbageCollection.to_command()) + .await?; Ok(()) } @@ -44,12 +109,153 @@ impl WebDocumentService for WebDocumentServiceImpl { before: Option, first: Option, last: Option, - active: bool, + is_active: bool, ) -> Result> { - Ok(vec![]) + let (limit, skip_id, backwards) = graphql_pagination_to_filter(after, before, first, last)?; + let urls = self + .db + .list_web_documents(limit, skip_id, backwards, true) + .await?; + + let mut converted_urls = vec![]; + let mut active_urls: HashSet = HashSet::default(); + + for url in urls { + active_urls.insert(url.name.clone()); + + if is_active { + let event = BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&url.id.as_id()), + url.url.clone(), + ); + + let job_info = self.job_service.get_job_info(event.to_command()).await?; + converted_urls.push(to_preset_web_document(url, job_info)); + } + } + + if !is_active { + for name in self.preset_web_documents.keys() { + if active_urls.contains(name) { + continue; + } + + converted_urls.push(PresetWebDocument { + id: ID::from(name.clone()), + name: name.clone(), + job_info: None, + updated_at: None, + }); + } + } + + Ok(converted_urls) + } + + async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result<()> { + if !active { + self.db.deactivate_preset_web_document(name).await?; + self.job_service + .trigger(BackgroundJobEvent::IndexGarbageCollection.to_command()) + .await?; + Ok(()) + } else { + let Some(url) = self.preset_web_documents.get(&name) else { + return Err(CoreError::Other(anyhow!(format!( + "name {} does not exist", + name + )))); + }; + let id = self.db.create_web_document(name, url.clone(), true).await?; + let _ = self + .job_service + .trigger( + BackgroundJobEvent::WebCrawler( + CustomWebDocument::format_source_id(&id.as_id()), + url.clone(), + ) + .to_command(), + ) + .await; + Ok(()) + } } +} + +fn to_custom_web_document(value: WebDocumentDAO, job_info: JobInfo) -> CustomWebDocument { + CustomWebDocument { + id: value.id.as_id(), + url: value.url.clone(), + created_at: value.created_at, + updated_at: value.updated_at, + job_info, + name: value.name, + } +} + +fn to_preset_web_document(value: WebDocumentDAO, job_info: JobInfo) -> PresetWebDocument { + PresetWebDocument { + id: ID::from(value.name.clone()), + name: value.name, + job_info: Some(job_info), + updated_at: Some(value.updated_at), + } +} +#[cfg(test)] +mod tests { + use tabby_db::DbConn; + + use super::*; + use crate::background_job::BackgroundJobEvent; + + #[tokio::test] + async fn test_list_web_documents() { + let db = DbConn::new_in_memory().await.unwrap(); + let job = Arc::new(crate::service::job::create(db.clone()).await); + let service = create(db.clone(), job.clone()); + + let url = "https://example.com".to_string(); + let id = service + .create_custom_web_document("example".to_string(), url.clone()) + .await + .unwrap(); + + let command = BackgroundJobEvent::WebCrawler(id.to_string(), "https://example.com".into()) + .to_command(); + + db.create_job_run("web".into(), command).await.unwrap(); + + let urls = service + .list_custom_web_documents(None, None, None, None) + .await + .unwrap(); + + assert_eq!(1, urls.len()); + assert_eq!(id, urls[0].id); + assert!(urls[0].job_info.last_job_run.is_some()); + + service + .set_preset_web_documents_active("React".to_string(), true) + .await + .unwrap(); + + let command = + BackgroundJobEvent::WebCrawler("id".into(), "https://react.dev/reference/".into()) + .to_command(); + + db.create_job_run("preset".into(), command).await.unwrap(); + let urls = service + .list_preset_web_documents(None, None, None, None, true) + .await + .unwrap(); + + assert_eq!(1, urls.len()); + assert!(urls[0].updated_at.is_some()); + let urls = service + .list_preset_web_documents(None, None, None, None, false) + .await + .unwrap(); - async fn set_preset_web_documents_active(&self, name: String, active: bool) -> Result { - Ok(ID::new("0")) + assert_eq!(383, urls.len()); } } From 913260f129817f5c585cd7dbca3f3d1fdc4532fe Mon Sep 17 00:00:00 2001 From: xxs-wallace Date: Tue, 20 Aug 2024 13:04:21 +0800 Subject: [PATCH 12/12] add more test Signed-off-by: xxs-wallace --- ee/tabby-webserver/src/service/web_documents.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ee/tabby-webserver/src/service/web_documents.rs b/ee/tabby-webserver/src/service/web_documents.rs index a31e6e294c2..a9614b76bf2 100644 --- a/ee/tabby-webserver/src/service/web_documents.rs +++ b/ee/tabby-webserver/src/service/web_documents.rs @@ -257,5 +257,22 @@ mod tests { .unwrap(); assert_eq!(383, urls.len()); + service + .set_preset_web_documents_active("React".to_string(), false) + .await + .unwrap(); + let urls = service + .list_preset_web_documents(None, None, None, None, false) + .await + .unwrap(); + assert_eq!(384, urls.len()); + + service.delete_custom_web_document(id).await.unwrap(); + let urls = service + .list_custom_web_documents(None, None, None, None) + .await + .unwrap(); + + assert_eq!(0, urls.len()); } }