refactor(backend): Config storage as an enum (#1543)

* Using dynamic Sotrage * Make it build * More comments * ckippy * clippy * remove worker feature * Add commercial license trial * Fix storage * Make it pass * clippy * Add commercial license * revert
reacherhq · Dec 11, 2024 · b784c77 · b784c77
1 parent 03e6c97
commit b784c77
Show file tree

Hide file tree

Showing 26 changed files with 470 additions and 283 deletions.
diff --git a/...07b69ee10ca90a712a6e38175c898b8e06b1.json → ...95660c56e852770a4eac47576089e704322a.json b/...07b69ee10ca90a712a6e38175c898b8e06b1.json → ...95660c56e852770a4eac47576089e704322a.json
diff --git a/...fe4d3bfaf55e3371f46a472ea3d5705d35b2.json → ...b712359553bc295db5689f254f623d172326.json b/...fe4d3bfaf55e3371f46a472ea3d5705d35b2.json → ...b712359553bc295db5689f254f623d172326.json
diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
@@ -12,13 +12,14 @@ check-if-email-exists = { path = "../core", features = ["sentry"] }
 config = "0.14"
 csv = "1.3.0"
 dotenv = "0.15.0"
-futures = { version = "0.3.30", optional = true }
-lapin = { version = "2.3.1", optional = true }
-tokio-executor-trait = { version = "2.1.1", optional = true }
-tokio-reactor-trait = { version = "1.1.0", optional = true }
+futures = { version = "0.3.30" }
+lapin = { version = "2.3.1" }
+tokio-executor-trait = { version = "2.1.1" }
+tokio-reactor-trait = { version = "1.1.0" }
 openssl = { version = "0.10.64", features = ["vendored"] }
-reqwest = { version = "0.12.5", features = ["json", "socks"], optional = true }
+reqwest = { version = "0.12.5", features = ["json", "socks"] }
 sentry = "0.23"
+sentry-anyhow = "0.23"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 sqlx = { version = "0.7", features = [
@@ -37,11 +38,5 @@ tracing-subscriber = "0.3.18"
 uuid = "1.10"
 warp = "0.3"
 
-[features]
-worker = [
-    "futures",
-    "lapin",
-    "tokio-executor-trait",
-    "tokio-reactor-trait",
-    "reqwest",
-]
+[dev-dependencies]
+toml = "0.8"
diff --git a/backend/Dockerfile b/backend/Dockerfile
@@ -14,7 +14,7 @@ COPY . .
 
 ENV SQLX_OFFLINE=true
 
-RUN cargo build --bin reacher_backend --features worker --release --target=x86_64-unknown-linux-musl
+RUN cargo build --bin reacher_backend --release --target=x86_64-unknown-linux-musl
 
 # ------------------------------------------------------------------------------
 # Final Stage

diff --git a/backend/README.md b/backend/README.md
@@ -57,7 +57,7 @@ $ git clone https://github.com/reacherhq/check-if-email-exists
 $ cd check-if-email-exists/backend
 
 # Run the backend binary in release mode (slower build, but more performant).
-$ cargo run --release --bin reacher_backend --features worker
+$ cargo run --release --bin reacher_backend
 ```
 
 The server will then be listening on `http://127.0.0.1:8080`.
diff --git a/backend/backend_config.toml b/backend/backend_config.toml
@@ -44,6 +44,11 @@ webdriver_addr = "http://localhost:9515"
 # Env variable: RCH__SMTP_TIMEOUT
 # smtp_timeout = 45
 
+# Optional Sentry DSN. If set, all errors will be sent to Sentry.
+#
+# Env variable: RCH__SENTRY_DSN
+# sentry_dsn = "<PASTE_YOUR_DSN_NOW>"
+
 # Uncomment the lines below to route all SMTP verification requests
 # through a specified proxy. Note that the proxy must be a SOCKS5 proxy to work
 # with the SMTP protocol. This proxy will not be used for headless
@@ -119,14 +124,22 @@ concurrency = 5
 # max_requests_per_hour = 1000
 # max_requests_per_day = 20000
 
-# Postgres configuration. Currently, a Postgres database is required to store
-# the results of the verifications. This might change in the future, allowing
-# for pluggable storage.
-[worker.postgres]
-# Env variable: RCH__WORKER__POSTGRES__DB_URL
-db_url = "postgresql://localhost/reacherdb"
+# Below are the configurations for the storage of the email verification
+# results. We currently support the following storage backends:
+# - Postgres
+#
+# Uncomment the following line to configure the storage to use Postgres.
+# [storage.postgres]
 
-# Optional Sentry DSN. If set, all errors will be sent to Sentry.
+# # URL to connect to the Postgres database.
 #
-# Env variable: RCH__SENTRY_DSN
-# sentry_dsn = "<PASTE_YOUR_DSN_NOW>"
+# Env variable: RCH__STORAGE__0__POSTGRES__DB_URL
+# db_url = "postgresql://localhost/reacherdb"
+#
+# If you wish to store additional data along with the verification results,
+# you can add a JSON object to the "extra" field. This object will be stored
+# as a JSONB column in the database. This is for example useful to track who
+# initiated the verification request in a multi-tenant system.
+# 
+# Env variable: RCH__STORAGE__0__POSTGRES__TABLE_NAME
+# extra = { "my_custom_key" = "my_custom_value" }
diff --git a/backend/migrations/20240929230957_v1_worker_results.up.sql b/backend/migrations/20240929230957_v1_worker_results.up.sql
@@ -6,9 +6,9 @@ CREATE TABLE v1_bulk_job (
 
 CREATE TABLE v1_task_result (
     id SERIAL PRIMARY KEY,
-    job_id INTEGER NOT NULL REFERENCES v1_bulk_job(id) ON DELETE CASCADE,
+    job_id INTEGER REFERENCES v1_bulk_job(id) ON DELETE CASCADE,
     payload JSONB NOT NULL,
-    backend_name TEXT NOT NULL,
+    extra JSONB, -- any extra data that needs to be stored
     result JSONB,
     error TEXT,
     created_at TIMESTAMPTZ DEFAULT NOW() NOT NULL