feat(propdefs): slow start, sql queries (#24676)

PostHog · Sep 2, 2024 · 6442885 · 6442885
1 parent 1027746
commit 6442885
Show file tree

Hide file tree

Showing 22 changed files with 595 additions and 43 deletions.
diff --git a/rust/.sqlx/query-16d533b5a15b0b9926a181f578b5b577efe424710b45f02e1ddeece8bca96f87.json b/rust/.sqlx/query-16d533b5a15b0b9926a181f578b5b577efe424710b45f02e1ddeece8bca96f87.json
diff --git a/...8bfe578ecaa8678268863bce917ced9d5d46.json → ...165376f43116b85921c62b36b1b8e85562b0.json b/...8bfe578ecaa8678268863bce917ced9d5d46.json → ...165376f43116b85921c62b36b1b8e85562b0.json
diff --git a/rust/.sqlx/query-42e393046a686e6a69daa920dc2ab521aa6f393027c399a0c40139f5f8a0a45e.json b/rust/.sqlx/query-42e393046a686e6a69daa920dc2ab521aa6f393027c399a0c40139f5f8a0a45e.json
diff --git a/rust/.sqlx/query-58dfd4671ac3497614b184384ac7f8d490dda8b27a150454e413d02f89c92050.json b/rust/.sqlx/query-58dfd4671ac3497614b184384ac7f8d490dda8b27a150454e413d02f89c92050.json
diff --git a/rust/.sqlx/query-8ab11a89bc4720985e130c58021f46045c332cc45e69b08708b289cc933b3b5c.json b/rust/.sqlx/query-8ab11a89bc4720985e130c58021f46045c332cc45e69b08708b289cc933b3b5c.json
diff --git a/rust/.sqlx/query-917e3d14c15558a1e0bb1d7015ed687eb545ee9d4ccbb8b69c958a357d49f687.json b/rust/.sqlx/query-917e3d14c15558a1e0bb1d7015ed687eb545ee9d4ccbb8b69c958a357d49f687.json
diff --git a/rust/.sqlx/query-aa595eaf28c1f4b872c278be407b59cc00f3125413f4032ac3647a6b5ee1a632.json b/rust/.sqlx/query-aa595eaf28c1f4b872c278be407b59cc00f3125413f4032ac3647a6b5ee1a632.json
diff --git a/rust/.sqlx/query-b160b785a0377b854341105e99e4ef7a5da523e168a5f9be47f6caaef09487d7.json b/rust/.sqlx/query-b160b785a0377b854341105e99e4ef7a5da523e168a5f9be47f6caaef09487d7.json
diff --git a/rust/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json b/rust/.sqlx/query-c6ff00fcbbc77c8f5c1b3fe2f3352ea79485e403b9e17b6c37259ea0612065ee.json
diff --git a/...4bb0b551e91c503e8b21c29814f1eb660a8e.json → ...2345454cd3323568c4bef5b8480380287068.json b/...4bb0b551e91c503e8b21c29814f1eb660a8e.json → ...2345454cd3323568c4bef5b8480380287068.json
diff --git a/rust/.sqlx/query-eecef0ce664dfe65dff4452d92a29c948a291ea8218bbbb4e25cd1ad36dbe9f4.json b/rust/.sqlx/query-eecef0ce664dfe65dff4452d92a29c948a291ea8218bbbb4e25cd1ad36dbe9f4.json
diff --git a/...2ce446cac731845c53cb926fc47ace708dd6.json → ...f30c682288988fc6f102e8a063be97c3e51c.json b/...2ce446cac731845c53cb926fc47ace708dd6.json → ...f30c682288988fc6f102e8a063be97c3e51c.json
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
diff --git a/rust/property-defs-rs/Cargo.toml b/rust/property-defs-rs/Cargo.toml
@@ -22,6 +22,7 @@ chrono = { workspace = true }
 quick_cache = { workspace = true }
 common-metrics = { path = "../common/metrics" }
 ahash = { workspace = true }
+uuid = { workspace = true }
 
 [lints]
 workspace = true
diff --git a/rust/property-defs-rs/README.md b/rust/property-defs-rs/README.md
@@ -0,0 +1,5 @@
+# Event and property definitions generator (propdefs for short)
+
+This consumes events from `clickhouse_events_json` and writes event and property definitions to postgres. It filters DB updates to avoid duplicate writes (writes are idempotent, but the DB load of writing every property definition every time would be too high).
+
+Hoglets should check out [the runbook](http://runbooks/ingestion/property-defs-rs) for a detailed breakdown of how it's tuned for our current scale, and what metrics to look at and levers to pull if responding to an incident.