From 2ab2a18583d010a34294b0c137b721c82c5f2e8e Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Tue, 27 Aug 2024 20:42:46 +0100 Subject: [PATCH] workaround for message is too big for the index (#50) --- src/VahterBanBot/DB.fs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/VahterBanBot/DB.fs b/src/VahterBanBot/DB.fs index d8c4e23..39266fb 100644 --- a/src/VahterBanBot/DB.fs +++ b/src/VahterBanBot/DB.fs @@ -9,6 +9,7 @@ open Dapper open VahterBanBot.Utils let private connString = getEnv "DATABASE_URL" +let private truncateTextLimit = getEnvOr "DATABASE_TRUNCATE_TEXT_LIMIT" "3000" |> int let upsertUser (user: DbUser): Task = task { @@ -184,7 +185,7 @@ WITH less_than_n_messages AS (SELECT u.id, COUNT(DISTINCT m.text) < @criticalMsg WHERE NOT EXISTS(SELECT 1 FROM false_positive_users fpu WHERE fpu.user_id = b.banned_user_id) AND NOT EXISTS(SELECT 1 FROM false_positive_messages fpm - WHERE fpm.text = b.message_text) + WHERE b.message_text LIKE fpm.text || '%') AND b.message_text IS NOT NULL AND b.banned_at >= @criticalDate), spam_or_ham AS (SELECT x.text, @@ -246,6 +247,11 @@ WHERE banned_user_id = @userId let markMessageAsFalsePositive (message: DbMessage): Task = task { use conn = new NpgsqlConnection(connString) + + // we need to truncate the text to work around the fact that + // text is being indexed and btree index has a limit of 2704 bytes + // https://dba.stackexchange.com/questions/25138/index-max-row-size-error/25139 + let message = { message with text = message.text.Substring(0, Math.Min(truncateTextLimit, message.text.Length)) } //language=postgresql let sql = @@ -336,7 +342,7 @@ WITH stats AS (SELECT m.message_id, ON m.message_id = bbb.message_id AND m.chat_id = bbb.banned_in_chat_id LEFT JOIN public.false_negative_messages fnm ON m.message_id = fnm.message_id AND m.chat_id = fnm.chat_id - LEFT JOIN false_positive_messages fpm ON m.text = fpm.text + LEFT JOIN false_positive_messages fpm ON m.text LIKE fpm.text || '%' WHERE m.user_id = @userId ORDER BY m.created_at DESC LIMIT @n),