Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

workaround for message is too big for the index #50

Merged
merged 1 commit into from
Aug 27, 2024
Merged

workaround for message is too big for the index #50

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/VahterBanBot/DB.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ open Dapper
open VahterBanBot.Utils

let private connString = getEnv "DATABASE_URL"
let private truncateTextLimit = getEnvOr "DATABASE_TRUNCATE_TEXT_LIMIT" "3000" |> int

let upsertUser (user: DbUser): Task<DbUser> =
task {
Expand Down Expand Up @@ -184,7 +185,7 @@ WITH less_than_n_messages AS (SELECT u.id, COUNT(DISTINCT m.text) < @criticalMsg
WHERE NOT EXISTS(SELECT 1 FROM false_positive_users fpu WHERE fpu.user_id = b.banned_user_id)
AND NOT EXISTS(SELECT 1
FROM false_positive_messages fpm
WHERE fpm.text = b.message_text)
WHERE b.message_text LIKE fpm.text || '%')
AND b.message_text IS NOT NULL
AND b.banned_at >= @criticalDate),
spam_or_ham AS (SELECT x.text,
Expand Down Expand Up @@ -246,6 +247,11 @@ WHERE banned_user_id = @userId
let markMessageAsFalsePositive (message: DbMessage): Task =
task {
use conn = new NpgsqlConnection(connString)

// we need to truncate the text to work around the fact that
// text is being indexed and btree index has a limit of 2704 bytes
// https://dba.stackexchange.com/questions/25138/index-max-row-size-error/25139
let message = { message with text = message.text.Substring(0, Math.Min(truncateTextLimit, message.text.Length)) }

//language=postgresql
let sql =
Expand Down Expand Up @@ -336,7 +342,7 @@ WITH stats AS (SELECT m.message_id,
ON m.message_id = bbb.message_id AND m.chat_id = bbb.banned_in_chat_id
LEFT JOIN public.false_negative_messages fnm
ON m.message_id = fnm.message_id AND m.chat_id = fnm.chat_id
LEFT JOIN false_positive_messages fpm ON m.text = fpm.text
LEFT JOIN false_positive_messages fpm ON m.text LIKE fpm.text || '%'
WHERE m.user_id = @userId
ORDER BY m.created_at DESC
LIMIT @n),
Expand Down