From 71d1a090146644929ac34e532150abe625280926 Mon Sep 17 00:00:00 2001 From: Lavshyak Date: Tue, 24 Dec 2024 19:15:24 +0500 Subject: [PATCH 1/2] add message with custom emoji deletion --- src/VahterBanBot/Bot.fs | 140 ++++++++++++++++++++++++++---------- src/VahterBanBot/Program.fs | 8 ++- src/VahterBanBot/Types.fs | 9 ++- 3 files changed, 119 insertions(+), 38 deletions(-) diff --git a/src/VahterBanBot/Bot.fs b/src/VahterBanBot/Bot.fs index f397b85..90f0261 100644 --- a/src/VahterBanBot/Bot.fs +++ b/src/VahterBanBot/Bot.fs @@ -7,6 +7,7 @@ open System.Threading.Tasks open Microsoft.Extensions.Logging open Telegram.Bot open Telegram.Bot.Types +open Telegram.Bot.Types.Enums open Telegram.Bot.Types.ReplyMarkups open VahterBanBot.ML open VahterBanBot.Types @@ -446,6 +447,40 @@ let killSpammerAutomated logger.LogInformation logMsg } +let deleteSpamMessageAutomated + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (message: Message) + (logger: ILogger) + (deleteType: string) + = task { + use banOnReplyActivity = botActivity.StartActivity("deleteSpamMessageAutomated") + %banOnReplyActivity + .SetTag("spammerId", message.From.Id) + .SetTag("spammerUsername", message.From.Username) + + // delete message + do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.MessageId) + |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete message {message.MessageId} from chat {message.Chat.Id}", e)) + // 0 here is the bot itself + do! DbBanned.banMessage 0 message + |> DB.banUserByBot + + let logMsg = $"Deleted {deleteType} spam in {prependUsername message.Chat.Username} ({message.Chat.Id}) from {prependUsername message.From.Username} ({message.From.Id}) with text:\n{message.TextOrCaption}" + + let! replyMarkup = task { + let data = CallbackMessage.NotASpam { message = message } + let! callback = DB.newCallback data + return InlineKeyboardMarkup [ + InlineKeyboardButton.WithCallbackData("✅ NOT a spam", string callback.id) + ] + } + + // log both to logger and to logs channel + do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), logMsg, replyMarkup = replyMarkup) |> taskIgnore + logger.LogInformation logMsg +} + let autoBan (botUser: DbUser) (botClient: ITelegramBotClient) @@ -470,6 +505,19 @@ let autoBan do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), msg) |> taskIgnore } + +let calculateCustomEmojiProportion + (message: Message) : float option = + + let textLen = message.Text |> Seq.where(fun c -> not (" :.,\n\r".Contains c)) |> Seq.length + let customEmojiCount = message.Entities |> Array.where(fun e -> e.Type = MessageEntityType.CustomEmoji) |> Array.length + + if customEmojiCount = 0 || textLen = 0 then + None + else + let proportion = float customEmojiCount / float textLen + Some proportion + let justMessage (botUser: DbUser) (botClient: ITelegramBotClient) @@ -492,48 +540,68 @@ let justMessage // just delete message and move on do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.MessageId) |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete message {message.MessageId} from chat {message.Chat.Id}", e)) - - elif botConfig.MlEnabled && message.TextOrCaption <> null then - use mlActivity = botActivity.StartActivity("mlPrediction") + else + // set to true if Ml or other antispam deleted message + let mutable messageIsDeleted = false - let shouldBeSkipped = - // skip prediction for vahters or local admins - if botConfig.AllowedUsers.ContainsValue message.From.Id - || UpdateChatAdmins.Admins.Contains message.From.Id then - true - else + // ML + if not messageIsDeleted && botConfig.MlEnabled && message.TextOrCaption <> null then + use mlActivity = botActivity.StartActivity("mlPrediction") + + let shouldBeSkipped = + // skip prediction for vahters or local admins + if botConfig.AllowedUsers.ContainsValue message.From.Id + || UpdateChatAdmins.Admins.Contains message.From.Id then + true + else - match botConfig.MlStopWordsInChats.TryGetValue message.Chat.Id with - | true, stopWords -> - stopWords - |> Seq.exists (fun sw -> message.TextOrCaption.Contains(sw, StringComparison.OrdinalIgnoreCase)) - | _ -> false - %mlActivity.SetTag("skipPrediction", shouldBeSkipped) - - if not shouldBeSkipped then - let! usrMsgCount = DB.countUniqueUserMsg message.From.Id + match botConfig.MlStopWordsInChats.TryGetValue message.Chat.Id with + | true, stopWords -> + stopWords + |> Seq.exists (fun sw -> message.TextOrCaption.Contains(sw, StringComparison.OrdinalIgnoreCase)) + | _ -> false + %mlActivity.SetTag("skipPrediction", shouldBeSkipped) - match ml.Predict(message.TextOrCaption, usrMsgCount) with - | Some prediction -> - %mlActivity.SetTag("spamScoreMl", prediction.Score) + if not shouldBeSkipped then + let! usrMsgCount = DB.countUniqueUserMsg message.From.Id - if prediction.Score >= botConfig.MlSpamThreshold then - // delete message - do! killSpammerAutomated botClient botConfig message logger botConfig.MlSpamDeletionEnabled prediction.Score - - if botConfig.MlSpamAutobanEnabled then - // trigger auto-ban check - do! autoBan botUser botClient botConfig message logger - elif prediction.Score >= botConfig.MlWarningThreshold then - // just warn - do! killSpammerAutomated botClient botConfig message logger false prediction.Score - else - // not a spam + match ml.Predict(message.TextOrCaption, usrMsgCount) with + | Some prediction -> + %mlActivity.SetTag("spamScoreMl", prediction.Score) + + if prediction.Score >= botConfig.MlSpamThreshold then + // delete message + do! killSpammerAutomated botClient botConfig message logger botConfig.MlSpamDeletionEnabled prediction.Score + messageIsDeleted <- true + + if botConfig.MlSpamAutobanEnabled then + // trigger auto-ban check + do! autoBan botUser botClient botConfig message logger + elif prediction.Score >= botConfig.MlWarningThreshold then + // just warn + do! killSpammerAutomated botClient botConfig message logger false prediction.Score + else + // not a spam + () + | None -> + // no prediction (error or not ready yet) () - | None -> - // no prediction (error or not ready yet) - () + // emoji + if not messageIsDeleted + && botConfig.BanCustomEmojiEnabled + && message.TextOrCaption <> null + && botUser.created_at > DateTime.Now.AddDays(-botConfig.BanCustomEmojiRegisteredInDbDaysToDontBan) + && message.TextOrCaption.Length >= botConfig.BanCustomEmojiTextMinLen + then + let! usrMsgCount = DB.countUniqueUserMsg message.From.Id + if usrMsgCount < botConfig.BanCustomEmojiPreviousMessagesCountToDontBan then + let customEmojiProportion = calculateCustomEmojiProportion message + if customEmojiProportion.IsSome && customEmojiProportion.Value >= botConfig.BanCustomEmojiProportionToBan then + // delete message + do! deleteSpamMessageAutomated botClient botConfig message logger "emoji" + messageIsDeleted <- true + do! message |> DbMessage.newMessage diff --git a/src/VahterBanBot/Program.fs b/src/VahterBanBot/Program.fs index 5615993..4dea216 100644 --- a/src/VahterBanBot/Program.fs +++ b/src/VahterBanBot/Program.fs @@ -76,7 +76,13 @@ let botConf = MlSpamThreshold = getEnvOr "ML_SPAM_THRESHOLD" "0.5" |> single MlWarningThreshold = getEnvOr "ML_WARNING_THRESHOLD" "0.0" |> single MlMaxNumberOfIterations = getEnvOr "ML_MAX_NUMBER_OF_ITERATIONS" "50" |> int - MlStopWordsInChats = getEnvOr "ML_STOP_WORDS_IN_CHATS" "{}" |> fromJson } + MlStopWordsInChats = getEnvOr "ML_STOP_WORDS_IN_CHATS" "{}" |> fromJson + + BanCustomEmojiEnabled = getEnvOr "BAN_CUSTOM_EMOJI_Enabled" "false" |> bool.Parse + BanCustomEmojiProportionToBan = getEnvOr "BAN_CUSTOM_EMOJI_PROPORTION_TO_BAN" "0.3" |> float + BanCustomEmojiPreviousMessagesCountToDontBan = getEnvOr "BAN_CUSTOM_EMOJI_PREVIOUS_MESSAGES_COUNT_TO_DONT_BAN" "5" |> int32 + BanCustomEmojiRegisteredInDbDaysToDontBan = getEnvOr "BAN_CUSTOM_EMOJI_REGISTERED_IN_DB_DAYS_TO_DONT_BAN" "80" |> int32 + BanCustomEmojiTextMinLen = getEnvOr "BAN_CUSTOM_TEXT_MIN_LEN" "10" |> int32} let validateApiKey (ctx : HttpContext) = match ctx.TryGetRequestHeader "X-Telegram-Bot-Api-Secret-Token" with diff --git a/src/VahterBanBot/Types.fs b/src/VahterBanBot/Types.fs index 4f190a7..0156278 100644 --- a/src/VahterBanBot/Types.fs +++ b/src/VahterBanBot/Types.fs @@ -42,7 +42,14 @@ type BotConfiguration = MlSpamThreshold: single MlWarningThreshold: single MlMaxNumberOfIterations: int - MlStopWordsInChats: Dictionary } + MlStopWordsInChats: Dictionary + + BanCustomEmojiEnabled : bool + BanCustomEmojiProportionToBan : float + BanCustomEmojiPreviousMessagesCountToDontBan : int32 + // if user registered in DB greater than N days, he won't be banned + BanCustomEmojiRegisteredInDbDaysToDontBan : int32 + BanCustomEmojiTextMinLen : int32} [] type DbUser = From 03124ea4ca9721a95758363d9ab75a67075f6be1 Mon Sep 17 00:00:00 2001 From: Lavshyak Date: Tue, 24 Dec 2024 19:30:18 +0500 Subject: [PATCH 2/2] emoji ban bug fix --- src/VahterBanBot/Bot.fs | 5 +++-- src/VahterBanBot/Program.fs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/VahterBanBot/Bot.fs b/src/VahterBanBot/Bot.fs index 90f0261..a60c1b5 100644 --- a/src/VahterBanBot/Bot.fs +++ b/src/VahterBanBot/Bot.fs @@ -509,7 +509,7 @@ let autoBan let calculateCustomEmojiProportion (message: Message) : float option = - let textLen = message.Text |> Seq.where(fun c -> not (" :.,\n\r".Contains c)) |> Seq.length + let textLen = message.TextOrCaption |> Seq.where(fun c -> not (" :.,\n\r".Contains c)) |> Seq.length let customEmojiCount = message.Entities |> Array.where(fun e -> e.Type = MessageEntityType.CustomEmoji) |> Array.length if customEmojiCount = 0 || textLen = 0 then @@ -518,6 +518,7 @@ let calculateCustomEmojiProportion let proportion = float customEmojiCount / float textLen Some proportion + let justMessage (botUser: DbUser) (botClient: ITelegramBotClient) @@ -591,7 +592,7 @@ let justMessage if not messageIsDeleted && botConfig.BanCustomEmojiEnabled && message.TextOrCaption <> null - && botUser.created_at > DateTime.Now.AddDays(-botConfig.BanCustomEmojiRegisteredInDbDaysToDontBan) + && botUser.created_at <= DateTime.Now.AddDays(-botConfig.BanCustomEmojiRegisteredInDbDaysToDontBan) && message.TextOrCaption.Length >= botConfig.BanCustomEmojiTextMinLen then let! usrMsgCount = DB.countUniqueUserMsg message.From.Id diff --git a/src/VahterBanBot/Program.fs b/src/VahterBanBot/Program.fs index 4dea216..2e7bf08 100644 --- a/src/VahterBanBot/Program.fs +++ b/src/VahterBanBot/Program.fs @@ -82,7 +82,7 @@ let botConf = BanCustomEmojiProportionToBan = getEnvOr "BAN_CUSTOM_EMOJI_PROPORTION_TO_BAN" "0.3" |> float BanCustomEmojiPreviousMessagesCountToDontBan = getEnvOr "BAN_CUSTOM_EMOJI_PREVIOUS_MESSAGES_COUNT_TO_DONT_BAN" "5" |> int32 BanCustomEmojiRegisteredInDbDaysToDontBan = getEnvOr "BAN_CUSTOM_EMOJI_REGISTERED_IN_DB_DAYS_TO_DONT_BAN" "80" |> int32 - BanCustomEmojiTextMinLen = getEnvOr "BAN_CUSTOM_TEXT_MIN_LEN" "10" |> int32} + BanCustomEmojiTextMinLen = getEnvOr "BAN_CUSTOM_EMOJI_TEXT_MIN_LEN" "10" |> int32} let validateApiKey (ctx : HttpContext) = match ctx.TryGetRequestHeader "X-Telegram-Bot-Api-Secret-Token" with