diff --git a/.env.example b/.env.example index b061ce9..35b8030 100644 --- a/.env.example +++ b/.env.example @@ -21,4 +21,5 @@ ML_SPAM_DELETION_ENABLED=false ML_TRAIN_BEFORE_DATE=2021-01-01 ML_TRAINING_SET_FRACTION=0.2 ML_SPAM_THRESHOLD=0.5 +ML_WARNING_THRESHOLD=0.0 ML_STOP_WORDS_IN_CHATS={"-123":["word1","word2"]} diff --git a/src/VahterBanBot.Tests/BanTests.fs b/src/VahterBanBot.Tests/BanTests.fs index a7b5028..bf9f3c5 100644 --- a/src/VahterBanBot.Tests/BanTests.fs +++ b/src/VahterBanBot.Tests/BanTests.fs @@ -76,4 +76,58 @@ type BanTests(fixture: VahterTestContainers) = Assert.False msgNotBanned } + [] + let ``Vahter can unban user`` () = task { + // record a message + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0]) + let! _ = fixture.SendMessage msgUpdate + + // send the ban message + let! banResp = + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + |> fixture.SendMessage + Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) + + // assert that the message got banned + let! msgBanned = fixture.MessageBanned msgUpdate.Message + Assert.True msgBanned + + // send the unban message from another vahter + let! banResp = + Tg.quickMsg($"/unban {msgUpdate.Message.From.Id}", chat = fixture.ChatsToMonitor[0], from = fixture.AdminUsers[1]) + |> fixture.SendMessage + Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) + + // assert that the message no longer banned + let! msgBanned = fixture.MessageBanned msgUpdate.Message + Assert.False msgBanned + } + + [] + let ``Only Vahter can unban user`` () = task { + // record a message + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0]) + let! _ = fixture.SendMessage msgUpdate + + // send the ban message + let! banResp = + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + |> fixture.SendMessage + Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) + + // assert that the message got banned + let! msgBanned = fixture.MessageBanned msgUpdate.Message + Assert.True msgBanned + + // send the unban message from a random user + let! banResp = + Tg.quickMsg($"/unban {msgUpdate.Message.From.Id}", chat = fixture.ChatsToMonitor[0]) + |> fixture.SendMessage + Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) + + // assert that the message still banned + let! msgBanned = fixture.MessageBanned msgUpdate.Message + Assert.True msgBanned + } + interface IAssemblyFixture diff --git a/src/VahterBanBot.Tests/ContainerTestBase.fs b/src/VahterBanBot.Tests/ContainerTestBase.fs index 3f2413d..0861c48 100644 --- a/src/VahterBanBot.Tests/ContainerTestBase.fs +++ b/src/VahterBanBot.Tests/ContainerTestBase.fs @@ -206,6 +206,34 @@ type VahterTestContainers() = let! count = conn.QuerySingleAsync(sql, {| chatId = msg.Chat.Id; messageId = msg.MessageId |}) return count > 0 } + + member _.GetCallbackId(msg: Message) (caseName: string) = task { + use conn = new NpgsqlConnection(publicConnectionString) + //language=postgresql + let sql = """ +SELECT id +FROM callback +WHERE data ->> 'Case' = @caseName + AND data -> 'Fields' -> 0 -> 'message' ->> 'message_id' = @messageId::TEXT + AND data -> 'Fields' -> 0 -> 'message' -> 'chat' ->> 'id' = @chatId::TEXT +""" + return! conn.QuerySingleAsync( + sql, {| chatId = msg.Chat.Id + messageId = msg.MessageId + caseName = caseName |}) + } + + member _.IsMessageFalsePositive(msg: Message) = task { + use conn = new NpgsqlConnection(publicConnectionString) + //language=postgresql + let sql = """ +SELECT COUNT(*) FROM false_positive_messages +WHERE chat_id = @chatId + AND message_id = @messageId +""" + let! result = conn.QuerySingleAsync(sql, {| chatId = msg.Chat.Id; messageId = msg.MessageId |}) + return result > 0 + } // workaround to wait for ML to be ready type MlAwaitFixture() = diff --git a/src/VahterBanBot.Tests/MLBanTests.fs b/src/VahterBanBot.Tests/MLBanTests.fs index 6ae35ac..9cadd33 100644 --- a/src/VahterBanBot.Tests/MLBanTests.fs +++ b/src/VahterBanBot.Tests/MLBanTests.fs @@ -1,9 +1,8 @@ module VahterBanBot.Tests.MLBanTests -open System.Net -open System.Threading.Tasks open VahterBanBot.Tests.ContainerTestBase open VahterBanBot.Tests.TgMessageUtils +open VahterBanBot.Types open Xunit open Xunit.Extensions.AssemblyFixture @@ -21,6 +20,19 @@ type MLBanTests(fixture: VahterTestContainers, _unused: MlAwaitFixture) = Assert.True msgBanned } + [] + let ``Message is NOT autobanned if it looks like a spam BUT vahter sent it`` () = task { + // record a message, where 2 is in a training set as spam word + // ChatsToMonitor[0] doesn't have stopwords + // but it was sent by vahter + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2", from = fixture.AdminUsers[0]) + let! _ = fixture.SendMessage msgUpdate + + // assert that the message got auto banned + let! msgBanned = fixture.MessageIsAutoBanned msgUpdate.Message + Assert.False msgBanned + } + [] let ``Message is NOT autobanned if it has a stopword in specific chat`` () = task { // record a message, where 2 is in a training set as spam word @@ -54,6 +66,52 @@ type MLBanTests(fixture: VahterTestContainers, _unused: MlAwaitFixture) = let! msgBanned = fixture.MessageIsAutoBanned msgUpdate.Message Assert.True msgBanned } + + [] + let ``If message got auto-deleted we can mark it as false-positive with a button click`` () = task { + // record a message, where 2 is in a training set as spam word + // ChatsToMonitor[0] doesn't have stopwords + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2") + let! _ = fixture.SendMessage msgUpdate + + // assert that the message got auto banned + let! msgBanned = fixture.MessageIsAutoBanned msgUpdate.Message + Assert.True msgBanned + // assert it is not false-positive + let! isFalsePositive = fixture.IsMessageFalsePositive msgUpdate.Message + Assert.False isFalsePositive + + // send a callback to mark it as false-positive + let! callbackId = fixture.GetCallbackId msgUpdate.Message "NotASpam" + let msgCallback = Tg.callback(string callbackId, from = fixture.AdminUsers[0]) + let! _ = fixture.SendMessage msgCallback + + // assert it is false-positive + let! isFalsePositive = fixture.IsMessageFalsePositive msgUpdate.Message + Assert.True isFalsePositive + } + + [] + let ``Only vahter can press THE BUTTON(s)`` () = task { + // record a message, where 2 is in a training set as spam word + // ChatsToMonitor[0] doesn't have stopwords + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2") + let! _ = fixture.SendMessage msgUpdate + + // assert that the message got auto banned + let! msgBanned = fixture.MessageIsAutoBanned msgUpdate.Message + Assert.True msgBanned + + // send a callback to mark it as false-positive + // we are sending this as a usual user + let! callbackId = fixture.GetCallbackId msgUpdate.Message (nameof CallbackMessage.NotASpam) + let msgCallback = Tg.callback(string callbackId) + let! _ = fixture.SendMessage msgCallback + + // assert it is still NOT a false-positive + let! isFalsePositive = fixture.IsMessageFalsePositive msgUpdate.Message + Assert.False isFalsePositive + } interface IAssemblyFixture interface IClassFixture diff --git a/src/VahterBanBot.Tests/TgMessageUtils.fs b/src/VahterBanBot.Tests/TgMessageUtils.fs index e9703c1..743e299 100644 --- a/src/VahterBanBot.Tests/TgMessageUtils.fs +++ b/src/VahterBanBot.Tests/TgMessageUtils.fs @@ -19,7 +19,20 @@ type Tg() = Id = (id |> Option.defaultValue (nextInt64())), Username = (username |> Option.defaultValue null) ) - static member quickMsg (?text: string, ?chat: Chat, ?from: User, ?date: DateTime) = + + static member callback(data: string, ?from: User) = + Update( + Id = next(), + Message = null, + CallbackQuery = CallbackQuery( + Id = Guid.NewGuid().ToString(), + Data = data, + From = (from |> Option.defaultValue (Tg.user())), + ChatInstance = Guid.NewGuid().ToString() + ) + ) + + static member quickMsg (?text: string, ?chat: Chat, ?from: User, ?date: DateTime, ?callback: CallbackQuery) = Update( Id = next(), Message = diff --git a/src/VahterBanBot.Tests/test_seed.sql b/src/VahterBanBot.Tests/test_seed.sql index de9c6ca..62efad3 100644 --- a/src/VahterBanBot.Tests/test_seed.sql +++ b/src/VahterBanBot.Tests/test_seed.sql @@ -130,31 +130,31 @@ VALUES (-666, 10001, 1001, '2021-01-01 00:00:00', 'a', '{}'), -- false positive (-42, 10099, 1006, '2021-01-01 00:00:09', '2', '{}'), -- this is not spam (-666, 10100, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10101, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10102, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10103, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10104, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10105, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10106, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10107, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10108, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10109, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10110, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10111, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10112, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10113, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10114, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10115, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10116, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10117, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10118, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10119, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10120, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10121, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10122, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10123, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10124, 1001, '2021-01-01 00:00:01', 'a', '{}'), - (-666, 10125, 1001, '2021-01-01 00:00:01', 'a', '{}'), + (-666, 10101, 1001, '2021-01-01 00:00:01', 'b', '{}'), + (-666, 10102, 1001, '2021-01-01 00:00:01', 'c', '{}'), + (-666, 10103, 1001, '2021-01-01 00:00:01', 'd', '{}'), + (-666, 10104, 1001, '2021-01-01 00:00:01', 'e', '{}'), + (-666, 10105, 1001, '2021-01-01 00:00:01', 'f', '{}'), + (-666, 10106, 1001, '2021-01-01 00:00:01', 'g', '{}'), + (-666, 10107, 1001, '2021-01-01 00:00:01', 'h', '{}'), + (-666, 10108, 1001, '2021-01-01 00:00:01', 'i', '{}'), + (-666, 10109, 1001, '2021-01-01 00:00:01', 'j', '{}'), + (-666, 10110, 1001, '2021-01-01 00:00:01', 'k', '{}'), + (-666, 10111, 1001, '2021-01-01 00:00:01', 'l', '{}'), + (-666, 10112, 1001, '2021-01-01 00:00:01', 'm', '{}'), + (-666, 10113, 1001, '2021-01-01 00:00:01', 'n', '{}'), + (-666, 10114, 1001, '2021-01-01 00:00:01', 'o', '{}'), + (-666, 10115, 1001, '2021-01-01 00:00:01', 'p', '{}'), + (-666, 10116, 1001, '2021-01-01 00:00:01', 'q', '{}'), + (-666, 10117, 1001, '2021-01-01 00:00:01', 'r', '{}'), + (-666, 10118, 1001, '2021-01-01 00:00:01', 's', '{}'), + (-666, 10119, 1001, '2021-01-01 00:00:01', 't', '{}'), + (-666, 10120, 1001, '2021-01-01 00:00:01', 'u', '{}'), + (-666, 10121, 1001, '2021-01-01 00:00:01', 'v', '{}'), + (-666, 10122, 1001, '2021-01-01 00:00:01', 'w', '{}'), + (-666, 10123, 1001, '2021-01-01 00:00:01', 'x', '{}'), + (-666, 10124, 1001, '2021-01-01 00:00:01', 'y', '{}'), + (-666, 10125, 1001, '2021-01-01 00:00:01', 'z', '{}'), (-666, 10126, 1001, '2021-01-01 00:00:01', 'a', '{}'), (-666, 10127, 1001, '2021-01-01 00:00:01', 'a', '{}'), (-666, 10128, 1001, '2021-01-01 00:00:01', 'a', '{}'), @@ -232,12 +232,12 @@ VALUES (-666, 10001, 1001, '2021-01-01 00:00:00', 'a', '{}'), -- false positive -- to enforce false-negative appearance (-666, 10200, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10201, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10202, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10203, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10204, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10205, 1001, '2021-01-01 00:00:01', '3', '{}'), - (-666, 10206, 1001, '2021-01-01 00:00:01', '3', '{}'), + (-666, 10201, 1001, '2021-01-01 00:00:01', '4', '{}'), + (-666, 10202, 1001, '2021-01-01 00:00:01', '5', '{}'), + (-666, 10203, 1001, '2021-01-01 00:00:01', '6', '{}'), + (-666, 10204, 1001, '2021-01-01 00:00:01', '7', '{}'), + (-666, 10205, 1001, '2021-01-01 00:00:01', '8', '{}'), + (-666, 10206, 1001, '2021-01-01 00:00:01', '9', '{}'), (-666, 10207, 1001, '2021-01-01 00:00:01', '3', '{}'), (-666, 10208, 1001, '2021-01-01 00:00:01', '3', '{}'), (-666, 10209, 1001, '2021-01-01 00:00:01', '3', '{}'), @@ -341,8 +341,8 @@ VALUES (100001, 10001, 'a', 1001, '2021-01-01 00:00:00', -666, 'pro.hell', 34), INSERT INTO public.false_positive_users(user_id) VALUES (1001); -INSERT INTO public.false_positive_messages(id) -VALUES (100002); +INSERT INTO public.false_positive_messages(chat_id, message_id) +VALUES (-666, 10008); INSERT INTO public.false_negative_messages(chat_id, message_id) VALUES (-42, 10008), diff --git a/src/VahterBanBot/Antispam.fs b/src/VahterBanBot/Antispam.fs deleted file mode 100644 index 52fad42..0000000 --- a/src/VahterBanBot/Antispam.fs +++ /dev/null @@ -1,107 +0,0 @@ -module VahterBanBot.Antispam - -open System -open System.Linq - -let cyrillicLikeCharacters = [| 'u'; 't'; 'a' |] -let cyrillicCharacters = "абвгдежзиклмнопрстуфхцчшщъыьэюяё".ToHashSet() - -let countFakeCyrillicWords (wl: string list) = - - let hasCyrillicLikeCharacters (w: string) = - w.IndexOfAny(cyrillicLikeCharacters) <> -1 - - let isMostlyCyrillic (w: string) = - let isCyrillic c = cyrillicCharacters.Contains(c) - - w.Count(isCyrillic) > (w.Length / 2) - - let isFakeCyrillicWord w = - isMostlyCyrillic w && hasCyrillicLikeCharacters w - - wl.Count(isFakeCyrillicWord) - -let phrases = [ - 10, [ "обучение"; "бесплатное" ] - 10, [ "бесплатное"; "обучение" ] - 7, [ "удаленная"; "работа" ] - 7, [ "удаленный"; "заработок" ] - 7, [ "удаленную"; "работу" ] - 3, [ "в"; "лс" ] - 3, [ "в"; "личку" ] - 3, [ "в"; "личные"; "сообщения" ] -] - -let countPhrases (wl: string list) = - // premium performance - let rec countPhrase wl totalScore (score, psx as phrase) = - // List.tail should be safe here as we are passing list of phrases above which is always non-empty - let p, ps = List.head psx, List.tail psx - - match wl with - | w :: ws when w = p -> - if ws.Take(ps.Length).SequenceEqual(ps) then - countPhrase (List.skip ps.Length ws) (totalScore + score) phrase - else - countPhrase ws totalScore phrase - | _ :: ws -> - countPhrase ws totalScore phrase - | _ -> totalScore - - List.sumBy (countPhrase wl 0) phrases - -let wordPrefixesWeighted = [ - 10, "крипт" - 10, "crypto" - 10, "defi" - 10, "usdt" - 10, "трейд" - 7, "вакансия" - 5, "партнер" - 5, "заработок" - 5, "заработк" - 3, "зарплата" -] - -let countWords (wl: string list) = - let checkWord wl word = - let score, (actualWord: string) = word - - let checkSingleWord (w: string) = if w.StartsWith(actualWord) then score else 0 - - List.sumBy checkSingleWord wl - - List.sumBy (checkWord wl) wordPrefixesWeighted - -let distillWords (str: string) = - // regexs are probably better - let isCyrLatAlphaChar c = - let isLat = c >= 'a' && c <= 'z' - let isCyr = c >= 'а' && c <= 'я' // who cares about Ё - let isDigit = c >= '0' && c <= '9' - let isDollar = c = '$' // useful - let isAnySpace = Char.IsWhiteSpace(c) - - isLat || isCyr || isDigit || isDollar || isAnySpace - - let filteredStr = String.filter isCyrLatAlphaChar (str.ToLower()) - - List.ofArray <| filteredStr.Split(' ', StringSplitOptions.TrimEntries ||| StringSplitOptions.RemoveEmptyEntries) - -let countEmojiLikeCharacters str = - let mutable emojis = 0 - - let countEmoji (c: char) = - if c >= char 0xDD00 then emojis <- emojis + 1 - - String.iter countEmoji str - - emojis - -let calcSpamScore msg = - let words = distillWords msg - - (countFakeCyrillicWords words) * 100 - + (countEmojiLikeCharacters msg) * 5 - + (countPhrases words) * 10 - + (countWords words) * 10 \ No newline at end of file diff --git a/src/VahterBanBot/Bot.fs b/src/VahterBanBot/Bot.fs index 3a9ddff..7260240 100644 --- a/src/VahterBanBot/Bot.fs +++ b/src/VahterBanBot/Bot.fs @@ -7,10 +7,10 @@ open System.Threading.Tasks open Microsoft.Extensions.Logging open Telegram.Bot open Telegram.Bot.Types +open Telegram.Bot.Types.ReplyMarkups open VahterBanBot.ML open VahterBanBot.Types open VahterBanBot.Utils -open VahterBanBot.Antispam let botActivity = new ActivitySource("VahterBanBot") @@ -42,11 +42,11 @@ let isBanOnReplyCommand (message: Message) = let isMessageFromAllowedChats (botConfig: BotConfiguration) (message: Message) = botConfig.ChatsToMonitor.ContainsValue message.Chat.Id -let isMessageFromAdmin (botConfig: BotConfiguration) (message: Message) = - botConfig.AllowedUsers.ContainsValue message.From.Id +let isUserVahter (botConfig: BotConfiguration) (user: DbUser) = + botConfig.AllowedUsers.ContainsValue user.id let isBannedPersonAdmin (botConfig: BotConfiguration) (message: Message) = - botConfig.AllowedUsers.ContainsValue message.ReplyToMessage.From.Id + botConfig.AllowedUsers.ContainsValue message.From.Id let isKnownCommand (message: Message) = message.Text <> null && @@ -57,31 +57,29 @@ let isKnownCommand (message: Message) = let isBanAuthorized (botConfig: BotConfiguration) - (message: Message) - (logger: ILogger) - (targetUserId: int64) - (targetUsername: string option) - (isBan: bool) - = - let banType = if isBan then "ban" else "unban" - let fromUserId = message.From.Id - let fromUsername = message.From.Username - let chatId = message.Chat.Id - let chatUsername = message.Chat.Username + (bannedMessage: Message) + (vahter: DbUser) + (logger: ILogger) = + let fromUserId = vahter.id + let fromUsername = defaultArg vahter.username null + let chatId = bannedMessage.Chat.Id + let chatUsername = bannedMessage.Chat.Username + let targetUserId = bannedMessage.From.Id + let targetUsername = bannedMessage.From.Username // check that user is allowed to ban others - if isMessageFromAdmin botConfig message then - if not(isMessageFromAllowedChats botConfig message) then - logger.LogWarning $"User {fromUsername} {fromUserId} tried to {banType} user {targetUsername} ({targetUserId}) from not allowed chat {chatUsername} ({chatId})" + if isUserVahter botConfig vahter then + if not(isMessageFromAllowedChats botConfig bannedMessage) then + logger.LogWarning $"User {fromUsername} {fromUserId} tried to ban user {prependUsername targetUsername} ({targetUserId}) from not allowed chat {chatUsername} ({chatId})" false // check that user is not trying to ban other admins - elif isBan && isBannedPersonAdmin botConfig message then - logger.LogWarning $"User {fromUsername} ({fromUserId}) tried to {banType} admin {targetUsername} ({targetUserId}) in chat {chatUsername} ({chatId}" + elif isBannedPersonAdmin botConfig bannedMessage then + logger.LogWarning $"User {fromUsername} ({fromUserId}) tried to ban admin {prependUsername targetUsername} ({targetUserId}) in chat {chatUsername} ({chatId}" false else true else - logger.LogWarning $"User {fromUsername} ({fromUserId}) tried to {banType} user {targetUsername} ({targetUserId}) without being admin in chat {chatUsername} ({chatId}" + logger.LogWarning $"User {fromUsername} ({fromUserId}) tried to ban user {prependUsername targetUsername} ({targetUserId}) without being admin in chat {chatUsername} ({chatId}" false let banInAllChats (botConfig: BotConfiguration) (botClient: ITelegramBotClient) targetUserId = task { @@ -192,8 +190,8 @@ let aggregateBanResultInLogMsg message = aggregateResultInLogMsg true message - message.ReplyToMessage.From.Id - (Some message.ReplyToMessage.From.Username) + message.From.Id + (Some message.From.Username) let aggregateUnbanResultInLogMsg message targetUserId targetUsername = aggregateResultInLogMsg @@ -202,11 +200,12 @@ let aggregateUnbanResultInLogMsg message targetUserId targetUsername = targetUserId targetUsername -let softBanResultInLogMsg (message: Message) (duration: int) = +let softBanResultInLogMsg (message: Message) (vahter: DbUser) (duration: int) = let logMsgBuilder = StringBuilder() + let vahterUsername = defaultArg vahter.username null let untilDate = (DateTime.UtcNow.AddHours duration).ToString "u" - %logMsgBuilder.Append $"Vahter {prependUsername message.From.Username}({message.From.Id}) " - %logMsgBuilder.Append $"softbanned {prependUsername message.ReplyToMessage.From.Username}({message.ReplyToMessage.From.Id}) " + %logMsgBuilder.Append $"Vahter {prependUsername vahterUsername}({vahter.id}) " + %logMsgBuilder.Append $"softbanned {prependUsername message.From.Username}({message.From.Id}) " %logMsgBuilder.Append $"in {prependUsername message.Chat.Username}({message.Chat.Id}) " %logMsgBuilder.Append $"until {untilDate}" string logMsgBuilder @@ -236,37 +235,39 @@ let deleteChannelMessage logger.LogInformation $"Deleted message from channel {probablyChannelName}" } -let banOnReply +let totalBan (botClient: ITelegramBotClient) (botConfig: BotConfiguration) (message: Message) + (vahter: DbUser) (logger: ILogger) = task { - use banOnReplyActivity = botActivity.StartActivity("banOnReply") + use banOnReplyActivity = botActivity.StartActivity("totalBan") %banOnReplyActivity - .SetTag("vahterId", message.From.Id) - .SetTag("vahterUsername", message.From.Username) - .SetTag("targetId", message.ReplyToMessage.From.Id) - .SetTag("targetUsername", message.ReplyToMessage.From.Username) - - // delete message that was replied to - let deleteReplyTask = task { + .SetTag("vahterId", vahter.id) + .SetTag("vahterUsername", (defaultArg vahter.username null)) + .SetTag("targetId", message.From.Id) + .SetTag("targetUsername", message.From.Username) + + // delete message + let deleteMsgTask = task { use _ = botActivity - .StartActivity("deleteReplyMsg") - .SetTag("msgId", message.ReplyToMessage.MessageId) + .StartActivity("deleteMsg") + .SetTag("msgId", message.MessageId) .SetTag("chatId", message.Chat.Id) .SetTag("chatUsername", message.Chat.Username) - do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.ReplyToMessage.MessageId) - |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete reply message {message.ReplyToMessage.MessageId} from chat {message.Chat.Id}", e)) + do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.MessageId) + |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete message {message.MessageId} from chat {message.Chat.Id}", e)) } + // update user in DB let updatedUser = - message.ReplyToMessage.From + message.From |> DbUser.newUser |> DB.upsertUser - + let deletedUserMessagesTask = task { - let fromUserId = message.ReplyToMessage.From.Id + let fromUserId = message.From.Id let! allUserMessages = DB.getUserMessages fromUserId logger.LogInformation($"Deleting {allUserMessages.Length} messages from user {fromUserId}") @@ -291,14 +292,14 @@ let banOnReply } // try ban user in all monitored chats - let! banResults = banInAllChats botConfig botClient message.ReplyToMessage.From.Id + let! banResults = banInAllChats botConfig botClient message.From.Id let! deletedUserMessages = deletedUserMessagesTask // produce aggregated log message let logMsg = aggregateBanResultInLogMsg message logger deletedUserMessages banResults // add ban record to DB - do! message.ReplyToMessage + do! message |> DbBanned.banMessage message.From.Id |> DB.banUser @@ -307,30 +308,52 @@ let banOnReply logger.LogInformation logMsg do! updatedUser.Ignore() - do! deleteReplyTask + do! deleteMsgTask +} + +let banOnReply + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (message: Message) + (vahter: DbUser) + (logger: ILogger) = task { + use banOnReplyActivity = botActivity.StartActivity("banOnReply") + %banOnReplyActivity + .SetTag("vahterId", message.From.Id) + .SetTag("vahterUsername", message.From.Username) + .SetTag("targetId", message.ReplyToMessage.From.Id) + .SetTag("targetUsername", message.ReplyToMessage.From.Username) + + do! totalBan + botClient + botConfig + message.ReplyToMessage + vahter + logger } -let softBanOnReply +let softBanMsg (botClient: ITelegramBotClient) (botConfig: BotConfiguration) (message: Message) + (vahter: DbUser) (logger: ILogger) = task { use banOnReplyActivity = botActivity.StartActivity("softBanOnReply") %banOnReplyActivity - .SetTag("vahterId", message.From.Id) - .SetTag("vahterUsername", message.From.Username) - .SetTag("targetId", message.ReplyToMessage.From.Id) - .SetTag("targetUsername", message.ReplyToMessage.From.Username) + .SetTag("vahterId", vahter.id) + .SetTag("vahterUsername", defaultArg vahter.username null) + .SetTag("targetId", message.From.Id) + .SetTag("targetUsername", message.From.Username) - let deleteReplyTask = task { + let deleteMsgTask = task { use _ = botActivity - .StartActivity("deleteReplyMsg") - .SetTag("msgId", message.ReplyToMessage.MessageId) + .StartActivity("deleteMsg") + .SetTag("msgId", message.MessageId) .SetTag("chatId", message.Chat.Id) .SetTag("chatUsername", message.Chat.Username) - do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.ReplyToMessage.MessageId) - |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete reply message {message.ReplyToMessage.MessageId} from chat {message.Chat.Id}", e)) + do! botClient.DeleteMessageAsync(ChatId(message.Chat.Id), message.MessageId) + |> safeTaskAwait (fun e -> logger.LogError ($"Failed to delete reply message {message.MessageId} from chat {message.Chat.Id}", e)) } let maybeDurationString = message.Text.Split " " |> Seq.last @@ -340,10 +363,10 @@ let softBanOnReply | true, x -> x | _ -> 24 // 1 day should be enough - let logText = softBanResultInLogMsg message duration + let logText = softBanResultInLogMsg message vahter duration - do! softBanInChat botClient (ChatId message.Chat.Id) message.ReplyToMessage.From.Id duration |> taskIgnore - do! deleteReplyTask + do! softBanInChat botClient (ChatId message.Chat.Id) message.From.Id duration |> taskIgnore + do! deleteMsgTask do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), logText) |> taskIgnore logger.LogInformation logText @@ -365,6 +388,10 @@ let unban if user.IsSome then %banOnReplyActivity.SetTag("targetUsername", user.Value.username) + // delete ban record from DB + do! user.Value.id + |> DB.unbanUser + let targetUsername = user |> Option.bind (_.username) // try unban user in all monitored chats @@ -401,8 +428,26 @@ let killSpammerAutomated let msgType = if deleteMessage then "Deleted" else "Detected" let logMsg = $"{msgType} spam (score: {score}) in {prependUsername message.Chat.Username} ({message.Chat.Id}) from {prependUsername message.From.Username} ({message.From.Id}) with text:\n{message.Text}" + let! replyMarkup = task { + if deleteMessage then + let data = CallbackMessage.NotASpam { message = message } + let! callback = DB.newCallback data + return InlineKeyboardMarkup [ + InlineKeyboardButton.WithCallbackData("NOT a spam", string callback.id) + ] + else + let spamData = CallbackMessage.Spam { message = message } + let notSpamData = CallbackMessage.NotASpam { message = message } + let! spamCallback = DB.newCallback spamData + let! notSpamCallback = DB.newCallback notSpamData + return InlineKeyboardMarkup [ + InlineKeyboardButton.WithCallbackData("KILL", string spamCallback.id) + InlineKeyboardButton.WithCallbackData("NOT a spam", string notSpamCallback.id) + ] + } + // log both to logger and to logs channel - do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), logMsg) |> taskIgnore + do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), logMsg, replyMarkup = replyMarkup) |> taskIgnore logger.LogInformation logMsg } @@ -412,18 +457,22 @@ let justMessage (logger: ILogger) (ml: MachineLearning) (message: Message) = task { - - use justMessageActivity = + + use _ = botActivity .StartActivity("justMessage") .SetTag("fromUserId", message.From.Id) .SetTag("fromUsername", message.From.Username) - - + if botConfig.MlEnabled && message.Text <> null then use mlActivity = botActivity.StartActivity("mlPrediction") let shouldBeSkipped = + // skip prediction for vahters + if botConfig.AllowedUsers.ContainsValue message.From.Id then + true + else + match botConfig.MlStopWordsInChats.TryGetValue message.Chat.Id with | true, stopWords -> stopWords @@ -439,7 +488,7 @@ let justMessage if prediction.Score >= botConfig.MlSpamThreshold then // delete message do! killSpammerAutomated botClient botConfig message logger botConfig.MlSpamDeletionEnabled prediction.Score - elif prediction.Score > 0.0f then + elif prediction.Score >= botConfig.MlWarningThreshold then // just warn do! killSpammerAutomated botClient botConfig message logger false prediction.Score else @@ -449,9 +498,6 @@ let justMessage // no prediction (error or not ready yet) () - let spamScore = if message.Text <> null then calcSpamScore message.Text else 0 - %justMessageActivity.SetTag("spamScore", spamScore) - do! message |> DbMessage.newMessage @@ -463,49 +509,34 @@ let adminCommand (botClient: ITelegramBotClient) (botConfig: BotConfiguration) (logger: ILogger) + (vahter: DbUser) (message: Message) = // aux functions to overcome annoying FS3511: This state machine is not statically compilable. let banOnReplyAux() = task { - let targetUserId = message.ReplyToMessage.From.Id - let targetUsername = Option.ofObj message.ReplyToMessage.From.Username let authed = isBanAuthorized botConfig - message + message.ReplyToMessage + vahter logger - targetUserId - targetUsername - true if authed then - do! banOnReply botClient botConfig message logger + do! banOnReply botClient botConfig message vahter logger } let unbanAux() = task { let targetUserId = message.Text.Split(" ", StringSplitOptions.RemoveEmptyEntries)[1] |> int64 - let authed = - isBanAuthorized - botConfig - message - logger - targetUserId - None - false - if authed then + if isUserVahter botConfig vahter then do! unban botClient botConfig message logger targetUserId } let softBanOnReplyAux() = task { - let targetUserId = message.ReplyToMessage.From.Id - let targetUsername = Option.ofObj message.ReplyToMessage.From.Username let authed = isBanAuthorized botConfig - message + message.ReplyToMessage + vahter logger - targetUserId - targetUsername - true if authed then - do! softBanOnReply botClient botConfig message logger + do! softBanMsg botClient botConfig message.ReplyToMessage vahter logger } task { @@ -532,15 +563,15 @@ let adminCommand elif isPingCommand message then do! ping botClient message do! deleteCmdTask - } + } -let onUpdate +let onMessage (botClient: ITelegramBotClient) (botConfig: BotConfiguration) (logger: ILogger) (ml: MachineLearning) (message: Message) = task { - use banOnReplyActivity = botActivity.StartActivity("onUpdate") + use banOnReplyActivity = botActivity.StartActivity("onMessage") // early return if we can't process it if isNull message || isNull message.From then @@ -557,20 +588,119 @@ let onUpdate .SetTag("chatUsername", message.Chat.Username) // upserting user to DB - let! _ = + let! user = DbUser.newUser message.From |> DB.upsertUser - |> taskIgnore // check if message comes from channel, we should delete it immediately if botConfig.ShouldDeleteChannelMessages && isChannelMessage message then do! deleteChannelMessage botClient message logger // check if message is a known command from authorized user - elif isKnownCommand message && isMessageFromAdmin botConfig message then - do! adminCommand botClient botConfig logger message + elif isKnownCommand message && isUserVahter botConfig user then + do! adminCommand botClient botConfig logger user message // if message is not a command from authorized user, just save it ID to DB else do! justMessage botClient botConfig logger ml message } + +let vahterMarkedAsNotSpam + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (logger: ILogger) + (vahter: DbUser) + (msg: MessageWrapper) = task { + let msgId = msg.message.MessageId + let chatId = msg.message.Chat.Id + let chatName = msg.message.Chat.Username + use _ = + botActivity + .StartActivity("vahterMarkedAsNotSpam") + .SetTag("messageId", msgId) + .SetTag("chatId", chatId) + let dbMessage = DbMessage.newMessage msg.message + do! DB.markMessageAsFalsePositive dbMessage + do! DB.unbanUserByBot dbMessage + + let vahterUsername = vahter.username |> Option.defaultValue null + + let logMsg = $"Vahter {prependUsername vahterUsername} ({vahter.id}) marked message {msgId} in {prependUsername chatName}({chatId}) as false-positive (NOT A SPAM)\n{msg.message.Text}" + do! botClient.SendTextMessageAsync(ChatId(botConfig.LogsChannelId), logMsg) |> taskIgnore + logger.LogInformation logMsg +} + +let vahterMarkedAsSpam + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (logger: ILogger) + (vahter: DbUser) + (message: MessageWrapper) = task { + let msgId = message.message.MessageId + let chatId = message.message.Chat.Id + use _ = + botActivity + .StartActivity("vahterMarkedAsSpam") + .SetTag("messageId", msgId) + .SetTag("chatId", chatId) + + let isAuthed = isBanAuthorized botConfig message.message vahter logger + if isAuthed then + do! totalBan + botClient + botConfig + message.message + vahter + logger +} + +let onCallback + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (logger: ILogger) + (callbackQuery: CallbackQuery) = task { + use onCallbackActivity = botActivity.StartActivity("onCallback") + %onCallbackActivity.SetTag("callbackId", callbackQuery.Data) + + let callbackId = Guid.Parse callbackQuery.Data + + match! DB.getCallback callbackId with + | None -> + logger.LogWarning $"Callback {callbackId} not found in DB" + | Some dbCallback -> + %onCallbackActivity.SetTag("callbackData", dbCallback.data) + let callback = dbCallback.data + match! DB.getUserById callbackQuery.From.Id with + | None -> + logger.LogWarning $"User {callbackQuery.From.Username} ({callbackQuery.From.Id}) tried to press callback button while not being in DB" + | Some vahter -> + %onCallbackActivity.SetTag("vahterUsername", vahter.username) + %onCallbackActivity.SetTag("vahterId", vahter.id) + + // only vahters should be able to press message buttons + let isAuthed = botConfig.AllowedUsers.ContainsValue vahter.id + if not isAuthed then + logger.LogWarning $"User {callbackQuery.From.Username} ({callbackQuery.From.Id}) tried to press callback button while not being a certified vahter" + else + match callback with + | NotASpam msg -> + %onCallbackActivity.SetTag("type", "NotASpam") + do! vahterMarkedAsNotSpam botClient botConfig logger vahter msg + | Spam msg -> + %onCallbackActivity.SetTag("type", "Spam") + do! vahterMarkedAsSpam botClient botConfig logger vahter msg + do! DB.deleteCallback callbackId +} + +let onUpdate + (botClient: ITelegramBotClient) + (botConfig: BotConfiguration) + (logger: ILogger) + (ml: MachineLearning) + (update: Update) = task { + use _ = botActivity.StartActivity("onUpdate") + if update.CallbackQuery <> null then + do! onCallback botClient botConfig logger update.CallbackQuery + else + do! onMessage botClient botConfig logger ml update.Message +} diff --git a/src/VahterBanBot/Cleanup.fs b/src/VahterBanBot/Cleanup.fs index 6cf92be..5087161 100644 --- a/src/VahterBanBot/Cleanup.fs +++ b/src/VahterBanBot/Cleanup.fs @@ -23,10 +23,12 @@ type CleanupService( if botConf.CleanupOldMessages then let! cleanupMsgs = DB.cleanupOldMessages botConf.CleanupOldLimit %sb.AppendLine $"Cleaned up {cleanupMsgs} messages from DB which are older than {timeSpanAsHumanReadable botConf.CleanupOldLimit}" + let! cleanupCallbacks = DB.cleanupOldCallbacks botConf.CleanupOldLimit + %sb.AppendLine $"Cleaned up {cleanupCallbacks} callbacks from DB which are older than {timeSpanAsHumanReadable botConf.CleanupOldLimit}" let! vahterStats = DB.getVahterStats (Some botConf.CleanupInterval) %sb.AppendLine(string vahterStats) - + let msg = sb.ToString() do! telegramClient.SendTextMessageAsync( ChatId(botConf.LogsChannelId), @@ -34,7 +36,7 @@ type CleanupService( ) |> taskIgnore logger.LogInformation msg } - + interface IHostedService with member this.StartAsync _ = if not botConf.IgnoreSideEffects then diff --git a/src/VahterBanBot/DB.fs b/src/VahterBanBot/DB.fs index 8602b9a..09326b5 100644 --- a/src/VahterBanBot/DB.fs +++ b/src/VahterBanBot/DB.fs @@ -59,8 +59,9 @@ let banUser (banned: DbBanned): Task = """ INSERT INTO banned (message_id, message_text, banned_user_id, banned_at, banned_in_chat_id, banned_in_chat_username, banned_by) VALUES (@message_id, @message_text, @banned_user_id, @banned_at, @banned_in_chat_id, @banned_in_chat_username, @banned_by) +ON CONFLICT (banned_user_id) DO NOTHING; """ - + let! _ = conn.ExecuteAsync(sql, banned) return banned } @@ -80,6 +81,20 @@ VALUES (@message_id, @message_text, @banned_user_id, @banned_at, @banned_in_chat return banned } +let unbanUserByBot (msg: DbMessage) : Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = + """ +DELETE FROM banned_by_bot WHERE message_id = @message_id and banned_in_chat_id = @chat_id + """ + + let! _ = conn.ExecuteAsync(sql, msg) + return () + } + let getUserMessages (userId: int64): Task = task { use conn = new NpgsqlConnection(connString) @@ -109,6 +124,15 @@ let cleanupOldMessages (howOld: TimeSpan): Task = let sql = "DELETE FROM message WHERE created_at < @thatOld" return! conn.ExecuteAsync(sql, {| thatOld = DateTime.UtcNow.Subtract howOld |}) } + +let cleanupOldCallbacks (howOld: TimeSpan): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = "DELETE FROM callback WHERE created_at < @thatOld" + return! conn.ExecuteAsync(sql, {| thatOld = DateTime.UtcNow.Subtract howOld |}) + } let getVahterStats(banInterval: TimeSpan option): Task = task { @@ -124,11 +148,12 @@ let getVahterStats(banInterval: TimeSpan option): Task = JOIN "user" vahter ON vahter.id = b.banned_by GROUP BY b.banned_by, vahter.username UNION - SELECT 'bot' AS vahter - , COUNT(*) AS killCountTotal - , COUNT(*) FILTER (WHERE bbb.banned_at > NOW() - @banInterval::INTERVAL) AS killCountInterval - FROM banned_by_bot bbb - GROUP BY bbb.banned_user_id) + SELECT 'bot' AS vahter, + COUNT(*) AS killCountTotal, + COUNT(*) FILTER (WHERE bbb.banned_at > NOW() - NULL::INTERVAL) AS killCountInterval + FROM (SELECT banned_user_id, MIN(banned_at) AS banned_at + FROM banned_by_bot + GROUP BY banned_user_id) bbb) ORDER BY killCountTotal DESC """ @@ -163,7 +188,10 @@ WITH really_banned AS (SELECT * FROM banned b -- known false positive spam messages WHERE NOT EXISTS(SELECT 1 FROM false_positive_users fpu WHERE fpu.user_id = b.banned_user_id) - AND NOT EXISTS(SELECT 1 FROM false_positive_messages fpm WHERE fpm.id = b.id) + AND NOT EXISTS(SELECT 1 + FROM false_positive_messages fpm + WHERE fpm.chat_id = b.banned_in_chat_id + AND fpm.message_id = b.message_id) AND b.message_text IS NOT NULL AND b.banned_at <= @criticalDate), spam_or_ham AS (SELECT DISTINCT COALESCE(m.text, re_id.message_text) AS text, @@ -190,3 +218,70 @@ ORDER BY RANDOM(); let! data = conn.QueryAsync(sql, {| criticalDate = criticalDate |}) return Array.ofSeq data } + +let unbanUser (userId: int64): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = + """ +DELETE FROM banned +WHERE banned_user_id = @userId + """ + + let! _ = conn.ExecuteAsync(sql, {| userId = userId |}) + return () + } + +let markMessageAsFalsePositive (message: DbMessage): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = + """ +INSERT INTO false_positive_messages (chat_id, message_id) +VALUES (@chat_id, @message_id) +ON CONFLICT DO NOTHING; + """ + + return! conn.ExecuteAsync(sql, message) + } + +let newCallback (data: CallbackMessage): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = + """ +INSERT INTO callback (data) +VALUES (@data::JSONB) +RETURNING *; + """ + + return! conn.QuerySingleAsync(sql, {| data = data |}) + } + +let getCallback (id: Guid): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = "SELECT * FROM callback WHERE id = @id" + + let! result = conn.QueryAsync(sql, {| id = id |}) + return Seq.tryHead result + } + +let deleteCallback (id: Guid): Task = + task { + use conn = new NpgsqlConnection(connString) + + //language=postgresql + let sql = "DELETE FROM callback WHERE id = @id" + + let! _ = conn.QueryAsync(sql, {| id = id |}) + return () + } diff --git a/src/VahterBanBot/ML.fs b/src/VahterBanBot/ML.fs index 359dd88..38efdb8 100644 --- a/src/VahterBanBot/ML.fs +++ b/src/VahterBanBot/ML.fs @@ -95,6 +95,7 @@ type MachineLearning( member this.StartAsync _ = task { if botConf.MlEnabled then try + logger.LogInformation "Training model..." do! trainModel() with ex -> logger.LogError(ex, "Error training model") diff --git a/src/VahterBanBot/Program.fs b/src/VahterBanBot/Program.fs index 5689be0..c5b961a 100644 --- a/src/VahterBanBot/Program.fs +++ b/src/VahterBanBot/Program.fs @@ -4,6 +4,7 @@ open System open System.Collections.Generic open System.Threading open System.Threading.Tasks +open Dapper open Microsoft.AspNetCore.Builder open Microsoft.AspNetCore.Http open Microsoft.Extensions.Logging @@ -31,6 +32,7 @@ open Azure.Monitor.OpenTelemetry.AspNetCore type Root = class end Dapper.FSharp.PostgreSQL.OptionTypes.register() +SqlMapper.AddTypeHandler(CallbackMessageTypeHandler()); let botConf = { BotToken = getEnv "BOT_TELEGRAM_TOKEN" @@ -52,6 +54,7 @@ let botConf = MlTrainBeforeDate = getEnvOrWith "ML_TRAIN_BEFORE_DATE" DateTime.UtcNow (DateTimeOffset.Parse >> _.UtcDateTime) MlTrainingSetFraction = getEnvOr "ML_TRAINING_SET_FRACTION" "0.2" |> float MlSpamThreshold = getEnvOr "ML_SPAM_THRESHOLD" "0.5" |> single + MlWarningThreshold = getEnvOr "ML_WARNING_THRESHOLD" "0.0" |> single MlStopWordsInChats = getEnvOr "ML_STOP_WORDS_IN_CHATS" "{}" |> JsonConvert.DeserializeObject<_> } let validateApiKey (ctx : HttpContext) = @@ -145,7 +148,7 @@ let webApp = choose [ let ml = scope.ServiceProvider.GetRequiredService() let logger = ctx.GetLogger() try - do! onUpdate telegramClient botConf (ctx.GetLogger "VahterBanBot.Bot") ml update.Message + do! onUpdate telegramClient botConf (ctx.GetLogger "VahterBanBot.Bot") ml update %topActivity.SetTag("update-error", false) with e -> logger.LogError(e, $"Unexpected error while processing update: {updateBodyJson}") @@ -172,7 +175,7 @@ if botConf.UsePolling then let logger = ctx.ServiceProvider.GetRequiredService>() let client = ctx.ServiceProvider.GetRequiredService() let ml = ctx.ServiceProvider.GetRequiredService() - do! onUpdate client botConf logger ml update.Message + do! onUpdate client botConf logger ml update } member x.HandlePollingErrorAsync (botClient: ITelegramBotClient, ex: Exception, cancellationToken: CancellationToken) = Task.CompletedTask diff --git a/src/VahterBanBot/Types.fs b/src/VahterBanBot/Types.fs index e5f0099..320172a 100644 --- a/src/VahterBanBot/Types.fs +++ b/src/VahterBanBot/Types.fs @@ -3,7 +3,9 @@ open System open System.Collections.Generic open System.Text +open Dapper open Newtonsoft.Json +open Telegram.Bot.Types open Utils [] @@ -27,6 +29,7 @@ type BotConfiguration = MlTrainBeforeDate: DateTime MlTrainingSetFraction: float MlSpamThreshold: single + MlWarningThreshold: single MlStopWordsInChats: Dictionary } [] @@ -42,7 +45,7 @@ type DbUser = updated_at = DateTime.UtcNow created_at = DateTime.UtcNow } - static member newUser(user: Telegram.Bot.Types.User) = + static member newUser(user: User) = DbUser.newUser (id = user.Id, ?username = Option.ofObj user.Username) [] @@ -55,7 +58,7 @@ type DbBanned = banned_in_chat_username: string option banned_by: int64 } module DbBanned = - let banMessage (vahter: int64) (message: Telegram.Bot.Types.Message) = + let banMessage (vahter: int64) (message: Message) = if isNull message.From || isNull message.Chat then failwith "Message should have a user and a chat" { message_id = Some message.MessageId @@ -115,3 +118,26 @@ type VahterStats = |> Array.iteri (fun i stat -> %sb.AppendLine $"%d{i+1} {prependUsername stat.Vahter} - {stat.KillCountTotal}") sb.ToString() + +// used as aux type to possibly extend in future without breaking changes +type MessageWrapper= { message: Message } + +// This type must be backwards compatible with the previous version +// as it is used to (de)serialize the button callback data +type CallbackMessage = + | NotASpam of MessageWrapper + | Spam of MessageWrapper + +[] +type DbCallback = + { id: Guid + data: CallbackMessage + created_at: DateTime } + +type CallbackMessageTypeHandler() = + inherit SqlMapper.TypeHandler() + + override this.SetValue(parameter, value) = + parameter.Value <- JsonConvert.SerializeObject value + override this.Parse(value) = + JsonConvert.DeserializeObject(value.ToString()) diff --git a/src/VahterBanBot/VahterBanBot.fsproj b/src/VahterBanBot/VahterBanBot.fsproj index 7075343..de9805f 100644 --- a/src/VahterBanBot/VahterBanBot.fsproj +++ b/src/VahterBanBot/VahterBanBot.fsproj @@ -11,7 +11,6 @@ - diff --git a/src/migrations/V10__unique-banned.sql b/src/migrations/V10__unique-banned.sql new file mode 100644 index 0000000..ea9924d --- /dev/null +++ b/src/migrations/V10__unique-banned.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS banned_banned_user_id_idx; +CREATE UNIQUE INDEX banned_banned_user_id_idx ON banned (banned_user_id); diff --git a/src/migrations/V9__callbacks.sql b/src/migrations/V9__callbacks.sql new file mode 100644 index 0000000..7563158 --- /dev/null +++ b/src/migrations/V9__callbacks.sql @@ -0,0 +1,34 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +CREATE TABLE callback +( + internal_id BIGSERIAL PRIMARY KEY, + id UUID NOT NULL DEFAULT uuid_generate_v4(), + data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT timezone('utc'::TEXT, NOW()) +); + +CREATE INDEX idx_callback_data ON callback USING GIN (data); +CREATE UNIQUE INDEX idx_callback_id ON callback (id); + +ALTER TABLE false_positive_messages + ADD COLUMN message_id INTEGER NULL, + ADD COLUMN chat_id BIGINT NULL; + +UPDATE false_positive_messages +SET message_id=b.message_id, + chat_id=b.banned_in_chat_id +FROM (SELECT b.message_id, b.banned_in_chat_id, b.id + FROM false_positive_messages fpm + JOIN public.banned b ON b.id = fpm.id) AS b +WHERE false_positive_messages.id = b.id; + +ALTER TABLE false_positive_messages + ALTER COLUMN message_id SET NOT NULL, + ALTER COLUMN chat_id SET NOT NULL; + +CREATE UNIQUE INDEX idx_false_positive_messages_chat_id_message_id + ON false_positive_messages (chat_id, message_id); + +ALTER TABLE false_positive_messages + DROP COLUMN id;