From 903c0d0dfd3e5f9a5763ec96285f98736f71296d Mon Sep 17 00:00:00 2001 From: Ayrat Hudaygulov Date: Wed, 7 Aug 2024 11:42:55 +0100 Subject: [PATCH] local admins in chats won't trigger ML deletion --- .env.example | 2 + src/VahterBanBot.Tests/BanTests.fs | 14 ++-- src/VahterBanBot.Tests/ContainerTestBase.fs | 8 ++- src/VahterBanBot.Tests/MLBanTests.fs | 17 ++++- src/VahterBanBot/Bot.fs | 6 +- src/VahterBanBot/FakeTgApi.fs | 74 ++++++++++++++------- src/VahterBanBot/Program.fs | 4 ++ src/VahterBanBot/Types.fs | 2 + src/VahterBanBot/UpdateChatAdmins.fs | 58 ++++++++++++++++ src/VahterBanBot/VahterBanBot.fsproj | 1 + 10 files changed, 150 insertions(+), 36 deletions(-) create mode 100644 src/VahterBanBot/UpdateChatAdmins.fs diff --git a/.env.example b/.env.example index 35b8030..062a3c9 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,5 @@ ML_TRAINING_SET_FRACTION=0.2 ML_SPAM_THRESHOLD=0.5 ML_WARNING_THRESHOLD=0.0 ML_STOP_WORDS_IN_CHATS={"-123":["word1","word2"]} +UPDATE_CHAT_ADMINS_INTERVAL_SEC=86400 +UPDATE_CHAT_ADMINS=true diff --git a/src/VahterBanBot.Tests/BanTests.fs b/src/VahterBanBot.Tests/BanTests.fs index bf9f3c5..e2853f9 100644 --- a/src/VahterBanBot.Tests/BanTests.fs +++ b/src/VahterBanBot.Tests/BanTests.fs @@ -16,7 +16,7 @@ type BanTests(fixture: VahterTestContainers) = // send the ban message let! banResp = - Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.Vahters[0]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) @@ -50,7 +50,7 @@ type BanTests(fixture: VahterTestContainers) = // send the ban message let! banResp = - Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.Vahters[0]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) @@ -62,12 +62,12 @@ type BanTests(fixture: VahterTestContainers) = [] let ``Vahter can't ban another vahter`` () = task { // record a message in a random chat - let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], from = fixture.AdminUsers[0]) + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], from = fixture.Vahters[0]) let! _ = fixture.SendMessage msgUpdate // send the ban message let! banResp = - Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[1]) + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.Vahters[1]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) @@ -84,7 +84,7 @@ type BanTests(fixture: VahterTestContainers) = // send the ban message let! banResp = - Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.Vahters[0]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) @@ -94,7 +94,7 @@ type BanTests(fixture: VahterTestContainers) = // send the unban message from another vahter let! banResp = - Tg.quickMsg($"/unban {msgUpdate.Message.From.Id}", chat = fixture.ChatsToMonitor[0], from = fixture.AdminUsers[1]) + Tg.quickMsg($"/unban {msgUpdate.Message.From.Id}", chat = fixture.ChatsToMonitor[0], from = fixture.Vahters[1]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) @@ -111,7 +111,7 @@ type BanTests(fixture: VahterTestContainers) = // send the ban message let! banResp = - Tg.replyMsg(msgUpdate.Message, "/ban", fixture.AdminUsers[0]) + Tg.replyMsg(msgUpdate.Message, "/ban", fixture.Vahters[0]) |> fixture.SendMessage Assert.Equal(HttpStatusCode.OK, banResp.StatusCode) diff --git a/src/VahterBanBot.Tests/ContainerTestBase.fs b/src/VahterBanBot.Tests/ContainerTestBase.fs index b36e756..cc9a562 100644 --- a/src/VahterBanBot.Tests/ContainerTestBase.fs +++ b/src/VahterBanBot.Tests/ContainerTestBase.fs @@ -103,6 +103,8 @@ type VahterTestContainers() = // https://learn.microsoft.com/en-us/dotnet/core/compatibility/containers/8.0/aspnet-port // Azure default port for containers is 80, se we need explicitly set it .WithEnvironment("ASPNETCORE_HTTP_PORTS", "80") + .WithEnvironment("UPDATE_CHAT_ADMINS", "true") + .WithEnvironment("UPDATE_CHAT_ADMINS_INTERVAL_SEC", "86400") .DependsOn(flywayContainer) .WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(80)) .Build() @@ -172,10 +174,14 @@ type VahterTestContainers() = return resp } - member _.AdminUsers = [ + member _.Vahters = [ Tg.user(id = 34, username = "vahter_1") Tg.user(id = 69, username = "vahter_2") ] + + member _.Admins = [ + Tg.user(id = 42, username = "just_admin") + ] member _.LogChat = Tg.chat(id = -123, username = "logs") member _.ChatsToMonitor = [ diff --git a/src/VahterBanBot.Tests/MLBanTests.fs b/src/VahterBanBot.Tests/MLBanTests.fs index 43e543b..37dd675 100644 --- a/src/VahterBanBot.Tests/MLBanTests.fs +++ b/src/VahterBanBot.Tests/MLBanTests.fs @@ -25,7 +25,20 @@ type MLBanTests(fixture: VahterTestContainers, _unused: MlAwaitFixture) = // record a message, where 2 is in a training set as spam word // ChatsToMonitor[0] doesn't have stopwords // but it was sent by vahter - let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2222222", from = fixture.AdminUsers[0]) + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2222222", from = fixture.Vahters[0]) + let! _ = fixture.SendMessage msgUpdate + + // assert that the message got auto banned + let! msgBanned = fixture.MessageIsAutoBanned msgUpdate.Message + Assert.False msgBanned + } + + [] + let ``Message is NOT autobanned if it looks like a spam BUT local admin sent it`` () = task { + // record a message, where 2 is in a training set as spam word + // ChatsToMonitor[0] doesn't have stopwords + // but it was sent by local admin + let msgUpdate = Tg.quickMsg(chat = fixture.ChatsToMonitor[0], text = "2222222", from = fixture.Admins[0]) let! _ = fixture.SendMessage msgUpdate // assert that the message got auto banned @@ -83,7 +96,7 @@ type MLBanTests(fixture: VahterTestContainers, _unused: MlAwaitFixture) = // send a callback to mark it as false-positive let! callbackId = fixture.GetCallbackId msgUpdate.Message "NotASpam" - let msgCallback = Tg.callback(string callbackId, from = fixture.AdminUsers[0]) + let msgCallback = Tg.callback(string callbackId, from = fixture.Vahters[0]) let! _ = fixture.SendMessage msgCallback // assert it is false-positive diff --git a/src/VahterBanBot/Bot.fs b/src/VahterBanBot/Bot.fs index 48f1538..b93547a 100644 --- a/src/VahterBanBot/Bot.fs +++ b/src/VahterBanBot/Bot.fs @@ -11,6 +11,7 @@ open Telegram.Bot.Types.ReplyMarkups open VahterBanBot.ML open VahterBanBot.Types open VahterBanBot.Utils +open VahterBanBot.UpdateChatAdmins let botActivity = new ActivitySource("VahterBanBot") @@ -463,8 +464,9 @@ let justMessage use mlActivity = botActivity.StartActivity("mlPrediction") let shouldBeSkipped = - // skip prediction for vahters - if botConfig.AllowedUsers.ContainsValue message.From.Id then + // skip prediction for vahters or local admins + if botConfig.AllowedUsers.ContainsValue message.From.Id + || UpdateChatAdmins.Admins.Contains message.From.Id then true else diff --git a/src/VahterBanBot/FakeTgApi.fs b/src/VahterBanBot/FakeTgApi.fs index 8c8bff7..d11261f 100644 --- a/src/VahterBanBot/FakeTgApi.fs +++ b/src/VahterBanBot/FakeTgApi.fs @@ -4,6 +4,7 @@ open System open System.Net open System.Net.Http open System.Text +open System.Threading.Tasks open Newtonsoft.Json open Telegram.Bot.Types open Telegram.Bot.Types.Enums @@ -11,34 +12,59 @@ open VahterBanBot.Types let fakeTgApi (botConf: BotConfiguration) = { new DelegatingHandler() with - member x.SendAsync(request, cancellationToken) = task { + member x.SendAsync(request, cancellationToken) = let apiResult text = let resp = new HttpResponseMessage(HttpStatusCode.OK) resp.Content <- new StringContent($"""{{"Ok":true,"Result":{text}}}""", Encoding.UTF8, "application/json") resp - + let url = request.RequestUri.ToString() - if not(url.StartsWith("https://api.telegram.org/bot" + botConf.BotToken)) then - // return 404 for any other request - return new HttpResponseMessage(HttpStatusCode.NotFound) - elif url.EndsWith "/deleteMessage" || url.EndsWith "/banChatMember" then - // respond with "true" - return apiResult "true" - elif url.EndsWith "/sendMessage" then - // respond with the request body as a string - let message = - Message( - MessageId = 1, - Date = DateTime.UtcNow, - Chat = Chat( - Id = 1L, - Type = ChatType.Private + let resp = + if not(url.StartsWith("https://api.telegram.org/bot" + botConf.BotToken)) then + // return 404 for any other request + new HttpResponseMessage(HttpStatusCode.NotFound) + elif url.EndsWith "/deleteMessage" || url.EndsWith "/banChatMember" then + // respond with "true" + apiResult "true" + elif url.EndsWith "/sendMessage" then + // respond with the request body as a string + let message = + Message( + MessageId = 1, + Date = DateTime.UtcNow, + Chat = Chat( + Id = 1L, + Type = ChatType.Private + ) ) - ) - |> JsonConvert.SerializeObject - return apiResult message - else - // return 500 for any other request - return new HttpResponseMessage(HttpStatusCode.InternalServerError) - } + |> JsonConvert.SerializeObject + apiResult message + elif url.EndsWith "/getChatAdministrators" then + // respond with the request body as a string + let message = + [| + ChatMemberAdministrator( + CanBeEdited = false, + IsAnonymous = false, + CanDeleteMessages = false, + CanManageVideoChats = false, + CanRestrictMembers = false, + CanPromoteMembers = false, + CanChangeInfo = false, + CanInviteUsers = false, + User = User( + Id = 42L, + FirstName = "just_admin", + Username = "just_admin" + ) + ) + |] + |> JsonConvert.SerializeObject + apiResult message + else + // return 500 for any other request + // TODO pass fucking ILogger here somehow -_- + Console.WriteLine $"Unhandled request: {url}" + new HttpResponseMessage(HttpStatusCode.InternalServerError) + Task.FromResult resp } diff --git a/src/VahterBanBot/Program.fs b/src/VahterBanBot/Program.fs index c5b961a..4950a6e 100644 --- a/src/VahterBanBot/Program.fs +++ b/src/VahterBanBot/Program.fs @@ -22,6 +22,7 @@ open VahterBanBot.Utils open VahterBanBot.Bot open VahterBanBot.Types open VahterBanBot.StartupMessage +open VahterBanBot.UpdateChatAdmins open VahterBanBot.FakeTgApi open OpenTelemetry.Trace open OpenTelemetry.Metrics @@ -48,6 +49,8 @@ let botConf = CleanupOldMessages = getEnvOr "CLEANUP_OLD_MESSAGES" "true" |> bool.Parse CleanupInterval = getEnvOr "CLEANUP_INTERVAL_SEC" "86400" |> int |> TimeSpan.FromSeconds CleanupOldLimit = getEnvOr "CLEANUP_OLD_LIMIT_SEC" "259200" |> int |> TimeSpan.FromSeconds + UpdateChatAdminsInterval = getEnvOrWith "UPDATE_CHAT_ADMINS_INTERVAL_SEC" None (int >> TimeSpan.FromSeconds >> Some) + UpdateChatAdmins = getEnvOr "UPDATE_CHAT_ADMINS" "false" |> bool.Parse MlEnabled = getEnvOr "ML_ENABLED" "false" |> bool.Parse MlSeed = getEnvOrWith "ML_SEED" (Nullable()) (int >> Nullable) MlSpamDeletionEnabled = getEnvOr "ML_SPAM_DELETION_ENABLED" "false" |> bool.Parse @@ -71,6 +74,7 @@ let builder = WebApplication.CreateBuilder() .AddGiraffe() .AddHostedService() .AddHostedService() + .AddHostedService() .AddSingleton() .AddHostedService(fun sp -> sp.GetRequiredService()) .AddHttpClient("telegram_bot_client") diff --git a/src/VahterBanBot/Types.fs b/src/VahterBanBot/Types.fs index 320172a..c1dd352 100644 --- a/src/VahterBanBot/Types.fs +++ b/src/VahterBanBot/Types.fs @@ -23,6 +23,8 @@ type BotConfiguration = CleanupOldMessages: bool CleanupInterval: TimeSpan CleanupOldLimit: TimeSpan + UpdateChatAdminsInterval: TimeSpan option + UpdateChatAdmins: bool MlEnabled: bool MlSeed: Nullable MlSpamDeletionEnabled: bool diff --git a/src/VahterBanBot/UpdateChatAdmins.fs b/src/VahterBanBot/UpdateChatAdmins.fs new file mode 100644 index 0000000..e65ec39 --- /dev/null +++ b/src/VahterBanBot/UpdateChatAdmins.fs @@ -0,0 +1,58 @@ +module VahterBanBot.UpdateChatAdmins + +open System.Collections.Generic +open System.Text +open System.Threading.Tasks +open Microsoft.Extensions.Logging +open Telegram.Bot +open Telegram.Bot.Types +open VahterBanBot.Types +open VahterBanBot.Utils +open System +open System.Threading +open Microsoft.Extensions.Hosting + +type UpdateChatAdmins( + logger: ILogger, + telegramClient: ITelegramBotClient, + botConf: BotConfiguration +) = + let mutable timer: Timer = null + static let mutable localAdmins: ISet = HashSet() + + let updateChatAdmins _ = task { + let sb = StringBuilder() + %sb.AppendLine("New chat admins:") + let result = HashSet() + for chatId in botConf.ChatsToMonitor.Values do + let! admins = telegramClient.GetChatAdministratorsAsync(ChatId chatId) + + // wait a bit so we don't get rate limited + do! Task.Delay 100 + + for admin in admins do + %result.Add admin.User.Id + %sb.AppendJoin(",", $"{prependUsername admin.User.Username} ({admin.User.Id})") + localAdmins <- result + logger.LogInformation (sb.ToString()) + } + + static member Admins = localAdmins + + interface IHostedService with + member this.StartAsync _ = + if not botConf.IgnoreSideEffects && botConf.UpdateChatAdmins then + if botConf.UpdateChatAdminsInterval.IsSome then + // recurring + timer <- new Timer(TimerCallback(updateChatAdmins >> ignore), null, TimeSpan.Zero, botConf.UpdateChatAdminsInterval.Value) + Task.CompletedTask + else + // once + updateChatAdmins() + else + Task.CompletedTask + + member this.StopAsync _ = + match timer with + | null -> Task.CompletedTask + | timer -> timer.DisposeAsync().AsTask() diff --git a/src/VahterBanBot/VahterBanBot.fsproj b/src/VahterBanBot/VahterBanBot.fsproj index de9805f..e49d896 100644 --- a/src/VahterBanBot/VahterBanBot.fsproj +++ b/src/VahterBanBot/VahterBanBot.fsproj @@ -11,6 +11,7 @@ +