-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[create-pull-request] automated change
- Loading branch information
Showing
1 changed file
with
85 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -27,6 +27,7 @@ final class Bots | |
|AmorankSpider | ||
|ApacheBench | ||
|Applebot | ||
|iTMS | ||
|AppSignalBot | ||
|Arachni | ||
|AspiegelBot | ||
|
@@ -40,6 +41,7 @@ final class Bots | |
|BazQux | ||
|Better\sUptime\sBot | ||
|MSNBot|msrbot|bingbot|bingadsbot|BingPreview|msnbot-(UDiscovery|NewsBlogs)|adidxbot | ||
|Blackbox\sExporter | ||
|Blekkobot | ||
|BLEXBot | ||
|Bloglovin | ||
|
@@ -58,6 +60,8 @@ final class Bots | |
|Cloudflare-SSLDetector | ||
|Cloudflare\sCustom\sHostname\sVerification | ||
|Cloudflare-Traffic-Manager | ||
|Cloudflare-Smart-Transit | ||
|CloudflareObservatory | ||
|https://developers\.cloudflare\.com/security-center/ | ||
|coccoc\.com | ||
|collectd | ||
|
@@ -72,14 +76,17 @@ final class Bots | |
|Domain\sRe-Animator\sBot|support@domainreanimator\.com | ||
|DotBot | ||
|DuckDuck(?:Go-Favicons-)?Bot | ||
|DuckAssistBot | ||
|EasouSpider | ||
|eCairn-Grabber | ||
|EMail\sExractor | ||
|evc-batch | ||
|Exabot|ExaleadCloudview | ||
|ExactSeek\sCrawler | ||
|Ezooms | ||
|facebookexternalhit|facebookplatform|facebookexternalua|facebookcatalog | ||
|facebook(?:catalog|externalhit|externalua|platform|scraper) | ||
|meta-externalagent | ||
|meta-externalfetcher | ||
|FacebookBot/[\d.]+ | ||
|Feedbin | ||
|FeedBurner | ||
|
@@ -104,18 +111,25 @@ final class Bots | |
|Google-Structured-Data-Testing-Tool | ||
|GoogleStackdriverMonitoring | ||
|Google-Transparency-Report | ||
|Google-CloudVertexBot | ||
|via\sggpht\.com\sGoogleImageProxy | ||
|Google-Document-Conversion | ||
|GoogleDocs;\sapps-spreadsheets | ||
|GoogleDocs;\sapps-presentations | ||
|GoogleDocs; | ||
|SeznamEmailProxy | ||
|Seznam-Zbozi-robot | ||
|Heurekabot-Feed | ||
|ShopAlike | ||
|deepcrawl\.com | ||
|Googlebot-News | ||
|Adwords-(?:DisplayAds|Express|Instant)|Google\sWeb\sPreview|Google[\s-]Publisher[\s-]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Safety|Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|DuplexWeb|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\+/web/snippet | ||
|Adwords-(?:DisplayAds|Express|Instant)|Google\sWeb\sPreview|Google[\s-]Publisher[\s-]Plugin|Google-(?:adstxt|Ads-Conversions|Ads-Qualify|Adwords|AMPHTML|Assess|Extended|HotelAdsVerifier|InspectionTool|Lens|PageRenderer|Read-Aloud|Shopping-Quality|Site-Verification|Sites-Thumbnails|speakr|Stale-Content-Probe|Test|Youtube-Links)|(?:AdsBot|APIs|Feedfetcher|Mediapartners)-Google(?:-Mobile)?|Google(?:AdSenseInfeed|AssociationService|bot|Other|Prober|Producer|Sites)|Google.*/\+/web/snippet | ||
|^Google$ | ||
|Google-Safety | ||
|DuplexWeb-Google | ||
|Google-Area120-PrivacyPolicyFetcher | ||
|heritrix | ||
|HubSpot\s | ||
|vuhuvBot | ||
|vuhuv(?:Bot|RBT) | ||
|HTTPMon/[\d.]+ | ||
|ICC-Crawler | ||
|inoreader\.com | ||
|
@@ -134,7 +148,7 @@ final class Bots | |
|magpie-crawler | ||
|MagpieRSS | ||
|masscan-ng/[\d.]+ | ||
|masscan | ||
|.*masscan | ||
|Mastodon/ | ||
|meanpathbot | ||
|MetaJobBot | ||
|
@@ -155,6 +169,7 @@ final class Bots | |
|nlcrawler | ||
|Nmap\sScripting\sEngine | ||
|Nuzzel | ||
|NodePing | ||
|Octopus\s[0-9] | ||
|OnlineOrNot\.com_bot | ||
|omgili | ||
|
@@ -174,7 +189,7 @@ final class Bots | |
|Quora-Bot | ||
|RamblerMail | ||
|QuerySeekerSpider | ||
|Qwantify | ||
|Qwantify|Qwantbot | ||
|Rainmeter | ||
|redditbot | ||
|Riddler | ||
|
@@ -252,6 +267,8 @@ final class Bots | |
|websitepulse[+\s]checker | ||
|WordPress.+isitwp\.com | ||
|Automattic\sAnalytics\sCrawler/[\d.]+ | ||
|WordPress\.com\smShots | ||
|wp\.com\sfeedbot | ||
|WordPress | ||
|Wotbox | ||
|XenForo | ||
|
@@ -332,6 +349,7 @@ final class Bots | |
|Startpagina-Linkchecker | ||
|MoodleBot-Linkchecker | ||
|GTmetrix | ||
|CyberFind\s?Crawler | ||
|Nutch | ||
|Seobility | ||
|Vercelbot | ||
|
@@ -346,7 +364,7 @@ final class Bots | |
|Barkrowler | ||
|BDCbot | ||
|adbeat | ||
|BW/[\d.]+ | ||
|(?:BuiltWith|BW)/[\d.]+ | ||
|https://whatis\.contentkingapp\.com | ||
|MicroAdBot | ||
|PingAdmin\.Ru | ||
|
@@ -403,6 +421,7 @@ final class Bots | |
|seolyt/[\d.]+ | ||
|YaK/[\d.]+ | ||
|KomodiaBot/[\d.]+ | ||
|KStandBot/[\d.]+ | ||
|Neevabot/[\d.]+ | ||
|LinkPreview/[\d.]+ | ||
|JungleKeyThumbnail/[\d.]+ | ||
|
@@ -426,7 +445,7 @@ final class Bots | |
|OnalyticaBot | ||
|deepnoc | ||
|Newslitbot/[\d.]+ | ||
|um-LN/[\d.]+ | ||
|um-(?:ANS|CC|FC|IC|LN)/[\d.]+ | ||
|Abonti/[\d.]+ | ||
|collection@infegy\.com | ||
|HTTP\sBanner\sDetection\s\(https://security\.ipip\.net\) | ||
|
@@ -452,6 +471,7 @@ final class Bots | |
|CriteoBot/ | ||
|PayPal\sIPN | ||
|MaCoCu | ||
|CLASSLA | ||
|dnt-policy@eff\.org | ||
|InfoTigerBot | ||
|(?:Birdcrawlerbot|CrawlaDeBot) | ||
|
@@ -477,7 +497,7 @@ final class Bots | |
|Sublinq | ||
|Gregarius/[\d.]+ | ||
|COMODO\sDCV | ||
|Sectigo\sDCV | ||
|Sectigo\sDCV|acme\.sectigo\.com | ||
|KlarnaBot-(?:DownloadProductImage|EnrichProducts|PriceWatcher)/[\d.]+ | ||
|Taboolabot/[\d.]+ | ||
|Asana/[\d.]+ | ||
|
@@ -502,14 +522,14 @@ final class Bots | |
|AdsTxtCrawler/[\d.]+ | ||
|Morningscore | ||
|Uptime-Kuma/[\d.]+ | ||
|OAI-SearchBot | ||
|GPTBot/[\d.]+ | ||
|ChatGPT-User | ||
|BrightEdge\sCrawler/[\d.]+ | ||
|sfFeedReader/[\d.]+ | ||
|cyberscan\.io | ||
|deepcrawl\.com | ||
|researchscan\.comsys\.rwth-aachen\.de | ||
|newspaper/[\d.]+ | ||
|GPTBot/[\d.]+ | ||
|Ant(?:\.com\sbeta|Bot)(?:/([\d+.]+))? | ||
|WebwikiBot/[\d.]+ | ||
|phpMyAdmin | ||
|
@@ -518,7 +538,7 @@ final class Bots | |
|ArchiveTeam\sArchiveBot | ||
|MADBbot/[\d.]+ | ||
|MeltwaterNews | ||
|(?:Owler@ows\.eu|OWLer)/[\d.]+ | ||
|owler | ||
|bbc\.co\.uk/display/men/Page\+Monitor | ||
|BBC-Forge-URL-Monitor-Twisted | ||
|ClaudeBot | ||
|
@@ -528,6 +548,7 @@ final class Bots | |
|DaspeedBot/([\d+.]+) | ||
|StractBot(?:/([\d+.]+))? | ||
|GeedoBot(?:/([\d+.]+))? | ||
|GeedoProductSearch | ||
|BackupLand(?:/([\d+.]+))? | ||
|Konturbot(?:/([\d+.]+))? | ||
|keys-so-bot | ||
|
@@ -619,7 +640,57 @@ final class Bots | |
|abuse\.xmco\.fr | ||
|CheckHost | ||
|LAC_IAHarvester/[\d.]+ | ||
|nuhk|grub-client|Download\sDemon|SearchExpress|Microsoft\sURL\sControl|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?:\sBuild|Plus))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google\sSketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib\sAutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re\sStudio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle\sClient|Hello,?\sworld|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M\sCODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux\sGnu\s\(cow\)|Test\sCertificate\sInfo|iplabel|Magellan|TheSafex?Internetx?Search|Searcherweb|kirkland-signature|LinkChain|survey-security-dot-txt|^xenu|^ZmEu|^(?:chrome|firefox|Zeus)$ | ||
|[a-z0-9_-]*(?:(?<!cu|power[\s_]|m[\s_])bot(?![\s_]TAB|[\s_]?5[0-9]|[\s_]Senior|[\s_]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|fetcher|indexer|inspector|monitor|project(?!or)|(?<!Google\sWap\s)proxy|research|resolver|robots|scanner|scraper|script|searcher|(?<!-)security|spider|study|transcoder|uptime|user[\s_]?agent|validator)(?:[^a-z]|$) | ||
|InsytfulBot/[\d.]+ | ||
|statista\.com | ||
|SubstackContentFetch/[\d.]+ | ||
|^ds9 | ||
|LiveJournal\.com | ||
|bitdiscovery | ||
|Castopod/[\d.]+ | ||
|Elastic/Synthetics | ||
|WDG_Validator/[\d.]+ | ||
|[email protected] | ||
|CrawlyProjectCrawler/[\d.]+ | ||
|BDFetch | ||
|PunkMap | ||
|GenomeCrawlerd/[\d.]+ | ||
|Gaisbot/[\d.]+ | ||
|FAST-WebCrawler/[\d.]+ | ||
|ducks\.party | ||
|DepSpid/[\d.]+ | ||
|Website-info\.net | ||
|RedekenBot | ||
|semaltbot | ||
|MakeMerryBot | ||
|Timpibot | ||
|Validbot | ||
|NPBot | ||
|domaincodex\.com | ||
|Swisscows\sFavicons | ||
|leak\.info | ||
|workona | ||
|Bloglines | ||
|heritrix | ||
|search\.marginalia\.nu | ||
|vu-server-health-scanner/[\d.]+ | ||
|Functionize | ||
|Prerender | ||
|bl\.uk_ldfc_bot | ||
|Miniature\.io | ||
|Convertify | ||
|ZoteroTranslationServer | ||
|MuckRack | ||
|Golfe | ||
|SpiderLing | ||
|Bravebot | ||
|1001FirmsBot | ||
|SteamChatURLLookup | ||
|ohdear\.app | ||
|Inspici | ||
|peer39_crawler | ||
|Pandalytics | ||
|CloudServerMarketSpider | ||
|nuhk|grub-client|Download\sDemon|SearchExpress|Microsoft\sURL\sControl|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?:\sBuild|Plus|\sCM62|\sHD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google\sSketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib\sAutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re\sStudio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle\sClient|Hello,?\sworld|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M\sCODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux\sGnu\s\(cow\)|Test\sCertificate\sInfo|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root\sSlut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|Wordup|Keydrop|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report\sRunner|url|Zeus|ZmEu)$ | ||
|[a-z0-9_-]*(?:(?<!cu|power[\s_]|m[\s_])bot(?![\s_]TAB|[\s_]?5[0-9]|[\s_]Senior|[\s_]Junior)|analyzer|appengine|archiver?|checker|collector|crawl|crawler|(?<!node-|uclient-|Mikrotik/\d\.[x\d]\s|electron-)fetch(?:er)?|indexer|inspector|monitor|(?<!Microsoft\s|banshee-)project(?!or)|(?<!Google\sWap\s|Blue\s|SpeedMode;\s)proxy|(?<!P)research|resolver|robots|(?<!Cam)scanner|scraper|script|searcher|(?<!-)security|spider(?!\s8)|study|transcoder|uptime|user[\s_]?agent|validator)(?:[^a-z]|$) | ||
#x"; | ||
} |