From c2e21b2540ffa1014a79b2614746f38c3b19fc5a Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sat, 9 Sep 2023 20:55:14 +0200 Subject: [PATCH 1/3] Some sites fixed & cloudflare detection --- maigret/errors.py | 6 +++ maigret/resources/data.json | 73 +++++++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/maigret/errors.py b/maigret/errors.py index f92b2636..8036fdbb 100644 --- a/maigret/errors.py +++ b/maigret/errors.py @@ -58,6 +58,12 @@ def desc(self): 'Сайт заблокирован хостинг-провайдером': CheckError( 'Site-specific', 'Site is disabled (Beget)' ), + 'Generated by cloudfront (CloudFront)': CheckError( + 'Request blocked', 'Cloudflare' + ), + '/cdn-cgi/challenge-platform/h/b/orchestrate/chl_page': CheckError( + 'Just a moment: bot redirect challenge', 'Cloudflare' + ) } ERRORS_TYPES = { diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 89f78b79..75a8b84d 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -3595,6 +3595,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Crossfire": { + "disabled": true, "tags": [ "gaming", "ru" @@ -4069,6 +4070,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Digitalspy": { + "disabled": true, "tags": [ "forum", "gb", @@ -13878,7 +13880,7 @@ "\u041f\u0440\u043e\u0444\u0438\u043b\u044c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f" ], "alexaRank": 495, - "urlMain": "http://37.1.216.121/", + "urlMain": "https://rutracker.org/", "mirrors": [ "https://rutracker.org/", "http://37.1.216.121/" @@ -14714,6 +14716,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Soloby": { + "disabled": true, "tags": [ "by", "ru" @@ -15228,12 +15231,15 @@ "regexCheck": "^[^\\.]+$", "checkType": "message", "absenceStrs": [ - "Strava | Run and Cycling Tracking on the Social Network for Athletes" + "Strava | " + ], + "presenseStrs": [ + "Profile" ], "alexaRank": 1099, "urlMain": "https://www.strava.com/", "url": "https://www.strava.com/athletes/{username}", - "usernameClaimed": "blue", + "usernameClaimed": "adam", "usernameUnclaimed": "noonewouldeverusethis7" }, "Studfile": { @@ -16612,6 +16618,9 @@ "tags": [ "messaging" ], + "headers": { + "Accept-Language": "en-US,en;q=0.5" + }, "regexCheck": "^[a-zA-Z0-9_]{1,15}$", "checkType": "message", "absenceStrs": [ @@ -16623,14 +16632,13 @@ "
" ], "mirrors": [ - "https://nitter.net/", "https://nitter.42l.fr/", "https://nitter.1d4.us/", "https://nitter.kavin.rocks/" ], "source": "Twitter", "alexaRank": 48, - "urlMain": "https://nitter.kavin.rocks/", + "urlMain": "https://nitter.net/", "url": "{urlMain}{username}", "usernameClaimed": "blue", "usernameUnclaimed": "noonewould123" @@ -17153,7 +17161,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2OTI4NjczMDAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.C_eyva4vBf9wXlW5NpN94tAuXL-fU0z27JrLyk5-B2M" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2OTQyODI1ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.dIKSNMvYu7qcZ0Q7o9j5XaQH8iNWa-7X5wi2qQofoRo" }, "activation": { "url": "https://vimeo.com/_rv/viewer", @@ -17494,6 +17502,7 @@ "usernameUnclaimed": "noonewouldeverusethis7" }, "Waveapps": { + "disabled": true, "tags": [ "ca", "us" @@ -30720,6 +30729,7 @@ "alexaRank": 141 }, "Weebly": { + "regexCheck": "^[a-zA-Z0-9]+$", "absenceStrs": [ "Error - Page Not Found" ], @@ -34421,6 +34431,54 @@ "tags": [ "crypto" ] + }, + "PromptBase": { + "absenceStrs": [ + "NotFound" + ], + "presenseStrs": [ + "1" + ], + "url": "https://promptbase.com/profile/{username}", + "urlMain": "https://promptbase.com", + "usernameClaimed": "admin", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "tags": [ + "ai" + ] + }, + "ngl.link": { + "absenceStrs": [ + "Could not find user" + ], + "presenseStrs": [ + "1" + ], + "url": "https://ngl.link/{username}", + "urlMain": "https://ngl.link", + "usernameClaimed": "youbutdumberr", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "tags": [ + "q&a" + ] + }, + "bitpapa.com": { + "absenceStrs": [ + "/static/page-crash.svg" + ], + "presenseStrs": [ + "lbcUsername" + ], + "url": "https://bitpapa.com/ru/user/{username}", + "urlMain": "https://bitpapa.com", + "usernameClaimed": "Larisa70", + "usernameUnclaimed": "noonewouldeverusethis7", + "checkType": "message", + "tags": [ + "crypto" + ] } }, "engines": { @@ -34699,6 +34757,7 @@ "tor", "i2p", "q&a", - "crypto" + "crypto", + "ai" ] } \ No newline at end of file From 5df4e7ebc00d4d708d3e960e3c72d32f05801991 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 15 Oct 2023 20:54:01 +0200 Subject: [PATCH 2/3] Fixed issue with tests --- test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-requirements.txt b/test-requirements.txt index b79de211..9a326f2a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,5 +4,5 @@ pytest==7.2.0 pytest-asyncio==0.16.0;python_version<"3.7" pytest-asyncio==0.20.1;python_version>="3.7" pytest-cov==4.0.0 -pytest-httpserver==1.0.6 +pytest-httpserver~=1.0.8 pytest-rerunfailures==10.2 From 29d6e820ac704bda5769a13618ff0ebdc679b1a8 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 15 Oct 2023 20:59:23 +0200 Subject: [PATCH 3/3] Updates GitHub test workflow and sites data --- .github/workflows/python-package.yml | 2 +- maigret/resources/data.json | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6deba938..d3a6615c 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 diff --git a/maigret/resources/data.json b/maigret/resources/data.json index 75a8b84d..02907c20 100644 --- a/maigret/resources/data.json +++ b/maigret/resources/data.json @@ -3616,11 +3616,14 @@ "movies", "us" ], + "headers": { + "'User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/117.0" + }, "checkType": "status_code", "alexaRank": 364, "urlMain": "https://www.crunchyroll.com/", "url": "https://www.crunchyroll.com/user/{username}", - "usernameClaimed": "blue", + "usernameClaimed": "adan", "usernameUnclaimed": "noonewouldeverusethis7" }, "CryptomatorForum": { @@ -17161,7 +17164,7 @@ "video" ], "headers": { - "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2OTQyODI1ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.dIKSNMvYu7qcZ0Q7o9j5XaQH8iNWa-7X5wi2qQofoRo" + "Authorization": "jwt eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJleHAiOjE2OTUxNjg3ODAsInVzZXJfaWQiOm51bGwsImFwcF9pZCI6NTg0NzksInNjb3BlcyI6InB1YmxpYyIsInRlYW1fdXNlcl9pZCI6bnVsbH0.WjozUble_6hWPVxipg4ub9pnCILTh5mTDUMV3806oSM" }, "activation": { "url": "https://vimeo.com/_rv/viewer",