Skip to content

Commit

Permalink
Improved search through UnstoppableDomains (#1040)
Browse files Browse the repository at this point in the history
  • Loading branch information
@soxoj
soxoj authored Jul 7, 2023
1 parent 5855cbf commit 656b9c1
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 209 deletions.
211 changes: 21 additions & 190 deletions maigret/resources/data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31315,6 +31315,7 @@
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"alexaRank": 362185,
"source": "Roblox",
"tags": [
"gaming"
]
Expand Down Expand Up @@ -34027,198 +34028,28 @@
"crypto"
]
},
"ud.me/nft": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"UnstoppableDomains": {
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
"reservedForUserId",
"\"registered\"",
"DomainProduct"
],
"url": "https://ud.me/{username}.nft",
"urlMain": "https://ud.me",
"usernameClaimed": "mlfed",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/crypto": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.crypto",
"urlMain": "https://ud.me",
"usernameClaimed": "unifiprotocol",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/x": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.x",
"urlMain": "https://ud.me",
"usernameClaimed": "mlfed",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/wallet": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.wallet",
"urlMain": "https://ud.me",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/polygon": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.polygon",
"urlMain": "https://ud.me",
"usernameClaimed": "alex",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/dao": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.dao",
"urlMain": "https://ud.me",
"usernameClaimed": "unifiprotocol",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/888": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.888",
"urlMain": "https://ud.me",
"usernameClaimed": "mlfed",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/zil": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.zil",
"urlMain": "https://ud.me",
"usernameClaimed": "everlast",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/blockchain": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.blockchain",
"urlMain": "https://ud.me",
"usernameClaimed": "unifiprotocol",
"usernameUnclaimed": "noonewouldeverusethis7",
"checkType": "message",
"tags": [
"crypto"
]
},
"ud.me/bitcoin": {
"absenceStrs": [
"https://unstoppabledomains.com/search?searchTerm"
],
"presenseStrs": [
"profile",
"profileData",
"displayName",
"main",
"profileServiceUrl"
],
"url": "https://ud.me/{username}.bitcoin",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/114.0",
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"Referer": "https://unstoppabledomains.com/",
"Connection": "keep-alive",
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Pragma": "no-cache",
"Cache-Control": "no-cache",
"TE": "trailers"
},
"urlProbe": "https://unstoppabledomains.com/api/domain/search?q={username}",
"url": "https://ud.me/{username}",
"urlMain": "https://ud.me",
"usernameClaimed": "mlfed",
"usernameUnclaimed": "noonewouldeverusethis7",
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ python-bidi==0.4.2
requests==2.31.0
requests-futures==1.0.0
six==1.16.0
socid-extractor>=0.0.21
socid-extractor>=0.0.24
soupsieve==2.3.2.post1
stem==1.8.1
torrequest==0.1.0
Expand Down
27 changes: 9 additions & 18 deletions sites.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

## List of supported sites (search methods): total 3101
## List of supported sites (search methods): total 3092

Rank data fetched from Alexa by domains.

Expand Down Expand Up @@ -3079,16 +3079,7 @@ Rank data fetched from Alexa by domains.
1. ![](https://www.google.com/s2/favicons?domain=) [bugbounty ()]()*: top 100M, hacking*
1. ![](https://www.google.com/s2/favicons?domain=) [universocraft ()]()*: top 100M, gaming*
1. ![](https://www.google.com/s2/favicons?domain=https://fragment.com) [fragment.com (https://fragment.com)](https://fragment.com)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/nft (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/crypto (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/x (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/wallet (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/polygon (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/dao (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/888 (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/zil (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/blockchain (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [ud.me/bitcoin (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://ud.me) [UnstoppableDomains (https://ud.me)](https://ud.me)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/meta (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/music (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://api.edns.domains) [edns.domains/ass (https://api.edns.domains)](https://api.edns.domains)*: top 100M, crypto*
Expand All @@ -3105,20 +3096,20 @@ Rank data fetched from Alexa by domains.
1. ![](https://www.google.com/s2/favicons?domain=https://peername.com/) [peername.com/emc (https://peername.com/)](https://peername.com/)*: top 100M, crypto*
1. ![](https://www.google.com/s2/favicons?domain=https://peername.com/) [peername.com/tor (https://peername.com/)](https://peername.com/)*: top 100M, crypto*

The list was updated at (2023-05-13 10:19:52.408695 UTC)
The list was updated at (2023-07-07 19:15:34.376816 UTC)
## Statistics

Enabled/total sites: 2845/3101 = 91.74%
Enabled/total sites: 2836/3092 = 91.72%

Incomplete message checks: 473/2845 = 16.63% (false positive risks)
Incomplete message checks: 474/2836 = 16.71% (false positive risks)

Status code checks: 726/2845 = 25.52% (false positive risks)
Status code checks: 726/2836 = 25.6% (false positive risks)

False positive risk (total): 42.15%
False positive risk (total): 42.31%

Top 20 profile URLs:
- (796) `{urlMain}/index/8-0-{username} (uCoz)`
- (294) `/{username}`
- (295) `/{username}`
- (221) `{urlMain}{urlSubpath}/members/?username={username} (XenForo)`
- (160) `/user/{username}`
- (133) `{urlMain}{urlSubpath}/member.php?username={username} (vBulletin)`
Expand Down Expand Up @@ -3157,5 +3148,5 @@ Top 20 tags:
- (9) `movies`
- (8) `hobby`
- (8) `hacking`
- (8) `crypto`
- (7) `sport`
- (7) `crypto`

0 comments on commit 656b9c1

Please sign in to comment.