From 41977dd901d88fcb37ef69d2463f83a9b27a6d45 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 11:16:56 -0400 Subject: [PATCH 01/29] Reduce non-CF false positives The following targets were fixed: Archive[.]org CGTrader CNET Contently IFTTT Linktree xHamster The following targets were removed: HexRPG (auth wall) ModelHub (defunct) Oracle Communities (auth wall) ModelHub was not added to ./removed_sites.md as the platform itself is shutting down (and will therefore never return to Sherlock). The other removed targets were documented normally. BitcoinForum is currently down and suspected to be defunct. Since this is uncertain, however, a test condition was added to suppress false positives while allowing for normal operation upon the forum's return. --- removed_sites.md | 37 ++++++++++++++++++++++ sherlock/resources/data.json | 59 +++++++++++------------------------- sites.md | 3 -- 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/removed_sites.md b/removed_sites.md index d870500..e9146c1 100644 --- a/removed_sites.md +++ b/removed_sites.md @@ -1882,4 +1882,41 @@ As of 2023.12.21, Ebio returns false positives. "urlMain": "https:/ebio.gg", "username_claimed": "dev" }, +``` + +## HexRPG +__2024-04-07 :__ HexRPG behind authentication wall. Unable to check usernames without logging in. +```json + "HexRPG": { + "errorMsg": "Error : User ", + "errorType": "message", + "regexCheck": "^[a-zA-Z0-9_ ]{3,20}$", + "url": "https://www.hexrpg.com/userinfo/{}", + "urlMain": "https://www.hexrpg.com/", + "username_claimed": "blue" + } +``` + +## Oracle Communities +__2024-04-07 :__ Oracle Communities behind authentication wall. Unable to check usernames without logging in. +```json + "Oracle Communities": { + "errorType": "status_code", + "url": "https://community.oracle.com/people/{}", + "urlMain": "https://community.oracle.com", + "username_claimed": "dev" + } +``` + +## Metacritic +__2024-04-07 :__ Non-existent users seemingly displayed as real users with no activity. Needs adjustment. +```json + "metacritic": { + "errorMsg": "User not found", + "errorType": "message", + "regexCheck": "^(?![-_].)[A-Za-z0-9-_]{3,15}$", + "url": "https://www.metacritic.com/user/{}", + "urlMain": "https://www.metacritic.com/", + "username_claimed": "blue" + } ``` \ No newline at end of file diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index b7500f4..d0a7fa3 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -142,9 +142,10 @@ "username_claimed": "test" }, "Archive.org": { - "errorMsg": "cannot find account", + "errorMsg": "could not fetch an account with user item identifier", "errorType": "message", "url": "https://archive.org/details/@{}", + "urlProbe": "https://archive.org/details/@{}?noscript=true", "urlMain": "https://archive.org", "username_claimed": "blue" }, @@ -260,7 +261,11 @@ "username_claimed": "white" }, "BitCoinForum": { - "errorMsg": "The user whose profile you are trying to view does not exist.", + "::::README::::": "The fate of BCF is uncertain. Site offline 2024-03-24. If the site remains offline, this can be removed.", + "errorMsg": [ + "The user whose profile you are trying to view does not exist.", + "Additionally, a 404 Not Found\nerror was encountered while trying to use an ErrorDocument to handle the request" + ], "errorType": "message", "url": "https://bitcoinforum.com/profile/{}", "urlMain": "https://bitcoinforum.com", @@ -321,15 +326,13 @@ "username_claimed": "blue" }, "CGTrader": { - "errorMsg": "3D models for CG digital design and artwork", - "errorType": "message", + "errorType": "status_code", "url": "https://www.cgtrader.com/{}", "urlMain": "https://www.cgtrader.com", "username_claimed": "blue" }, "CNET": { - "errorMsg": "null null - CNET", - "errorType": "message", + "errorType": "status_code", "url": "https://www.cnet.com/profiles/{}/", "urlMain": "https://www.cnet.com/", "username_claimed": "elliottcnet" @@ -502,8 +505,8 @@ "username_claimed": "blue" }, "Contently": { - "errorMsg": "Leading Content Marketing Platform | Contently", - "errorType": "message", + "errorType": "response_url", + "errorUrl": "https://contently.com", "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$", "url": "https://{}.contently.com/", "urlMain": "https://contently.com/", @@ -926,14 +929,6 @@ "urlMain": "https://www.gutefrage.net/", "username_claimed": "gutefrage" }, - "HEXRPG": { - "errorMsg": "Error : User ", - "errorType": "message", - "regexCheck": "^[a-zA-Z0-9_ ]{3,20}$", - "url": "https://www.hexrpg.com/userinfo/{}", - "urlMain": "https://www.hexrpg.com/", - "username_claimed": "blue" - }, "HackTheBox": { "errorType": "status_code", "url": "https://forum.hackthebox.eu/profile/{}", @@ -1033,8 +1028,7 @@ "username_claimed": "Micheal" }, "IFTTT": { - "errorMsg": "The requested page or file does not exist", - "errorType": "message", + "errorType": "status_code", "regexCheck": "^[A-Za-z0-9]{3,35}$", "url": "https://www.ifttt.com/p/{}", "urlMain": "https://www.ifttt.com/", @@ -1211,7 +1205,8 @@ "username_claimed": "blue" }, "Linktree": { - "errorType": "status_code", + "errorMsg": "\"statusCode\":404", + "errorType": "message", "url": "https://linktr.ee/{}", "urlMain": "https://linktr.ee/", "username_claimed": "anne" @@ -1291,13 +1286,6 @@ "urlProbe": "https://api.mixcloud.com/{}/", "username_claimed": "jenny" }, - "Modelhub": { - "errorType": "status_code", - "isNSFW": true, - "url": "https://www.modelhub.com/{}/videos", - "urlMain": "https://www.modelhub.com/", - "username_claimed": "secretcrush" - }, "Monkeytype": { "errorType": "status_code", "url": "https://monkeytype.com/profile/{}", @@ -1450,12 +1438,6 @@ "urlMain": "https://opensource.com/", "username_claimed": "red" }, - "Oracle Community": { - "errorType": "status_code", - "url": "https://community.oracle.com/people/{}", - "urlMain": "https://community.oracle.com", - "username_claimed": "dev" - }, "OurDJTalk": { "errorMsg": "The specified member cannot be found", "errorType": "message", @@ -1550,8 +1532,8 @@ "username_claimed": "swiftstickler" }, "Polymart": { - "errorMsg": "Looks like we couldn't find this user. Sorry!", - "errorType": "message", + "errorType": "response_url", + "errorUrl": "https://polymart.org/user/-1", "url": "https://polymart.org/user/{}", "urlMain": "https://polymart.org/", "username_claimed": "craciu25yt" @@ -2497,14 +2479,6 @@ "urlMain": "https://www.mercadolivre.com.br", "username_claimed": "blue" }, - "metacritic": { - "errorMsg": "User not found", - "errorType": "message", - "regexCheck": "^(?![-_].)[A-Za-z0-9-_]{3,15}$", - "url": "https://www.metacritic.com/user/{}", - "urlMain": "https://www.metacritic.com/", - "username_claimed": "blue" - }, "minds": { "errorMsg": "\"valid\":true", "errorType": "message", @@ -2657,6 +2631,7 @@ "isNSFW": true, "url": "https://xhamster.com/users/{}", "urlMain": "https://xhamster.com", + "urlProbe": "https://xhamster.com/users/{}?old_browser=true", "username_claimed": "blue" }, "znanylekarz.pl": { diff --git a/sites.md b/sites.md index 1cb704c..1a3ca2d 100644 --- a/sites.md +++ b/sites.md @@ -135,7 +135,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=http://en.gravatar.com/) [Gravatar](http://en.gravatar.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.gumroad.com/) [Gumroad](https://www.gumroad.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.gutefrage.net/) [Gutefrage](https://www.gutefrage.net/) -1. ![](https://www.google.com/s2/favicons?domain=https://www.hexrpg.com/) [HEXRPG](https://www.hexrpg.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://forum.hackthebox.eu/) [HackTheBox](https://forum.hackthebox.eu/) 1. ![](https://www.google.com/s2/favicons?domain=https://hackaday.io/) [Hackaday](https://hackaday.io/) 1. ![](https://www.google.com/s2/favicons?domain=https://hackerearth.com/) [HackerEarth](https://hackerearth.com/) @@ -188,7 +187,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://www.memrise.com/) [Memrise](https://www.memrise.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://minecraft.net/) [Minecraft](https://minecraft.net/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.mixcloud.com/) [MixCloud](https://www.mixcloud.com/) -1. ![](https://www.google.com/s2/favicons?domain=https://www.modelhub.com/) [Modelhub](https://www.modelhub.com/) **(NSFW)** 1. ![](https://www.google.com/s2/favicons?domain=https://monkeytype.com/) [Monkeytype](https://monkeytype.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://motherless.com/) [Motherless](https://motherless.com/) **(NSFW)** 1. ![](https://www.google.com/s2/favicons?domain=https://www.motorradfrage.net/) [Motorradfrage](https://www.motorradfrage.net/) @@ -212,7 +210,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://ogu.gg/) [OGUsers](https://ogu.gg/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.openstreetmap.org/) [OpenStreetMap](https://www.openstreetmap.org/) 1. ![](https://www.google.com/s2/favicons?domain=https://opensource.com/) [Opensource](https://opensource.com/) -1. ![](https://www.google.com/s2/favicons?domain=https://community.oracle.com) [Oracle Community](https://community.oracle.com) 1. ![](https://www.google.com/s2/favicons?domain=https://ourdjtalk.com/) [OurDJTalk](https://ourdjtalk.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://pcgamer.com) [PCGamer](https://pcgamer.com) 1. ![](https://www.google.com/s2/favicons?domain=https://psnprofiles.com/) [PSNProfiles.com](https://psnprofiles.com/) From 0b610b49ae03b4661898edf5fd55a45fdf633eb1 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 15:13:28 -0400 Subject: [PATCH 02/29] Update removed sites list --- removed_sites.json | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/removed_sites.json b/removed_sites.json index bc0cba9..0f559ea 100644 --- a/removed_sites.json +++ b/removed_sites.json @@ -869,5 +869,27 @@ "url": "https://ebio.gg/{}", "urlMain": "https:/ebio.gg", "username_claimed": "dev" + }, + "metacritic": { + "errorMsg": "User not found", + "errorType": "message", + "regexCheck": "^(?![-_].)[A-Za-z0-9-_]{3,15}$", + "url": "https://www.metacritic.com/user/{}", + "urlMain": "https://www.metacritic.com/", + "username_claimed": "blue" + }, + "Oracle Communities": { + "errorType": "status_code", + "url": "https://community.oracle.com/people/{}", + "urlMain": "https://community.oracle.com", + "username_claimed": "dev" + }, + "HexRPG": { + "errorMsg": "Error : User ", + "errorType": "message", + "regexCheck": "^[a-zA-Z0-9_ ]{3,20}$", + "url": "https://www.hexrpg.com/userinfo/{}", + "urlMain": "https://www.hexrpg.com/", + "username_claimed": "blue" } } From d660c1a2dd049bb5a26e21135c411be6646f12cd Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 18:07:14 -0400 Subject: [PATCH 03/29] Filter WAF Hits --- sherlock/notify.py | 11 ++++++++++- sherlock/result.py | 1 + sherlock/sherlock.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/sherlock/notify.py b/sherlock/notify.py index 87bfa0b..65502ab 100644 --- a/sherlock/notify.py +++ b/sherlock/notify.py @@ -224,7 +224,7 @@ class QueryNotifyPrint(QueryNotify): elif result.status == QueryStatus.UNKNOWN: if self.print_all: print(Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "-" + + Fore.RED + "?" + Fore.WHITE + "]" + Fore.GREEN + f" {self.result.site_name}:" + Fore.RED + f" {self.result.context}" + @@ -238,6 +238,15 @@ class QueryNotifyPrint(QueryNotify): Fore.WHITE + "]" + Fore.GREEN + f" {self.result.site_name}:" + Fore.YELLOW + f" {msg}") + + elif result.status == QueryStatus.WAF: + if self.print_all: + print(Style.BRIGHT + Fore.WHITE + "[" + + Fore.RED + "?" + + Fore.WHITE + "]" + + Fore.GREEN + f" {self.result.site_name}:" + + Fore.RED + f" Blocked by WAF" + + Fore.YELLOW + " (proxy recommended)") else: # It should be impossible to ever get here... diff --git a/sherlock/result.py b/sherlock/result.py index 6e6ddc1..c4d68b1 100644 --- a/sherlock/result.py +++ b/sherlock/result.py @@ -14,6 +14,7 @@ class QueryStatus(Enum): AVAILABLE = "Available" # Username Not Detected UNKNOWN = "Unknown" # Error Occurred While Trying To Detect Username ILLEGAL = "Illegal" # Username Not Allowable For This Site + WAF = "WAF" # Request blocked by WAF (i.e. Cloudflare) def __str__(self): """Convert Object To String. diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index fb9f524..93fafff 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -378,9 +378,19 @@ def sherlock( query_status = QueryStatus.UNKNOWN error_context = None + # As WAFs advance and evolve, they will occasionally block Sherlock and lead to false positives + # and negatives. Fingerprints should be added here to filter results that fail to bypass WAFs. + # Fingerprints should be highly targetted. Comment at the end of each fingerprint to indicate target and date. + WAFHitMsgs = [ + '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark .big-button,body.dark .pow-button{background-color:#4693ff;color:#1d1d1d}body.dark #challenge-success-text{background-image:url(data:image/svg+xml;base64,' # 2024-04-08 Cloudflare + ] + if error_text is not None: error_context = error_text + elif any(hitMsg in r.text for hitMsg in WAFHitMsgs): + query_status = QueryStatus.WAF + elif error_type == "message": # error_flag True denotes no error found in the HTML # error_flag False denotes error found in the HTML From 8a05ce081702f66521ed7d404f7cebdc5a67c018 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 18:23:57 -0400 Subject: [PATCH 04/29] Update CNET (again) --- sherlock/resources/data.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index d0a7fa3..5829dab 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -332,10 +332,11 @@ "username_claimed": "blue" }, "CNET": { - "errorType": "status_code", + "errorMsg": "null null - CNET", + "errorType": "message", "url": "https://www.cnet.com/profiles/{}/", "urlMain": "https://www.cnet.com/", - "username_claimed": "elliottcnet" + "username_claimed": "melliott" }, "CTAN": { "errorType": "status_code", From 2ba87e23a76b80b2950062266fa162e68befaa0b Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 20:04:12 -0400 Subject: [PATCH 05/29] Fix Genius[.]com Artists --- sherlock/resources/data.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 5829dab..05f354c 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -821,6 +821,7 @@ }, "Genius (Artists)": { "errorType": "status_code", + "regexCheck": "^[a-zA-Z0-9-_]{5,50}$", "url": "https://genius.com/artists/{}", "urlMain": "https://genius.com/", "username_claimed": "genius" From ca1d6f1418333059553118751bb2098612aa511c Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 20:35:00 -0400 Subject: [PATCH 06/29] Fix Telegram F+/F- --- sherlock/resources/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 05f354c..ae0c291 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1896,7 +1896,7 @@ "username_claimed": "traktrain" }, "Telegram": { - "errorMsg": "", + "errorMsg": "Telegram Messenger", "errorType": "message", "regexCheck": "^[a-zA-Z0-9_]{5,32}[^_]$", "url": "https://t.me/{}", From eb89787dcd0b6f51e7ac832de04c9672e6ed494c Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 22:44:24 -0400 Subject: [PATCH 07/29] Fix CNET F+ for bad unames --- sherlock/resources/data.json | 1 + 1 file changed, 1 insertion(+) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index ae0c291..ed22960 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -334,6 +334,7 @@ "CNET": { "errorMsg": "null null - CNET", "errorType": "message", + "regexCheck": "^[a-zA-Z].*$", "url": "https://www.cnet.com/profiles/{}/", "urlMain": "https://www.cnet.com/", "username_claimed": "melliott" From e6d0f1ee7b788e40a118fb6f60ab313685410668 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Mon, 8 Apr 2024 23:19:50 -0400 Subject: [PATCH 08/29] Fix Slides F+, Add multi err code support Error codes module expanded to support arrays of error codes rather than only one. Using this new functionality, Slides was set to error codes 404 (as standard) AND 204 (non standard), to accomodate for that website's odd edge case. --- sherlock/resources/data.json | 1 + sherlock/sherlock.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index ed22960..2bc8ff0 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1757,6 +1757,7 @@ "username_claimed": "blue" }, "Slides": { + "errorCode": [204, 404], "errorType": "status_code", "url": "https://slides.com/{}", "urlMain": "https://slides.com/", diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index fb9f524..96e41e6 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -351,7 +351,6 @@ def sherlock( # Get the expected error type error_type = net_info["errorType"] - error_code = net_info.get("errorCode") # Retrieve future and ensure it has finished future = net_info["request_future"] @@ -407,14 +406,19 @@ def sherlock( else: query_status = QueryStatus.AVAILABLE elif error_type == "status_code": - # Checks if the Status Code is equal to the optional "errorCode" given in 'data.json' - if error_code == r.status_code: - query_status = QueryStatus.AVAILABLE + query_status = QueryStatus.AVAILABLE + error_codes = net_info.get("errorCode") + + if error_codes: # (if set in data.json) + if isinstance(error_codes, int): + if error_codes != r.status_code: + query_status = QueryStatus.CLAIMED + else: + if r.status_code not in error_codes: + query_status = QueryStatus.CLAIMED # Checks if the status code of the response is 2XX elif not r.status_code >= 300 or r.status_code < 200: query_status = QueryStatus.CLAIMED - else: - query_status = QueryStatus.AVAILABLE elif error_type == "response_url": # For this detection method, we have turned off the redirect. # So, there is no need to check the response URL: it will always From b6564a8527d71d264ccda839d07ba07ae54ee1ab Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 00:02:31 -0400 Subject: [PATCH 09/29] Even more specific CNET --- sherlock/resources/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 2bc8ff0..8b7d050 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -334,7 +334,7 @@ "CNET": { "errorMsg": "null null - CNET", "errorType": "message", - "regexCheck": "^[a-zA-Z].*$", + "regexCheck": "^[a-z].*$", "url": "https://www.cnet.com/profiles/{}/", "urlMain": "https://www.cnet.com/", "username_claimed": "melliott" From 1b14b8804315e2460b4e8e08d8df890de1531129 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 15:59:44 -0400 Subject: [PATCH 10/29] Fix Yandex Captcha F+ --- sherlock/resources/data.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 8b7d050..4adf1b8 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -2194,7 +2194,13 @@ "username_claimed": "blue" }, "YandexMusic": { - "errorType": "status_code", + "::::README::::": "The first and third errorMsg relate to geo-restrictions and bot detection/captchas.", + "errorMsg": [ + "Ошибка 404", + " Date: Tue, 9 Apr 2024 17:15:21 -0400 Subject: [PATCH 11/29] Add PerimeterX fingerprint --- sherlock/notify.py | 4 ++-- sherlock/sherlock.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sherlock/notify.py b/sherlock/notify.py index 65502ab..3e550fd 100644 --- a/sherlock/notify.py +++ b/sherlock/notify.py @@ -245,8 +245,8 @@ class QueryNotifyPrint(QueryNotify): Fore.RED + "?" + Fore.WHITE + "]" + Fore.GREEN + f" {self.result.site_name}:" + - Fore.RED + f" Blocked by WAF" + - Fore.YELLOW + " (proxy recommended)") + Fore.RED + " Blocked by bot detection" + + Fore.YELLOW + " (proxy may help)") else: # It should be impossible to ever get here... diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 93fafff..fa927cc 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -382,7 +382,8 @@ def sherlock( # and negatives. Fingerprints should be added here to filter results that fail to bypass WAFs. # Fingerprints should be highly targetted. Comment at the end of each fingerprint to indicate target and date. WAFHitMsgs = [ - '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark .big-button,body.dark .pow-button{background-color:#4693ff;color:#1d1d1d}body.dark #challenge-success-text{background-image:url(data:image/svg+xml;base64,' # 2024-04-08 Cloudflare + '.loading-spinner{visibility:hidden}body.no-js .challenge-running{display:none}body.dark{background-color:#222;color:#d9d9d9}body.dark a{color:#fff}body.dark a:hover{color:#ee730a;text-decoration:underline}body.dark .lds-ring div{border-color:#999 transparent transparent}body.dark .font-red{color:#b20f03}body.dark .big-button,body.dark .pow-button{background-color:#4693ff;color:#1d1d1d}body.dark #challenge-success-text{background-image:url(data:image/svg+xml;base64,', # 2024-04-08 Cloudflare + '{return l.onPageView}}),Object.defineProperty(r,"perimeterxIdentifiers",{enumerable:' # 2024-04-09 PerimeterX / Human Security ] if error_text is not None: From 5d5d8075b9e613b27ba6f7aa520d00cced9068a3 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 17:24:44 -0400 Subject: [PATCH 12/29] Revert from ? to - --- sherlock/notify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sherlock/notify.py b/sherlock/notify.py index 3e550fd..4af1ff1 100644 --- a/sherlock/notify.py +++ b/sherlock/notify.py @@ -224,7 +224,7 @@ class QueryNotifyPrint(QueryNotify): elif result.status == QueryStatus.UNKNOWN: if self.print_all: print(Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "?" + + Fore.RED + "-" + Fore.WHITE + "]" + Fore.GREEN + f" {self.result.site_name}:" + Fore.RED + f" {self.result.context}" + @@ -242,7 +242,7 @@ class QueryNotifyPrint(QueryNotify): elif result.status == QueryStatus.WAF: if self.print_all: print(Style.BRIGHT + Fore.WHITE + "[" + - Fore.RED + "?" + + Fore.RED + "-" + Fore.WHITE + "]" + Fore.GREEN + f" {self.result.site_name}:" + Fore.RED + " Blocked by bot detection" + From cc95d4eea9f967584a01faddc2fb3d81cf1228b6 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 18:13:59 -0400 Subject: [PATCH 13/29] Fix HackerNews rate limit F+ --- sherlock/resources/data.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 4adf1b8..e03158f 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -952,7 +952,11 @@ "username_claimed": "naveennamani877" }, "HackerNews": { - "errorMsg": "No such user.", + "::::README::::": "First errMsg invalid, second errMsg rate limited. Not ideal. Adjust for better rate limit filtering.", + "errorMsg": [ + "No such user.", + "Sorry." + ], "errorType": "message", "url": "https://news.ycombinator.com/user?id={}", "urlMain": "https://news.ycombinator.com/", From d5c235df7167daab0ad887bc29cbb8ea60965409 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 18:19:27 -0400 Subject: [PATCH 14/29] Fix Minecraft rate limit F+ --- sherlock/resources/data.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index e03158f..87ef6b2 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1280,7 +1280,7 @@ "username_claimed": "blue" }, "Minecraft": { - "errorCode": 204, + "errorCode": [204, 404, 405], "errorType": "status_code", "url": "https://api.mojang.com/users/profiles/minecraft/{}", "urlMain": "https://minecraft.net/", From c4777afd7f3a5e47a250e8a9d1d4d65b0bbf71e6 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Tue, 9 Apr 2024 19:01:03 -0400 Subject: [PATCH 15/29] Fix . F+ epidemic --- sherlock/resources/data.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 87ef6b2..b04d76f 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -136,6 +136,7 @@ "username_claimed": "jason" }, "Archive of Our Own": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://archiveofourown.org/users/{}", "urlMain": "https://archiveofourown.org/", @@ -326,6 +327,7 @@ "username_claimed": "blue" }, "CGTrader": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://www.cgtrader.com/{}", "urlMain": "https://www.cgtrader.com", @@ -634,6 +636,7 @@ "username_claimed": "blue" }, "Eintracht Frankfurt Forum": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://community.eintracht.de/fans/{}", "urlMain": "https://community.eintracht.de/", @@ -920,6 +923,7 @@ "username_claimed": "blue" }, "Gumroad": { + "regexCheck": "^[^.]*?$", "errorMsg": "Page not found (404) - Gumroad", "errorType": "message", "url": "https://www.gumroad.com/{}", @@ -970,6 +974,7 @@ "username_claimed": "stok" }, "HackerRank": { + "regexCheck": "^[^.]*?$", "errorMsg": "Something went wrong", "errorType": "message", "url": "https://hackerrank.com/{}", @@ -1434,6 +1439,7 @@ "username_claimed": "ogusers" }, "OpenStreetMap": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://www.openstreetmap.org/user/{}", "urlMain": "https://www.openstreetmap.org/", @@ -1500,6 +1506,7 @@ "username_claimed": "blue" }, "Pinkbike": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://www.pinkbike.com/u/{}/", "urlMain": "https://www.pinkbike.com/", @@ -1821,6 +1828,7 @@ "username_claimed": "splice" }, "Splits.io": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://splits.io/users/{}", "urlMain": "https://splits.io", @@ -1868,6 +1876,7 @@ "username_claimed": "blue" }, "Strava": { + "regexCheck": "^[^.]*?$", "errorMsg": "Strava | Running, Cycling & Hiking App - Train, Track & Share", "errorType": "message", "url": "https://www.strava.com/athletes/{}", @@ -2315,6 +2324,7 @@ "username_claimed": "blue" }, "eintracht": { + "regexCheck": "^[^.]*?$", "errorType": "status_code", "url": "https://community.eintracht.de/fans/{}", "urlMain": "https://eintracht.de", From a56b301e3f7d20d5f775029f04edbaab7fd3074f Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 14:04:33 -0400 Subject: [PATCH 16/29] Fix GeeksForGeeks F+ --- sherlock/resources/data.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index b04d76f..350ae44 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -817,8 +817,7 @@ "username_claimed": "blue" }, "GeeksforGeeks": { - "errorMsg": "Login GeeksforGeeks", - "errorType": "message", + "errorType": "status_code", "url": "https://auth.geeksforgeeks.org/user/{}", "urlMain": "https://www.geeksforgeeks.org/", "username_claimed": "adam" From 936c507bfb8810441da4083834e8a888cb85f364 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 14:45:02 -0400 Subject: [PATCH 17/29] Fix jeuxvideo 403 F+ --- sherlock/resources/data.json | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 350ae44..6007822 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -2428,10 +2428,11 @@ "username_claimed": "blue" }, "jeuxvideo": { - "errorMsg": "Vous \u00eates", - "errorType": "message", - "url": "http://www.jeuxvideo.com/profil/{}?mode=infos", - "urlMain": "http://www.jeuxvideo.com", + "errorType": "status_code", + "request_method": "GET", + "url": "https://www.jeuxvideo.com/profil/{}", + "urlMain": "https://www.jeuxvideo.com", + "urlProbe": "https://www.jeuxvideo.com/profil/{}?mode=infos", "username_claimed": "adam" }, "kofi": { From 48212f09bec7e5a7bd285c74101a2a771c4fcc98 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 17:50:42 -0400 Subject: [PATCH 18/29] Fix adtl Genius A/U F+ --- sherlock/resources/data.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 6007822..f9a7c50 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -824,13 +824,14 @@ }, "Genius (Artists)": { "errorType": "status_code", - "regexCheck": "^[a-zA-Z0-9-_]{5,50}$", + "regexCheck": "^[a-zA-Z0-9]{5,50}$", "url": "https://genius.com/artists/{}", "urlMain": "https://genius.com/", "username_claimed": "genius" }, "Genius (Users)": { "errorType": "status_code", + "regexCheck": "^[a-zA-Z0-9]*?$", "url": "https://genius.com/{}", "urlMain": "https://genius.com/", "username_claimed": "genius" From 4006556daac122ebc462eba80e3f6ed5293132b2 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 21:50:36 -0400 Subject: [PATCH 19/29] Remove G2G for F+ --- removed_sites.json | 8 ++++++++ removed_sites.md | 13 +++++++++++++ sherlock/resources/data.json | 8 -------- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/removed_sites.json b/removed_sites.json index 0f559ea..05a3108 100644 --- a/removed_sites.json +++ b/removed_sites.json @@ -891,5 +891,13 @@ "url": "https://www.hexrpg.com/userinfo/{}", "urlMain": "https://www.hexrpg.com/", "username_claimed": "blue" + }, + "G2G": { + "errorType": "response_url", + "errorUrl": "https://www.g2g.com/{}", + "regexCheck": "^[A-Za-z][A-Za-z0-9_]{2,11}$", + "url": "https://www.g2g.com/{}", + "urlMain": "https://www.g2g.com/", + "username_claimed": "user" } } diff --git a/removed_sites.md b/removed_sites.md index e9146c1..1731f8e 100644 --- a/removed_sites.md +++ b/removed_sites.md @@ -1919,4 +1919,17 @@ __2024-04-07 :__ Non-existent users seemingly displayed as real users with no ac "urlMain": "https://www.metacritic.com/", "username_claimed": "blue" } +``` + +## G2G +__2024-04-10 :__ Seems to be loading profiles with some wierd javascript setup that sherlock doesn't like, leading to difficult to control false positives +```json + "G2G": { + "errorType": "response_url", + "errorUrl": "https://www.g2g.com/{}", + "regexCheck": "^[A-Za-z][A-Za-z0-9_]{2,11}$", + "url": "https://www.g2g.com/{}", + "urlMain": "https://www.g2g.com/", + "username_claimed": "user" + } ``` \ No newline at end of file diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index f9a7c50..50c950a 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -787,14 +787,6 @@ "urlMain": "https://freesound.org/", "username_claimed": "blue" }, - "G2G": { - "errorType": "response_url", - "errorUrl": "https://www.g2g.com/{}", - "regexCheck": "^[A-Za-z][A-Za-z0-9_]{2,11}$", - "url": "https://www.g2g.com/{}", - "urlMain": "https://www.g2g.com/", - "username_claimed": "user" - }, "GNOME VCS": { "errorType": "response_url", "errorUrl": "https://gitlab.gnome.org/{}", From 482f216ff99ac0ef8e9869d22c11e1a21643b68c Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 22:24:36 -0400 Subject: [PATCH 20/29] Fix Kongregate F+ Attempts were met with a Varnish error page presenting 54113 (possibly Fastly related). Change to User Agent necessary to avoid Varnish/Fastly issues. Change to Accept necessary to avoid infinite 302 redirection. Without BOTH of these changes, attempts will fail. Both changes being made also permit the use of status_code rather than message. --- sherlock/resources/data.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 50c950a..1233795 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1163,8 +1163,11 @@ "username_claimed": "blue" }, "Kongregate": { - "errorMsg": "Sorry, no account with that name was found.", - "errorType": "message", + "errorType": "status_code", + "headers": { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0", + "Accept": "text/html" + }, "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$", "url": "https://www.kongregate.com/accounts/{}", "urlMain": "https://www.kongregate.com/", From da3c90fce85b195b75372e2cc15ff7c9b323e80f Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Wed, 10 Apr 2024 22:39:55 -0400 Subject: [PATCH 21/29] Fix Telegram F+ --- sherlock/resources/data.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 1233795..2ca46a9 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1906,12 +1906,15 @@ "username_claimed": "traktrain" }, "Telegram": { - "errorMsg": "Telegram Messenger", + "errorMsg": [ + "Telegram Messenger", + "If you have Telegram, you can contact Date: Thu, 11 Apr 2024 17:28:39 -0400 Subject: [PATCH 22/29] Fix status_code logic bug --- sherlock/resources/data.json | 4 ++-- sherlock/sherlock.py | 20 +++++++++----------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 2ca46a9..e058d4c 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -1280,7 +1280,7 @@ "username_claimed": "blue" }, "Minecraft": { - "errorCode": [204, 404, 405], + "errorCode": 204, "errorType": "status_code", "url": "https://api.mojang.com/users/profiles/minecraft/{}", "urlMain": "https://minecraft.net/", @@ -1763,7 +1763,7 @@ "username_claimed": "blue" }, "Slides": { - "errorCode": [204, 404], + "errorCode": 204, "errorType": "status_code", "url": "https://slides.com/{}", "urlMain": "https://slides.com/", diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 96e41e6..6045918 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -406,19 +406,17 @@ def sherlock( else: query_status = QueryStatus.AVAILABLE elif error_type == "status_code": - query_status = QueryStatus.AVAILABLE error_codes = net_info.get("errorCode") + query_status = QueryStatus.CLAIMED - if error_codes: # (if set in data.json) - if isinstance(error_codes, int): - if error_codes != r.status_code: - query_status = QueryStatus.CLAIMED - else: - if r.status_code not in error_codes: - query_status = QueryStatus.CLAIMED - # Checks if the status code of the response is 2XX - elif not r.status_code >= 300 or r.status_code < 200: - query_status = QueryStatus.CLAIMED + # Type consistency, allowing for both singlets and lists in manifest + if isinstance(error_codes, int): + error_codes = [error_codes] + + if r.status_code in error_codes: + query_status = QueryStatus.AVAILABLE + elif r.status_code >= 300 or r.status_code < 200: + query_status = QueryStatus.AVAILABLE elif error_type == "response_url": # For this detection method, we have turned off the redirect. # So, there is no need to check the response URL: it will always From e43c6047bed3006154f1a82910975240d589bc58 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Thu, 11 Apr 2024 17:39:23 -0400 Subject: [PATCH 23/29] Fix status_code default logic bug --- sherlock/sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index 6045918..b3d7216 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -413,7 +413,7 @@ def sherlock( if isinstance(error_codes, int): error_codes = [error_codes] - if r.status_code in error_codes: + if error_codes is not None and r.status_code in error_codes: query_status = QueryStatus.AVAILABLE elif r.status_code >= 300 or r.status_code < 200: query_status = QueryStatus.AVAILABLE From d59661903a6553e618e88acb09444b7465251e6a Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Thu, 11 Apr 2024 17:43:32 -0400 Subject: [PATCH 24/29] Update site lists --- sherlock/resources/data.json | 26 +++++++++++++------------- sites.md | 6 ++---- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index e058d4c..4399023 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -136,8 +136,8 @@ "username_claimed": "jason" }, "Archive of Our Own": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://archiveofourown.org/users/{}", "urlMain": "https://archiveofourown.org/", "username_claimed": "test" @@ -146,8 +146,8 @@ "errorMsg": "could not fetch an account with user item identifier", "errorType": "message", "url": "https://archive.org/details/@{}", - "urlProbe": "https://archive.org/details/@{}?noscript=true", "urlMain": "https://archive.org", + "urlProbe": "https://archive.org/details/@{}?noscript=true", "username_claimed": "blue" }, "ArtStation": { @@ -327,8 +327,8 @@ "username_claimed": "blue" }, "CGTrader": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://www.cgtrader.com/{}", "urlMain": "https://www.cgtrader.com", "username_claimed": "blue" @@ -636,8 +636,8 @@ "username_claimed": "blue" }, "Eintracht Frankfurt Forum": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://community.eintracht.de/fans/{}", "urlMain": "https://community.eintracht.de/", "username_claimed": "mmammu" @@ -915,9 +915,9 @@ "username_claimed": "blue" }, "Gumroad": { - "regexCheck": "^[^.]*?$", "errorMsg": "Page not found (404) - Gumroad", "errorType": "message", + "regexCheck": "^[^.]*?$", "url": "https://www.gumroad.com/{}", "urlMain": "https://www.gumroad.com/", "username_claimed": "blue" @@ -966,9 +966,9 @@ "username_claimed": "stok" }, "HackerRank": { - "regexCheck": "^[^.]*?$", "errorMsg": "Something went wrong", "errorType": "message", + "regexCheck": "^[^.]*?$", "url": "https://hackerrank.com/{}", "urlMain": "https://hackerrank.com/", "username_claimed": "satznova" @@ -1165,8 +1165,8 @@ "Kongregate": { "errorType": "status_code", "headers": { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0", - "Accept": "text/html" + "Accept": "text/html", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0" }, "regexCheck": "^[a-zA-Z][a-zA-Z0-9_-]*$", "url": "https://www.kongregate.com/accounts/{}", @@ -1434,8 +1434,8 @@ "username_claimed": "ogusers" }, "OpenStreetMap": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://www.openstreetmap.org/user/{}", "urlMain": "https://www.openstreetmap.org/", "username_claimed": "blue" @@ -1501,8 +1501,8 @@ "username_claimed": "blue" }, "Pinkbike": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://www.pinkbike.com/u/{}/", "urlMain": "https://www.pinkbike.com/", "username_claimed": "blue" @@ -1823,8 +1823,8 @@ "username_claimed": "splice" }, "Splits.io": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://splits.io/users/{}", "urlMain": "https://splits.io", "username_claimed": "cambosteve" @@ -1871,9 +1871,9 @@ "username_claimed": "blue" }, "Strava": { - "regexCheck": "^[^.]*?$", "errorMsg": "Strava | Running, Cycling & Hiking App - Train, Track & Share", "errorType": "message", + "regexCheck": "^[^.]*?$", "url": "https://www.strava.com/athletes/{}", "urlMain": "https://www.strava.com/", "username_claimed": "blue" @@ -2322,8 +2322,8 @@ "username_claimed": "blue" }, "eintracht": { - "regexCheck": "^[^.]*?$", "errorType": "status_code", + "regexCheck": "^[^.]*?$", "url": "https://community.eintracht.de/fans/{}", "urlMain": "https://eintracht.de", "username_claimed": "blue" diff --git a/sites.md b/sites.md index 1a3ca2d..4688235 100644 --- a/sites.md +++ b/sites.md @@ -1,4 +1,4 @@ -## List Of Supported Sites (395 Sites In Total!) +## List Of Supported Sites (390 Sites In Total!) 1. ![](https://www.google.com/s2/favicons?domain=https://2Dimensions.com/) [2Dimensions](https://2Dimensions.com/) 1. ![](https://www.google.com/s2/favicons?domain=http://forum.3dnews.ru/) [3dnews](http://forum.3dnews.ru/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.7cups.com/) [7Cups](https://www.7cups.com/) @@ -113,7 +113,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://freelance.habr.com/) [Freelance.habr](https://freelance.habr.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.freelancer.com/) [Freelancer](https://www.freelancer.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://freesound.org/) [Freesound](https://freesound.org/) -1. ![](https://www.google.com/s2/favicons?domain=https://www.g2g.com/) [G2G](https://www.g2g.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://gitlab.gnome.org/) [GNOME VCS](https://gitlab.gnome.org/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.gaiaonline.com/) [GaiaOnline](https://www.gaiaonline.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.gamespot.com/) [Gamespot](https://www.gamespot.com/) @@ -353,7 +352,7 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://www.interpals.net/) [interpals](https://www.interpals.net/) 1. ![](https://www.google.com/s2/favicons?domain=https://irecommend.ru/) [irecommend](https://irecommend.ru/) 1. ![](https://www.google.com/s2/favicons?domain=https://jbzd.com.pl/) [jbzd.com.pl](https://jbzd.com.pl/) -1. ![](https://www.google.com/s2/favicons?domain=http://www.jeuxvideo.com) [jeuxvideo](http://www.jeuxvideo.com) +1. ![](https://www.google.com/s2/favicons?domain=https://www.jeuxvideo.com) [jeuxvideo](https://www.jeuxvideo.com) 1. ![](https://www.google.com/s2/favicons?domain=https://ko-fi.com) [kofi](https://ko-fi.com) 1. ![](https://www.google.com/s2/favicons?domain=https://www.kwork.ru/) [kwork](https://www.kwork.ru/) 1. ![](https://www.google.com/s2/favicons?domain=https://lab.pentestit.ru/) [labpentestit](https://lab.pentestit.ru/) @@ -365,7 +364,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://mastodon.xyz/) [mastodon.technology](https://mastodon.xyz/) 1. ![](https://www.google.com/s2/favicons?domain=https://mastodon.xyz/) [mastodon.xyz](https://mastodon.xyz/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.mercadolivre.com.br) [mercadolivre](https://www.mercadolivre.com.br) -1. ![](https://www.google.com/s2/favicons?domain=https://www.metacritic.com/) [metacritic](https://www.metacritic.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.minds.com) [minds](https://www.minds.com) 1. ![](https://www.google.com/s2/favicons?domain=https://moikrug.ru/) [moikrug](https://moikrug.ru/) 1. ![](https://www.google.com/s2/favicons?domain=https://mstdn.io/) [mstdn.io](https://mstdn.io/) From 5c23b1555c0ec7b92db4095be3ff6b1c32f98763 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Thu, 11 Apr 2024 21:23:11 -0400 Subject: [PATCH 25/29] Update default User-Agent I've noticed that many bot-detection pages are able to be avoided by using this UA. Unless there's a reason to stay on the old old old one, we may as well update it and reduce our WAF hits. --- sherlock/sherlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sherlock/sherlock.py b/sherlock/sherlock.py index b3d7216..8c0ae80 100644 --- a/sherlock/sherlock.py +++ b/sherlock/sherlock.py @@ -232,7 +232,7 @@ def sherlock( # A user agent is needed because some sites don't return the correct # information since they think that we are bots (Which we actually are...) headers = { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:55.0) Gecko/20100101 Firefox/55.0", + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/116.0", } if "headers" in net_info: From 2a5fbd9e97d440efc11821feac43fb4d1afbd425 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Sun, 14 Apr 2024 15:32:19 -0400 Subject: [PATCH 26/29] Remove BitcoinForum instead of suppress --- removed_sites.json | 7 +++++++ removed_sites.md | 12 ++++++++++++ sherlock/resources/data.json | 11 ----------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/removed_sites.json b/removed_sites.json index 05a3108..c4c7ad8 100644 --- a/removed_sites.json +++ b/removed_sites.json @@ -899,5 +899,12 @@ "url": "https://www.g2g.com/{}", "urlMain": "https://www.g2g.com/", "username_claimed": "user" + }, + "BitCoinForum": { + "errorMsg": "The user whose profile you are trying to view does not exist.", + "errorType": "message", + "url": "https://bitcoinforum.com/profile/{}", + "urlMain": "https://bitcoinforum.com", + "username_claimed": "bitcoinforum.com" } } diff --git a/removed_sites.md b/removed_sites.md index 1731f8e..21d4251 100644 --- a/removed_sites.md +++ b/removed_sites.md @@ -1932,4 +1932,16 @@ __2024-04-10 :__ Seems to be loading profiles with some wierd javascript setup t "urlMain": "https://www.g2g.com/", "username_claimed": "user" } +``` + +## Bitcoin Forum +__2024-04-24 :__ BCF seems to have gone defunct. Uncertain. +```json +"BitCoinForum": { + "errorMsg": "The user whose profile you are trying to view does not exist.", + "errorType": "message", + "url": "https://bitcoinforum.com/profile/{}", + "urlMain": "https://bitcoinforum.com", + "username_claimed": "bitcoinforum.com" + } ``` \ No newline at end of file diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index 4399023..b8adafe 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -261,17 +261,6 @@ "urlMain": "https://bitbucket.org/", "username_claimed": "white" }, - "BitCoinForum": { - "::::README::::": "The fate of BCF is uncertain. Site offline 2024-03-24. If the site remains offline, this can be removed.", - "errorMsg": [ - "The user whose profile you are trying to view does not exist.", - "Additionally, a 404 Not Found\nerror was encountered while trying to use an ErrorDocument to handle the request" - ], - "errorType": "message", - "url": "https://bitcoinforum.com/profile/{}", - "urlMain": "https://bitcoinforum.com", - "username_claimed": "bitcoinforum.com" - }, "Bitwarden Forum": { "errorType": "status_code", "regexCheck": "^(?![.-])[a-zA-Z0-9_.-]{3,20}$", From 51959f95a7b83b8c38f904f13a1a64ec98226c01 Mon Sep 17 00:00:00 2001 From: Paul Pfeister Date: Sun, 14 Apr 2024 15:35:23 -0400 Subject: [PATCH 27/29] Update sites list --- sites.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sites.md b/sites.md index 4688235..fb4b327 100644 --- a/sites.md +++ b/sites.md @@ -1,4 +1,4 @@ -## List Of Supported Sites (390 Sites In Total!) +## List Of Supported Sites (389 Sites In Total!) 1. ![](https://www.google.com/s2/favicons?domain=https://2Dimensions.com/) [2Dimensions](https://2Dimensions.com/) 1. ![](https://www.google.com/s2/favicons?domain=http://forum.3dnews.ru/) [3dnews](http://forum.3dnews.ru/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.7cups.com/) [7Cups](https://www.7cups.com/) @@ -37,7 +37,6 @@ 1. ![](https://www.google.com/s2/favicons?domain=https://www.bikemap.net/) [Bikemap](https://www.bikemap.net/) 1. ![](https://www.google.com/s2/favicons?domain=https://forum.dangerousthings.com/) [BioHacking](https://forum.dangerousthings.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://bitbucket.org/) [BitBucket](https://bitbucket.org/) -1. ![](https://www.google.com/s2/favicons?domain=https://bitcoinforum.com) [BitCoinForum](https://bitcoinforum.com) 1. ![](https://www.google.com/s2/favicons?domain=https://bitwarden.com/) [Bitwarden Forum](https://bitwarden.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://www.blogger.com/) [Blogger](https://www.blogger.com/) 1. ![](https://www.google.com/s2/favicons?domain=https://bodyspace.bodybuilding.com/) [BodyBuilding](https://bodyspace.bodybuilding.com/) From 9137ee4f090b9176342061d1a76943b4a5effb43 Mon Sep 17 00:00:00 2001 From: Ryan <110407320+dalryan@users.noreply.github.com> Date: Sat, 27 Apr 2024 12:41:30 +0200 Subject: [PATCH 28/29] Add ruff to pr tests (#2093) --- .github/workflows/pull_request.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index fb44d4f..a70ad44 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -34,13 +34,12 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest + pip install ruff flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint With flake8 + - name: Lint With Ruff run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - + ruff check . --output-format=github --select=E9,F63,F7,F82 # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Sherlock Site Detect Tests From 71bdf63c19387bd153e34aaeab27b2b59ef5726e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 13:08:04 +0000 Subject: [PATCH 29/29] Updated Site List --- sherlock/resources/data.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sherlock/resources/data.json b/sherlock/resources/data.json index b8adafe..493a613 100644 --- a/sherlock/resources/data.json +++ b/sherlock/resources/data.json @@ -2196,8 +2196,8 @@ "YandexMusic": { "::::README::::": "The first and third errorMsg relate to geo-restrictions and bot detection/captchas.", "errorMsg": [ - "Ошибка 404", - "