diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml index bbf6b4c..0db8663 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -19,6 +19,15 @@ body: - Other (indicate below) validations: required: true + - type: input + id: package-version + attributes: + label: Package version + description: | + Knowing the version of the package you are using can help us diagnose your issue more quickly. + You can find the version by running `sherlock --version`. + validations: + required: true - type: textarea id: description attributes: diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 1ff20f9..2e5ea94 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -49,7 +49,6 @@ jobs: macos-latest, ] python-version: [ - '3.8', '3.9', '3.10', '3.11', diff --git a/docs/README.md b/docs/README.md index 3090a15..0fa75ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,18 +22,21 @@ ## Installation +> [!WARNING] +> Packages for ParrotOS and Ubuntu 24.04, maintained by a third party, appear to be __broken__. +> Users of these systems should defer to pipx/pip or Docker. -| | Command | Notes | -| - | - | - | -| PyPI | `pipx install sherlock-project` | `pip` may be used in place of `pipx` | -| Docker | `docker pull sherlock/sherlock` | | -| Debian family | `apt install sherlock` | Kali, Parrot, Debian Testing and Sid | -| BlackArch | `pacman -S sherlock` | | -| Homebrew | `brew install sherlock` | | +| Method | Notes | +| - | - | +| `pipx install sherlock-project` | `pip` may be used in place of `pipx` | +| `docker run -it --rm sherlock/sherlock` | +| `dnf install sherlock-project` | | + +Community-maintained packages are available for Debian (>= 13), Ubuntu (>= 22.10), Homebrew, Kali, and BlackArch. These packages are not directly supported or maintained by the Sherlock Project. See all alternative installation methods [here](https://sherlockproject.xyz/installation) -## Usage +## General usage To search for only one user: ```bash diff --git a/pyproject.toml b/pyproject.toml index 4118459..069cb9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ repository = "https://github.com/sherlock-project/sherlock" "Bug Tracker" = "https://github.com/sherlock-project/sherlock/issues" [tool.poetry.dependencies] -python = "^3.8" +python = "^3.9" certifi = ">=2019.6.16" colorama = "^0.4.1" PySocks = "^1.7.0" @@ -48,8 +48,7 @@ requests = "^2.22.0" requests-futures = "^1.0.0" stem = "^1.8.0" torrequest = "^0.1.0" -# pandas can likely be bumped up to ^2.0.0 after fc39 EOL -pandas = ">=1.0.0,<3.0.0" +pandas = "^2.2.1" openpyxl = "^3.0.10" [tool.poetry.extras] diff --git a/sherlock_project/__main__.py b/sherlock_project/__main__.py index 3193831..a252de0 100644 --- a/sherlock_project/__main__.py +++ b/sherlock_project/__main__.py @@ -14,8 +14,8 @@ if __name__ == "__main__": # Check if the user is using the correct version of Python python_version = sys.version.split()[0] - if sys.version_info < (3, 8): - print(f"Sherlock requires Python 3.8+\nYou are using Python {python_version}, which is not supported by Sherlock.") + if sys.version_info < (3, 9): + print(f"Sherlock requires Python 3.9+\nYou are using Python {python_version}, which is not supported by Sherlock.") sys.exit(1) from sherlock_project import sherlock diff --git a/sherlock_project/py.typed b/sherlock_project/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/sherlock_project/resources/data.json b/sherlock_project/resources/data.json index bbfb8b2..72411b0 100644 --- a/sherlock_project/resources/data.json +++ b/sherlock_project/resources/data.json @@ -31,9 +31,13 @@ "username_claimed": "blue" }, "8tracks": { - "errorMsg": "This page has vanished", "errorType": "message", + "errorMsg": "\"available\":true", + "headers": { + "Accept-Language": "en-US,en;q=0.5" + }, "url": "https://8tracks.com/{}", + "urlProbe": "https://8tracks.com/users/check_username?login={}&format=jsonh", "urlMain": "https://8tracks.com/", "username_claimed": "blue" }, @@ -159,7 +163,8 @@ "__comment__": "'The resource could not be found' relates to archive downtime", "errorMsg": [ "could not fetch an account with user item identifier", - "The resource could not be found" + "The resource could not be found", + "Internet Archive services are temporarily offline" ], "errorType": "message", "url": "https://archive.org/details/@{}", @@ -193,6 +198,12 @@ "urlMain": "https://ask.fm/", "username_claimed": "blue" }, + "Atcoder": { + "errorType": "status_code", + "url": "https://atcoder.jp/users/{}", + "urlMain": "https://atcoder.jp/", + "username_claimed": "ksun48" + }, "Audiojungle": { "errorType": "status_code", "regexCheck": "^[a-zA-Z0-9_]+$", @@ -298,11 +309,12 @@ "urlMain": "https://www.blogger.com/", "username_claimed": "blue" }, - "BodyBuilding": { - "errorType": "response_url", - "errorUrl": "https://bodyspace.bodybuilding.com/", - "url": "https://bodyspace.bodybuilding.com/{}", - "urlMain": "https://bodyspace.bodybuilding.com/", + "BoardGameGeek": { + "errorType": "message", + "regexCheck": "^[a-zA-Z0-9_]*$", + "errorMsg": "User not found", + "url": "https://boardgamegeek.com/user/{}", + "urlMain": "https://boardgamegeek.com", "username_claimed": "blue" }, "BongaCams": { @@ -695,6 +707,12 @@ "urlMain": "https://community.eintracht.de/", "username_claimed": "mmammu" }, + "Empretienda AR": { + "errorType": "status_code", + "url": "https://{}.empretienda.com.ar", + "urlMain": "https://empretienda.com", + "username_claimed": "camalote" + }, "Envato Forum": { "errorType": "status_code", "url": "https://forums.envato.com/u/{}", @@ -714,6 +732,12 @@ "urlMain": "https://exposure.co/", "username_claimed": "jonasjacobsson" }, + "exophase": { + "errorType": "status_code", + "url": "https://www.exophase.com/user/{}/", + "urlMain": "https://www.exophase.com/", + "username_claimed": "blue" + }, "EyeEm": { "errorType": "status_code", "url": "https://www.eyeem.com/u/{}", @@ -1688,6 +1712,12 @@ "urlMain": "https://www.rajce.idnes.cz/", "username_claimed": "blue" }, + "Rarible": { + "errorType": "status_code", + "url": "https://rarible.com/marketplace/api/v4/urls/{}", + "urlMain": "https://rarible.com/", + "username_claimed": "blue" + }, "Rate Your Music": { "errorType": "status_code", "url": "https://rateyourmusic.com/~{}", @@ -1918,11 +1948,10 @@ "username_claimed": "adam" }, "Speedrun.com": { - "errorMsg": "Not found", - "errorType": "message", - "url": "https://speedrun.com/user/{}", + "errorType": "status_code", + "url": "https://speedrun.com/users/{}", "urlMain": "https://speedrun.com/", - "username_claimed": "3Tau" + "username_claimed": "example" }, "Spells8": { "errorType": "status_code", @@ -2011,6 +2040,12 @@ "urlProbe": "https://ch.tetr.io/api/users/{}", "username_claimed": "osk" }, + "Tiendanube": { + "url": "https://{}.mitiendanube.com/", + "urlMain": "https://www.tiendanube.com/", + "errorType": "status_code", + "username_claimed": "blue" + }, "TLDR Legal": { "errorType": "status_code", "regexCheck": "^[a-zA-Z0-9]{3,20}$", @@ -2018,6 +2053,14 @@ "urlMain": "https://tldrlegal.com/", "username_claimed": "kevin" }, + "Topcoder": { + "errorType": "status_code", + "url": "https://profiles.topcoder.com/{}/", + "urlMain": "https://topcoder.com/", + "username_claimed": "USER", + "urlProbe": "https://api.topcoder.com/v5/members/{}", + "regexCheck": "[a-zA-Z0-9 ]" + }, "TRAKTRAIN": { "errorType": "status_code", "url": "https://traktrain.com/{}", @@ -2139,7 +2182,7 @@ "regexCheck": "^[a-zA-Z0-9_]{1,15}$", "url": "https://x.com/{}", "urlMain": "https://x.com/", - "urlProbe": "https://nitter.net/{}", + "urlProbe": "https://nitter.privacydev.net/{}", "username_claimed": "blue" }, "Typeracer": { @@ -2181,6 +2224,12 @@ "urlMain": "https://vsco.co/", "username_claimed": "blue" }, + "Velog": { + "errorType": "status_code", + "url": "https://velog.io/@{}/posts", + "urlMain": "https://velog.io/", + "username_claimed": "qlgks1" + }, "Velomania": { "errorMsg": "\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u043d\u0435 \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043d \u0438 \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043f\u0440\u043e\u0444\u0438\u043b\u044f \u0434\u043b\u044f \u043f\u0440\u043e\u0441\u043c\u043e\u0442\u0440\u0430.", "errorType": "message", @@ -2223,6 +2272,12 @@ "urlProbe": "https://www.virustotal.com/ui/users/{}/avatar", "username_claimed": "blue" }, + "VLR": { + "errorType": "status_code", + "url": "https://www.vlr.gg/user/{}", + "urlMain": "https://www.vlr.gg", + "username_claimed": "optms" + }, "WICG Forum": { "errorType": "status_code", "regexCheck": "^(?![.-])[a-zA-Z0-9_.-]{3,20}$", diff --git a/sherlock_project/sherlock.py b/sherlock_project/sherlock.py index c87dfd5..996a9d6 100644 --- a/sherlock_project/sherlock.py +++ b/sherlock_project/sherlock.py @@ -24,6 +24,7 @@ import re from argparse import ArgumentParser, RawDescriptionHelpFormatter from json import loads as json_loads from time import monotonic +from typing import Optional import requests from requests_futures.sessions import FuturesSession @@ -167,14 +168,14 @@ def multiple_usernames(username): def sherlock( - username, - site_data, + username: str, + site_data: dict, query_notify: QueryNotify, tor: bool = False, unique_tor: bool = False, dump_response: bool = False, - proxy=None, - timeout=60, + proxy: Optional[str] = None, + timeout: int = 60, ): """Run Sherlock Analysis. @@ -474,7 +475,7 @@ def sherlock( raise ValueError( f"Unknown Error Type '{error_type}' for " f"site '{social_network}'" ) - + if dump_response: print("+++++++++++++++++++++") print(f"TARGET NAME : {social_network}") @@ -784,7 +785,24 @@ def main(): os.path.join(os.path.dirname(__file__), "resources/data.json") ) else: - sites = SitesInformation(args.json_file) + json_file_location = args.json_file + if args.json_file: + # If --json parameter is a number, interpret it as a pull request number + if args.json_file.isnumeric(): + pull_number = args.json_file + pull_url = f"https://api.github.com/repos/sherlock-project/sherlock/pulls/{pull_number}" + pull_request_raw = requests.get(pull_url).text + pull_request_json = json_loads(pull_request_raw) + + # Check if it's a valid pull request + if "message" in pull_request_json: + print(f"ERROR: Pull request #{pull_number} not found.") + sys.exit(1) + + head_commit_sha = pull_request_json["head"]["sha"] + json_file_location = f"https://raw.githubusercontent.com/sherlock-project/sherlock/{head_commit_sha}/sherlock_project/resources/data.json" + + sites = SitesInformation(json_file_location) except Exception as error: print(f"ERROR: {error}") sys.exit(1) diff --git a/tests/sherlock_interactives.py b/tests/sherlock_interactives.py index 2c9d9d9..c28b9dc 100644 --- a/tests/sherlock_interactives.py +++ b/tests/sherlock_interactives.py @@ -7,8 +7,8 @@ class Interactives: def run_cli(args:str = "") -> str: """Pass arguments to Sherlock as a normal user on the command line""" # Adapt for platform differences (Windows likes to be special) - if platform.system == "Windows": - command:str = f"py -m sherlock {args}" + if platform.system() == "Windows": + command:str = f"py -m sherlock_project {args}" else: command:str = f"sherlock {args}" @@ -20,8 +20,7 @@ class Interactives: raise InteractivesSubprocessError(e.output.decode()) - # -> list[str] is prefered, but will require deprecation of support for Python 3.8 - def walk_sherlock_for_files_with(pattern: str) -> list: + def walk_sherlock_for_files_with(pattern: str) -> list[str]: """Check all files within the Sherlock package for matching patterns""" pattern:re.Pattern = re.compile(pattern) matching_files:list[str] = [] diff --git a/tests/test_probes.py b/tests/test_probes.py index 21bbe1d..11fc8f8 100644 --- a/tests/test_probes.py +++ b/tests/test_probes.py @@ -44,7 +44,7 @@ class TestLiveTargets: # Known positives should only use sites trusted to be reliable and unchanging @pytest.mark.parametrize('site,username',[ - ('BodyBuilding', 'blue'), + ('Keybase', 'blue'), ('devRant', 'blue'), ]) def test_known_positives_via_response_url(self, sites_info, site, username): diff --git a/tox.ini b/tox.ini index 0c18215..1e9a47d 100644 --- a/tox.ini +++ b/tox.ini @@ -41,4 +41,3 @@ python = 3.11: py311 3.10: py310 3.9: py39 - 3.8: py38