Mirrors/sherlock
2
0
Fork 0
mirror of https://github.com/sherlock-project/sherlock synced 2024-11-15 08:47:08 +00:00
Code Issues Projects Releases Packages Wiki Activity

Fixed mixed of tabs and spaces in a single file.

Browse source
This commit is contained in:
jenchieh 2019-03-09 19:33:42 +08:00
parent 05353201b3
commit 8b6b14fc6a
1 changed files with 36 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
site_list.py
View file

@ -12,18 +12,18 @@ from argparse import ArgumentParser, RawDescriptionHelpFormatter
pool = list() pool = list()
def get_rank(domain_to_query, dest): def get_rank(domain_to_query, dest):
result = -1 result = -1
url = "http://www.alexa.com/siteinfo/" + domain_to_query url = "http://www.alexa.com/siteinfo/" + domain_to_query
page = requests.get(url).text page = requests.get(url).text
soup = bs(page, features="lxml") soup = bs(page, features="lxml")
for span in soup.find_all('span'): for span in soup.find_all('span'):
if span.has_attr("class"): if span.has_attr("class"):
if "globleRank" in span["class"]: if "globleRank" in span["class"]:
for strong in span.find_all("strong"): for strong in span.find_all("strong"):
if strong.has_attr("class"): if strong.has_attr("class"):
if "metrics-data" in strong["class"]: if "metrics-data" in strong["class"]:
result = int(strong.text.strip().replace(',', '')) result = int(strong.text.strip().replace(',', ''))
dest['rank'] = result dest['rank'] = result
parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter parser = ArgumentParser(formatter_class=RawDescriptionHelpFormatter
) )
@ -34,38 +34,38 @@ parser.add_argument("--rank","-r",
args = parser.parse_args() args = parser.parse_args()
with open("data.json", "r", encoding="utf-8") as data_file: with open("data.json", "r", encoding="utf-8") as data_file:
data = json.load(data_file) data = json.load(data_file)
with open("sites.md", "w") as site_file: with open("sites.md", "w") as site_file:
data_length = len(data) data_length = len(data)
site_file.write(f'## List Of Supported Sites ({data_length} Sites In Total!)\n') site_file.write(f'## List Of Supported Sites ({data_length} Sites In Total!)\n')
for social_network in data: for social_network in data:
url_main = data.get(social_network).get("urlMain") url_main = data.get(social_network).get("urlMain")
data.get(social_network)["rank"] = 0 data.get(social_network)["rank"] = 0
if args.rank: if args.rank:
th = threading.Thread(target=get_rank, args=(url_main, data.get(social_network))) th = threading.Thread(target=get_rank, args=(url_main, data.get(social_network)))
else: else:
th = None th = None
pool.append((social_network, url_main, th)) pool.append((social_network, url_main, th))
if args.rank: if args.rank:
th.start() th.start()
index = 1 index = 1
for social_network, url_main, th in pool: for social_network, url_main, th in pool:
if args.rank: if args.rank:
th.join() th.join()
site_file.write(f'{index}. [{social_network}]({url_main})\n') site_file.write(f'{index}. [{social_network}]({url_main})\n')
sys.stdout.write("\r{0}".format(f"Updated {index} out of {data_length} entries")) sys.stdout.write("\r{0}".format(f"Updated {index} out of {data_length} entries"))
sys.stdout.flush() sys.stdout.flush()
index = index + 1 index = index + 1
if args.rank: if args.rank:
site_file.write(f'\nAlexa.com rank data fetched at ({datetime.utcnow()} UTC)\n') site_file.write(f'\nAlexa.com rank data fetched at ({datetime.utcnow()} UTC)\n')
sorted_json_data = json.dumps(data, indent=2, sort_keys=True) sorted_json_data = json.dumps(data, indent=2, sort_keys=True)
with open("data.json", "w") as data_file: with open("data.json", "w") as data_file:
data_file.write(sorted_json_data) data_file.write(sorted_json_data)
print("\nFinished updating supported site listing!") print("\nFinished updating supported site listing!")