Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 20:23:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

config.py: update function exclude_blacklisted(links)

Browse source
This commit is contained in:
mlazana 2019-03-27 20:10:05 +02:00
parent a3705e31c6
commit 8502fa5cc3
1 changed files with 11 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
archivebox/links.py
View file

@ -34,11 +34,11 @@ from config import (
def validate_links(links): def validate_links(links):
check_links_structure(links) check_links_structure(links)
links = archivable_links(links) # remove chrome://, about:, mailto: etc. links = archivable_links(links) # remove chrome://, about:, mailto: etc.
links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls links = uniquefied_links(links) # merge/dedupe duplicate timestamps & urls
links = sorted_links(links) # deterministically sort the links based on timstamp, url links = sorted_links(links) # deterministically sort the links based on timstamp, url
links = exclude_links(links) # exclude links that are in blacklist links = list(exclude_links(links)) # exclude URLs that match the blacklisted url pattern regex
if not links: if not links:
print('[X] No links found :(') print('[X] No links found :(')
raise SystemExit(1) raise SystemExit(1)
@ -46,7 +46,7 @@ def validate_links(links):
for link in links: for link in links:
link['title'] = unescape(link['title'].strip()) if link['title'] else None link['title'] = unescape(link['title'].strip()) if link['title'] else None
check_link_structure(link) check_link_structure(link)
return list(links) return list(links)
@ -120,9 +120,8 @@ def lowest_uniq_timestamp(used_timestamps, timestamp):
return new_timestamp return new_timestamp
def exclude_links(links): def exclude_blacklisted(links):
"""exclude links that are in blacklist""" """exclude URLs that match the blacklisted url pattern regex"""
return (link for link in links if not URL_BLACKLIST.match(link['url']))
links = [link for link in links if not URL_BLACKLIST.match(link['url'])]
return links