Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-26 06:00:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

support negation patterns by checking both re.search and re.match

Browse source
This commit is contained in:
Nick Sweeting 2021-07-06 23:17:03 -04:00
parent 65d452fe7b
commit e4974d3536
1 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
archivebox/index/__init__.py
View file

@ -141,7 +141,10 @@ def archivable_links(links: Iterable[Link]) -> Iterable[Link]:
continue
if scheme(link.url) not in ('http', 'https', 'ftp'):
continue
if URL_BLACKLIST_PTN and URL_BLACKLIST_PTN.search(link.url):
if URL_BLACKLIST_PTN and (URL_BLACKLIST_PTN.match(link.url) or URL_BLACKLIST_PTN.search(link.url)):
# https://stackoverflow.com/questions/180986/what-is-the-difference-between-re-search-and-re-match
# we want both behaviors in order to support multiple patterns in the regex,
# and negation regexes like (?!someptnhere) to allow for whitelisting
continue
yield link