Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 12:13:05 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix the URL_REGEX used in generic_html parsers

Browse source 
Signed-off-by: longzai <437172242@qq.com>
This commit is contained in:
longzai 2024-04-08 04:53:05 +08:00
parent 1d49bee90b
commit 4ae765ec27
1 changed files with 6 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
archivebox/util.py
View file

@ -56,16 +56,15 @@ short_ts = lambda ts: str(parse_date(ts).timestamp()).split('.')[0]
ts_to_date_str = lambda ts: ts and parse_date(ts).strftime('%Y-%m-%d %H:%M')
ts_to_iso = lambda ts: ts and parse_date(ts).isoformat()
URL_REGEX = re.compile(
r'(?=('
r'http[s]?://' # start matching from allowed schemes
r'(?:[a-zA-Z]|[0-9]' # followed by allowed alphanum characters
r'|[-_$@.&+!*\(\),]' # or allowed symbols (keep hyphen first to match literal hyphen)
r'|(?:%[0-9a-fA-F][0-9a-fA-F]))' # or allowed unicode bytes
r'[^\]\[\(\)<>"\'\s]+' # stop parsing at these symbols
r'https?://' #match schemes http and https,but can't match ftp
r'(?:[A-Za-z0-9-]+\.)+[A-Za-z0-9-]+'#match domain
r'(?::\d+)?' #match port,mabey not occur
r'(?:/[^\\#\f\n\r\t\v]*)?' #match path and query,maybe not occur
## r'(?:#[^\]\[\(\)<>"\'\s]*){0,1}' #match fragment,but we don't need it actually
r'))',
re.IGNORECASE,
## re.IGNORECASE, #don't need to consider case problem
)
COLOR_REGEX = re.compile(r'\[(?P<arg_1>\d+)(;(?P<arg_2>\d+)(;(?P<arg_3>\d+))?)?m')