Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 20:23:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix: Improve encoding detection for rss+xml content types

Browse source
This commit is contained in:
Cristian 2020-07-02 12:11:23 -05:00
parent 0c48449aa6
commit 528fc8f1f6
1 changed files with 9 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
archivebox/util.py
View file

@ -160,6 +160,15 @@ def download_url(url: str, timeout: int=None) -> str:
verify=CHECK_SSL_VALIDITY,
timeout=timeout,
)
if response.headers.get('Content-Type') == 'application/rss+xml':
# Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
_TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
_XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P<xmlcharset>[\w-]+)')
_BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE)
_BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE)
match = _BODY_ENCODING_STR_RE.search(response.text[:1024])
if match:
response.encoding = match.group('xmlcharset')
return response.text