Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-23 04:33:11 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #350 from cdvv7788/title-encoding-issue

Browse source 
Title encoding issue
This commit is contained in:
Nick Sweeting 2020-07-02 13:21:15 -04:00 committed by GitHub
commit b4d1fe8b40
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
archivebox/util.py
View file

@ -160,6 +160,15 @@ def download_url(url: str, timeout: int=None) -> str:
verify=CHECK_SSL_VALIDITY,
timeout=timeout,
)
if response.headers.get('Content-Type') == 'application/rss+xml':
# Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
_TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
_XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P<xmlcharset>[\w-]+)')
_BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE)
_BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE)
match = _BODY_ENCODING_STR_RE.search(response.text[:1024])
if match:
response.encoding = match.group('xmlcharset')
return response.text