Fix website loader test

This commit is contained in:
Sascha Ißbrücker 2023-01-14 12:26:04 +01:00
parent 4f9170c48d
commit 43d52642a6

View file

@ -74,7 +74,8 @@ def load_page(url: str):
if size > MAX_CONTENT_LIMIT:
logger.debug(f'Cancel reading document after {size} bytes')
break
logger.debug(f'Request consumed: {r._content_consumed}')
if hasattr(r, '_content_consumed'):
logger.debug(f'Request consumed: {r._content_consumed}')
# Use charset_normalizer to determine encoding that best matches the response content
# Several sites seem to specify the response encoding incorrectly, so we ignore it and use custom logic instead