diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index bf2eec4e..56009cd1 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -33,6 +33,7 @@ from config import ( WGET_USER_AGENT, CHECK_SSL_VALIDITY, COOKIES_FILE, + WGET_AUTO_COMPRESSION ) from util import ( domain, @@ -227,6 +228,7 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT): '-e', 'robots=off', '--restrict-file-names=unix', '--timeout={}'.format(timeout), + *(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()), *(() if FETCH_WARC else ('--timestamping',)), *(('--warc-file={}'.format(warc_path),) if FETCH_WARC else ()), *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()),