From ed7ad04fd9406f8b7df27b5fb356d09309c3917a Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Tue, 26 Mar 2019 15:46:57 +0000 Subject: [PATCH 1/4] wget: Remove unsupported compression option --compression=auto is not supported on Raspberry Pis. This PR removes it from ArchiveBox. --- archivebox/archive_methods.py | 1 - 1 file changed, 1 deletion(-) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index b3915e2f..bf2eec4e 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -224,7 +224,6 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT): '--backup-converted', '--span-hosts', '--no-parent', - '--compression=auto', '-e', 'robots=off', '--restrict-file-names=unix', '--timeout={}'.format(timeout), From 51859731d382642f4520cbdeaa04a47308537f5c Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Tue, 26 Mar 2019 23:34:45 +0000 Subject: [PATCH 2/4] config.py: add WGET_AUTO_COMPRESSION --- archivebox/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/archivebox/config.py b/archivebox/config.py index d8e01b24..26f67b1a 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -72,6 +72,7 @@ TEMPLATES_DIR = os.path.join(PYTHON_PATH, 'templates') CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC +WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode ########################### Environment & Dependencies ######################### From ab72a2dad10d2379023940d3e7bd4a19108206d7 Mon Sep 17 00:00:00 2001 From: Starbeamrainbowlabs Date: Tue, 26 Mar 2019 23:38:50 +0000 Subject: [PATCH 3/4] Try to add --compression back again --- archivebox/archive_methods.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/archivebox/archive_methods.py b/archivebox/archive_methods.py index bf2eec4e..56009cd1 100644 --- a/archivebox/archive_methods.py +++ b/archivebox/archive_methods.py @@ -33,6 +33,7 @@ from config import ( WGET_USER_AGENT, CHECK_SSL_VALIDITY, COOKIES_FILE, + WGET_AUTO_COMPRESSION ) from util import ( domain, @@ -227,6 +228,7 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT): '-e', 'robots=off', '--restrict-file-names=unix', '--timeout={}'.format(timeout), + *(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()), *(() if FETCH_WARC else ('--timestamping',)), *(('--warc-file={}'.format(warc_path),) if FETCH_WARC else ()), *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()), From 6bec1709f6e5f936695a81bc7db688981ef90496 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Wed, 27 Mar 2019 04:48:38 -0400 Subject: [PATCH 4/4] Update config.py --- archivebox/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivebox/config.py b/archivebox/config.py index 26f67b1a..29ed2df2 100644 --- a/archivebox/config.py +++ b/archivebox/config.py @@ -72,7 +72,7 @@ TEMPLATES_DIR = os.path.join(PYTHON_PATH, 'templates') CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC -WGET_AUTO_COMPRESSION = not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode +WGET_AUTO_COMPRESSION = USE_WGET and WGET_BINARY and (not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode) ########################### Environment & Dependencies #########################