Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-27 06:30:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #193 from sbrl/patch-1

Browse source 
wget: Remove unsupported compression option
This commit is contained in:
Nick Sweeting 2019-03-27 04:49:10 -04:00 committed by GitHub
commit e797a7a5f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/archive_methods.py
View file

@ -33,6 +33,7 @@ from config import (
WGET_USER_AGENT, WGET_USER_AGENT,
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
COOKIES_FILE, COOKIES_FILE,
WGET_AUTO_COMPRESSION
) )
from util import ( from util import (
domain, domain,
@ -224,10 +225,10 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT):
'--backup-converted', '--backup-converted',
'--span-hosts', '--span-hosts',
'--no-parent', '--no-parent',
'--compression=auto',
'-e', 'robots=off', '-e', 'robots=off',
'--restrict-file-names=unix', '--restrict-file-names=unix',
'--timeout={}'.format(timeout), '--timeout={}'.format(timeout),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*(() if FETCH_WARC else ('--timestamping',)), *(() if FETCH_WARC else ('--timestamping',)),
*(('--warc-file={}'.format(warc_path),) if FETCH_WARC else ()), *(('--warc-file={}'.format(warc_path),) if FETCH_WARC else ()),
*(('--page-requisites',) if FETCH_WGET_REQUISITES else ()), *(('--page-requisites',) if FETCH_WGET_REQUISITES else ()),

1
archivebox/config.py
View file

@ -72,6 +72,7 @@ TEMPLATES_DIR = os.path.join(PYTHON_PATH, 'templates')
CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true' CHROME_SANDBOX = os.getenv('CHROME_SANDBOX', 'True').lower() == 'true'
USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM USE_CHROME = FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM
USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC USE_WGET = FETCH_WGET or FETCH_WGET_REQUISITES or FETCH_WARC
WGET_AUTO_COMPRESSION = USE_WGET and WGET_BINARY and (not run([WGET_BINARY, "--compression=auto", "--help"], stdout=DEVNULL).returncode)
########################### Environment & Dependencies ######################### ########################### Environment & Dependencies #########################