Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-10 22:54:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

use hints system to pass stdout and errors up to the traceback

Browse source
This commit is contained in:
Nick Sweeting 2019-03-20 21:10:09 -04:00
parent c38de3bbe7
commit c79e1df8b2
1 changed files with 104 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

201
archivebox/archive_methods.py
View file

@ -81,6 +81,12 @@ def load_link_index(link_dir, link):
return link return link
class ArchiveError(Exception):
def __init__(self, message, hints=None):
super().__init__(message)
self.hints = hints
def archive_link(link_dir, link, overwrite=True): def archive_link(link_dir, link, overwrite=True):
"""download the DOM, PDF, and a screenshot into a folder named after the link's timestamp""" """download the DOM, PDF, and a screenshot into a folder named after the link's timestamp"""
@ -122,8 +128,6 @@ def print_link_status_line(link_dir, link, is_new):
)) ))
print(' > {}{}'.format(pretty_path(link_dir), ' (new)' if is_new else '')) print(' > {}{}'.format(pretty_path(link_dir), ' (new)' if is_new else ''))
# if link['type']:
# print(' i {}'.format(link['type']))
@ -222,7 +226,11 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
end() end()
output = wget_output_path(link) output = wget_output_path(link)
output_tail = [' ' + line for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:] if line.strip()] output_tail = [
line.strip()
for line in (result.stdout + result.stderr).decode().rsplit('\n', 3)[-3:]
if line.strip()
]
# parse out number of files downloaded from "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)" # parse out number of files downloaded from "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
files_downloaded = ( files_downloaded = (
@ -233,15 +241,17 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
# Check for common failure cases # Check for common failure cases
if result.returncode > 0 and files_downloaded < 1: if result.returncode > 0 and files_downloaded < 1:
print(' Got wget response code {}:'.format(result.returncode)) hints = (
print('\n'.join(output_tail)) 'Got wget response code {}:\n'.format(result.returncode),
*output_tail,
)
if b'403: Forbidden' in result.stderr: if b'403: Forbidden' in result.stderr:
raise Exception('403 Forbidden (try changing WGET_USER_AGENT)') raise ArchiveError('403 Forbidden (try changing WGET_USER_AGENT)', hints)
if b'404: Not Found' in result.stderr: if b'404: Not Found' in result.stderr:
raise Exception('404 Not Found') raise ArchiveError('404 Not Found', hints)
if b'ERROR 500: Internal Server Error' in result.stderr: if b'ERROR 500: Internal Server Error' in result.stderr:
raise Exception('500 Internal Server Error') raise ArchiveError('500 Internal Server Error', hints)
raise Exception('Got an error from the server') raise ArchiveError('Got an error from the server', hints)
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -254,37 +264,36 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, warc=FETCH_WARC
@attach_result_to_link('pdf') @attach_result_to_link('pdf')
def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR): def fetch_pdf(link_dir, link, timeout=TIMEOUT, **chrome_kwargs):
"""print PDF of site to file using chrome --headless""" """print PDF of site to file using chrome --headless"""
if link['type'] in ('PDF', 'image'): if is_static_file(link['url']):
return {'output': wget_output_path(link)} return {'output': wget_output_path(link), 'status': 'skipped'}
if os.path.exists(os.path.join(link_dir, 'output.pdf')): output = 'output.pdf'
return {'output': 'output.pdf', 'status': 'skipped'} if os.path.exists(os.path.join(link_dir, output)):
return {'output': output, 'status': 'skipped'}
CMD = [ CMD = [
*chrome_headless(user_data_dir=user_data_dir), *chrome_headless(timeout=timeout, **chrome_kwargs),
'--print-to-pdf', '--print-to-pdf',
'--hide-scrollbars',
'--timeout={}'.format((timeout) * 1000),
*(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
link['url'] link['url']
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
hints = None
try: try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout) result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout)
end() end()
if result.returncode: if result.returncode:
print(' ', (result.stderr or result.stdout).decode()) hints = (result.stderr or result.stdout).decode()
raise Exception('Failed to print PDF') raise ArchiveError('Failed to print PDF', hints)
chmod_file('output.pdf', cwd=link_dir) chmod_file('output.pdf', cwd=link_dir)
output = 'output.pdf'
except Exception as e: except Exception as e:
end() end()
output = e output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e) print_error_hints(cmd=CMD, pwd=link_dir, err=e, hints=hints)
return { return {
'cmd': CMD, 'cmd': CMD,
@ -292,24 +301,19 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
} }
@attach_result_to_link('screenshot') @attach_result_to_link('screenshot')
def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION): def fetch_screenshot(link_dir, link, timeout=TIMEOUT, **chrome_kwargs):
"""take screenshot of site using chrome --headless""" """take screenshot of site using chrome --headless"""
if link['type'] in ('PDF', 'image'): if is_static_file(link['url']):
return {'output': wget_output_path(link)} return {'output': wget_output_path(link), 'status': 'skipped'}
if os.path.exists(os.path.join(link_dir, 'screenshot.png')): output = 'screenshot.png'
return {'output': 'screenshot.png', 'status': 'skipped'} if os.path.exists(os.path.join(link_dir, output)):
return {'output': output, 'status': 'skipped'}
CMD = [ CMD = [
*chrome_headless(user_data_dir=user_data_dir), *chrome_headless(timeout=timeout, **chrome_kwargs),
'--screenshot', '--screenshot',
'--window-size={}'.format(resolution),
'--hide-scrollbars',
'--timeout={}'.format((timeout) * 1000),
*(() if CHECK_SSL_VALIDITY else ('--disable-web-security', '--ignore-certificate-errors')),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
# '--full-page', # TODO: make this actually work using ./bin/screenshot fullPage: true
link['url'], link['url'],
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
@ -317,10 +321,10 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout) result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout)
end() end()
if result.returncode: if result.returncode:
print(' ', (result.stderr or result.stdout).decode()) hints = (result.stderr or result.stdout).decode()
raise Exception('Failed to take screenshot') raise ArchiveError('Failed to take screenshot', hints)
chmod_file('screenshot.png', cwd=link_dir)
output = 'screenshot.png' chmod_file(output, cwd=link_dir)
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -332,22 +336,19 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
} }
@attach_result_to_link('dom') @attach_result_to_link('dom')
def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR): def fetch_dom(link_dir, link, timeout=TIMEOUT, **chrome_kwargs):
"""print HTML of site to file using chrome --dump-html""" """print HTML of site to file using chrome --dump-html"""
if link['type'] in ('PDF', 'image'): if is_static_file(link['url']):
return {'output': wget_output_path(link)} return {'output': wget_output_path(link), 'status': 'skipped'}
output_path = os.path.join(link_dir, 'output.html') output = 'output.html'
if os.path.exists(os.path.join(link_dir, output)):
if os.path.exists(output_path): return {'output': output, 'status': 'skipped'}
return {'output': 'output.html', 'status': 'skipped'}
CMD = [ CMD = [
*chrome_headless(user_data_dir=user_data_dir), *chrome_headless(timeout=timeout, **chrome_kwargs),
'--dump-dom', '--dump-dom',
'--timeout={}'.format((timeout) * 1000),
*(('--user-agent={}'.format(CHROME_USER_AGENT),) if CHROME_USER_AGENT else ()),
link['url'] link['url']
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
@ -356,10 +357,10 @@ def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
result = run(CMD, stdout=f, stderr=PIPE, cwd=link_dir, timeout=timeout) result = run(CMD, stdout=f, stderr=PIPE, cwd=link_dir, timeout=timeout)
end() end()
if result.returncode: if result.returncode:
print(' ', (result.stderr).decode()) hints = result.stderr.decode()
raise Exception('Failed to fetch DOM') raise ArchiveError('Failed to fetch DOM', hints)
chmod_file('output.html', cwd=link_dir)
output = 'output.html' chmod_file(output, cwd=link_dir)
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -393,13 +394,11 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT):
output = 'archive.org.txt' output = 'archive.org.txt'
archive_org_url = None archive_org_url = None
path = os.path.join(link_dir, output) path = os.path.join(link_dir, output)
if os.path.exists(path): if os.path.exists(path):
archive_org_url = open(path, 'r').read().strip() archive_org_url = open(path, 'r').read().strip()
return {'output': archive_org_url, 'status': 'skipped'} return {'output': archive_org_url, 'status': 'skipped'}
submit_url = 'https://web.archive.org/save/{}'.format(link['url']) submit_url = 'https://web.archive.org/save/{}'.format(link['url'])
CMD = [ CMD = [
CURL_BINARY, CURL_BINARY,
@ -414,23 +413,21 @@ def archive_dot_org(link_dir, link, timeout=TIMEOUT):
try: try:
result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=link_dir, timeout=timeout) result = run(CMD, stdout=PIPE, stderr=DEVNULL, cwd=link_dir, timeout=timeout)
end() end()
content_location, errors = parse_archive_dot_org_response(result.stdout) content_location, errors = parse_archive_dot_org_response(result.stdout)
if content_location: if content_location:
archive_org_url = 'https://web.archive.org{}'.format(content_location[0]) archive_org_url = 'https://web.archive.org{}'.format(content_location[0])
elif len(errors) == 1 and 'RobotAccessControlException' in errors[0]: elif len(errors) == 1 and 'RobotAccessControlException' in errors[0]:
archive_org_url = None archive_org_url = None
# raise Exception('Archive.org denied by {}/robots.txt'.format(domain(link['url']))) # raise ArchiveError('Archive.org denied by {}/robots.txt'.format(domain(link['url'])))
elif errors: elif errors:
raise Exception(', '.join(errors)) raise ArchiveError(', '.join(errors))
else: else:
raise Exception('Failed to find "content-location" URL header in Archive.org response.') raise ArchiveError('Failed to find "content-location" URL header in Archive.org response.')
except Exception as e: except Exception as e:
end() end()
output = e output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e) print_error_hints(cmd=CMD, pwd=link_dir, err=e)
if not isinstance(output, Exception): if not isinstance(output, Exception):
# instead of writing None when archive.org rejects the url write the # instead of writing None when archive.org rejects the url write the
# url to resubmit it to archive.org. This is so when the user visits # url to resubmit it to archive.org. This is so when the user visits
@ -452,7 +449,6 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
"""download site favicon from google's favicon api""" """download site favicon from google's favicon api"""
output = 'favicon.ico' output = 'favicon.ico'
if os.path.exists(os.path.join(link_dir, output)): if os.path.exists(os.path.join(link_dir, output)):
return {'output': output, 'status': 'skipped'} return {'output': output, 'status': 'skipped'}
@ -468,8 +464,7 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
try: try:
run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout) run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout)
end() end()
chmod_file('favicon.ico', cwd=link_dir) chmod_file(output, cwd=link_dir)
output = 'favicon.ico'
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -488,6 +483,9 @@ def fetch_title(link_dir, link, timeout=TIMEOUT):
if link['title'] and not link['title'].lower().startswith('http'): if link['title'] and not link['title'].lower().startswith('http'):
return {'output': link['title'], 'status': 'skipped'} return {'output': link['title'], 'status': 'skipped'}
if is_static_file(link['url']):
return {'output': None, 'status': 'skipped'}
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
try: try:
title = fetch_page_title(link['url'], timeout=timeout, progress=False) title = fetch_page_title(link['url'], timeout=timeout, progress=False)
@ -495,14 +493,12 @@ def fetch_title(link_dir, link, timeout=TIMEOUT):
output = title output = title
except Exception as e: except Exception as e:
end() end()
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e output = e
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
# titles should show up in the global index immediatley for better UX, if title and title.strip():
# do a hacky immediate replacement to add them in as we're archiving
# TODO: figure out how to do this without gnarly string replacement
if title:
link['title'] = title link['title'] = title
output = title
return { return {
'cmd': 'fetch_page_title("{}")'.format(link['url']), 'cmd': 'fetch_page_title("{}")'.format(link['url']),
@ -513,14 +509,13 @@ def fetch_title(link_dir, link, timeout=TIMEOUT):
def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False): def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
"""Download playlists or individual video, audio, and subtitles using youtube-dl""" """Download playlists or individual video, audio, and subtitles using youtube-dl"""
output = 'media'
output_path = os.path.join(link_dir, 'media')
# import ipdb; ipdb.set_trace() if os.path.exists(output_path) and not overwrite:
output = os.path.join(link_dir, 'media') return {'output': output, 'status': 'skipped'}
already_done = os.path.exists(output) # and os.listdir(output)
if already_done and not overwrite:
return {'output': 'media', 'status': 'skipped'}
os.makedirs(output, exist_ok=True) os.makedirs(output_path, exist_ok=True)
CMD = [ CMD = [
YOUTUBEDL_BINARY, YOUTUBEDL_BINARY,
'--write-description', '--write-description',
@ -546,9 +541,8 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
try: try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=output, timeout=timeout + 1) result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=output_path, timeout=timeout + 1)
chmod_file('media', cwd=link_dir) chmod_file(output, cwd=link_dir)
output = 'media'
end() end()
if result.returncode: if result.returncode:
if (b'ERROR: Unsupported URL' in result.stderr if (b'ERROR: Unsupported URL' in result.stderr
@ -559,9 +553,11 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
# These happen too frequently on non-media pages to warrant printing to console # These happen too frequently on non-media pages to warrant printing to console
pass pass
else: else:
print(' got youtubedl response code {}:'.format(result.returncode)) hints = (
print(result.stderr) 'got youtubedl response code {}:'.format(result.returncode)),
raise Exception('Failed to download media') *result.stderr.decode().split('\n'),
)
raise ArchiveError('Failed to download media', hints)
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -580,18 +576,17 @@ def fetch_git(link_dir, link, timeout=TIMEOUT):
url_is_clonable = ( url_is_clonable = (
domain(link['url']) in GIT_DOMAINS domain(link['url']) in GIT_DOMAINS
or link['url'].endswith('.git') or link['url'].endswith('.git')
or link['type'] == 'git'
) )
if not url_is_clonable or is_static_file(link['url']):
if not url_is_clonable:
return {'output': None, 'status': 'skipped'} return {'output': None, 'status': 'skipped'}
git_dir = os.path.join(link_dir, 'git')
if os.path.exists(git_dir):
return {'output': 'git', 'status': 'skipped'}
os.makedirs(git_dir, exist_ok=True)
output = 'git' output = 'git'
output_path = os.path.join(link_dir, 'git')
if os.path.exists(output_path):
return {'output': output, 'status': 'skipped'}
os.makedirs(output_path, exist_ok=True)
CMD = [ CMD = [
GIT_BINARY, GIT_BINARY,
'clone', 'clone',
@ -602,15 +597,15 @@ def fetch_git(link_dir, link, timeout=TIMEOUT):
] ]
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
try: try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=git_dir, timeout=timeout + 1) result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=output_path, timeout=timeout + 1)
end() end()
if result.returncode == 128: if result.returncode == 128:
# ignore failed re-download when the folder already exists # ignore failed re-download when the folder already exists
pass pass
elif result.returncode > 0: elif result.returncode > 0:
print(' got git response code {}:'.format(result.returncode)) hints = 'got git response code {}:'.format(result.returncode)
raise Exception('Failed git download') raise ArchiveError('Failed git download', hints)
except Exception as e: except Exception as e:
end() end()
output = e output = e
@ -621,9 +616,9 @@ def fetch_git(link_dir, link, timeout=TIMEOUT):
'output': output, 'output': output,
} }
def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR, headless=CHROME_HEADLESS, sandbox=CHROME_SANDBOX): def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR, headless=CHROME_HEADLESS, sandbox=CHROME_SANDBOX, check_ssl_validity=CHECK_SSL_VALIDITY, user_agent=CHROME_USER_AGENT, resolution=RESOLUTION, timeout=TIMEOUT):
global USER_DATA_DIR global CACHED_USER_DATA_DIR
user_data_dir = user_data_dir or USER_DATA_DIR user_data_dir = user_data_dir or CACHED_USER_DATA_DIR
cmd_args = [binary] cmd_args = [binary]
if headless: if headless:
@ -633,7 +628,18 @@ def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR, he
# dont use GPU or sandbox when running inside docker container # dont use GPU or sandbox when running inside docker container
cmd_args += ('--no-sandbox', '--disable-gpu') cmd_args += ('--no-sandbox', '--disable-gpu')
if not check_ssl_validity:
cmd_args += ('--disable-web-security', '--ignore-certificate-errors')
if user_agent:
cmd_args += ('--user-agent={}'.format(user_agent),)
if resolution:
cmd_args += ('--window-size={}'.format(RESOLUTION),)
if timeout:
cmd_args += ('--timeout={}'.format((timeout) * 1000),)
# Find chrome user data directory # Find chrome user data directory
default_profile_paths = ( default_profile_paths = (
'~/.config/chromium', '~/.config/chromium',
@ -653,10 +659,11 @@ def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR, he
for path in default_profile_paths: for path in default_profile_paths:
full_path = os.path.expanduser(path) full_path = os.path.expanduser(path)
if os.path.exists(full_path): if os.path.exists(full_path):
USER_DATA_DIR = full_path CACHED_USER_DATA_DIR = full_path
cmd_args.append('--user-data-dir={}'.format(full_path)) cmd_args.append('--user-data-dir={}'.format(full_path))
break break
return cmd_args return cmd_args
USER_DATA_DIR = CHROME_USER_DATA_DIR CACHED_USER_DATA_DIR = CHROME_USER_DATA_DIR