Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 20:23:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

log wget 500 errors

Browse source
This commit is contained in:
Nick Sweeting 2018-06-10 21:14:46 -04:00
parent c1c689cb94
commit d2d1b977fe
1 changed files with 4 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
archiver/archive_methods.py
View file

@ -204,17 +204,19 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, timeout=TIMEOUT
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # index.html
end()
output = wget_output_path(link, look_in=domain_dir)
# Check for common failure cases
if result.returncode > 0 and result.returncode != 8:
print(' got wget response code {}:'.format(result.returncode))
print('\n'.join(' ' + line for line in (result.stderr or result.stdout).decode().rsplit('\n', 10)[-10:] if line.strip()))
if result.returncode == 4:
raise Exception('Failed wget download')
if result.returncode > 0 and b'403: Forbidden' in result.stderr:
raise Exception('403 Forbidden (try changing WGET_USER_AGENT)')
if result.returncode > 0 and b'404: Not Found' in result.stderr:
raise Exception('404 Not Found')
if result.returncode > 0 and b'ERROR 500: Internal Server Error' in result.stderr:
raise Exception('500 Internal Server Error')
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))