From d2d1b977feb7932669c99833a5dffe20985678a7 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Sun, 10 Jun 2018 21:14:46 -0400 Subject: [PATCH] log wget 500 errors --- archiver/archive_methods.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/archiver/archive_methods.py b/archiver/archive_methods.py index 46032f88..4bb0e9e0 100644 --- a/archiver/archive_methods.py +++ b/archiver/archive_methods.py @@ -204,17 +204,19 @@ def fetch_wget(link_dir, link, requisites=FETCH_WGET_REQUISITES, timeout=TIMEOUT result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # index.html end() output = wget_output_path(link, look_in=domain_dir) + + # Check for common failure cases if result.returncode > 0 and result.returncode != 8: print(' got wget response code {}:'.format(result.returncode)) print('\n'.join(' ' + line for line in (result.stderr or result.stdout).decode().rsplit('\n', 10)[-10:] if line.strip())) if result.returncode == 4: raise Exception('Failed wget download') - if result.returncode > 0 and b'403: Forbidden' in result.stderr: raise Exception('403 Forbidden (try changing WGET_USER_AGENT)') - if result.returncode > 0 and b'404: Not Found' in result.stderr: raise Exception('404 Not Found') + if result.returncode > 0 and b'ERROR 500: Internal Server Error' in result.stderr: + raise Exception('500 Internal Server Error') except Exception as e: end() print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))