nicer timeout hints

archivebox/logging_util.py

@@ -356,6 +356,21 @@ def log_archive_method_finished(result: "ArchiveResult"):
     )
 
     if result.status == 'failed':
+        if result.output.__class__.__name__ == 'TimeoutExpired':
+            duration = (result.end_ts - result.start_ts).seconds
+            hint_header = [
+                '{lightyellow}Extractor timed out after {}s.{reset}'.format(duration, **ANSI),
+            ]
+        else:
+            hint_header = [
+                '{lightyellow}Extractor failed:{reset}'.format(**ANSI),
+                '    {reset}{} {red}{}{reset}'.format(
+                    result.output.__class__.__name__.replace('ArchiveError', ''),
+                    result.output, 
+                    **ANSI,
+                ),
+            ]
+
         # Prettify error output hints string and limit to five lines
         hints = getattr(result.output, 'hints', None) or ()
         if hints:
@@ -365,14 +380,10 @@ def log_archive_method_finished(result: "ArchiveResult"):
                 for line in hints[:5] if line.strip()
             )
 
+
         # Collect and prefix output lines with indentation
         output_lines = [
-            '{lightyellow}Extractor failed:{reset}'.format(**ANSI),
+            *hint_header,
-            '    {reset}{} {red}{}{reset}'.format(
-                result.output.__class__.__name__.replace('ArchiveError', ''),
-                result.output, 
-                **ANSI,
-            ),
             *hints,
             '{}Run to see full output:{}'.format(ANSI['lightred'], ANSI['reset']),
             *(['    cd {};'.format(result.pwd)] if result.pwd else []),

archivebox/util.py

@@ -15,7 +15,7 @@ from datetime import datetime
 from dateparser import parse as dateparser
 
 import requests
-from requests.exceptions import RequestException
+from requests.exceptions import RequestException, ReadTimeout
 from base32_crockford import encode as base32_encode                            # type: ignore
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
 
@@ -186,10 +186,12 @@ def get_headers(url: str, timeout: int=None) -> str:
             headers={'User-Agent': WGET_USER_AGENT},
             verify=CHECK_SSL_VALIDITY,
             timeout=timeout,
-            allow_redirects=True
+            allow_redirects=True,
         )
         if response.status_code >= 400:
             raise RequestException
+    except ReadTimeout:
+        raise
     except RequestException:
         response = requests.get(
             url,

bin/test.sh

@@ -14,4 +14,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && cd .. && pwd )"
 
 source "$DIR/.venv/bin/activate"
 
-pytest
+pytest -s