Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-25 13:40:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

show more detail in readabiliity error messages

Browse source
This commit is contained in:
Nick Sweeting 2024-01-03 20:09:31 -08:00
parent c26d9dc1bb
commit 78d942ac22
1 changed files with 2 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
archivebox/extractors/readability.py
View file

@ -73,7 +73,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
result_json = json.loads(result.stdout) result_json = json.loads(result.stdout)
assert result_json and 'content' in result_json, 'Readability output is not valid JSON' assert result_json and 'content' in result_json, 'Readability output is not valid JSON'
except json.JSONDecodeError: except json.JSONDecodeError:
raise ArchiveError('Readability was not able to archive the page', result.stdout + result.stderr) raise ArchiveError('Readability was not able to archive the page (invalid JSON)', result.stdout + result.stderr)
output_folder.mkdir(exist_ok=True) output_folder.mkdir(exist_ok=True)
readability_content = result_json.pop("textContent") readability_content = result_json.pop("textContent")
@ -81,8 +81,6 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
atomic_write(str(output_folder / "content.txt"), readability_content) atomic_write(str(output_folder / "content.txt"), readability_content)
atomic_write(str(output_folder / "article.json"), result_json) atomic_write(str(output_folder / "article.json"), result_json)
# parse out number of files downloaded from last line of stderr:
# "Downloaded: 76 files, 4.0M in 1.6s (2.52 MB/s)"
output_tail = [ output_tail = [
line.strip() line.strip()
for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:] for line in (result.stdout + result.stderr).decode().rsplit('\n', 5)[-5:]
@ -95,7 +93,7 @@ def save_readability(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEO
# Check for common failure cases # Check for common failure cases
if (result.returncode > 0): if (result.returncode > 0):
raise ArchiveError('Readability was not able to archive the page', hints) raise ArchiveError(f'Readability was not able to archive the page (status={result.returncode})', hints)
except (Exception, OSError) as err: except (Exception, OSError) as err:
status = 'failed' status = 'failed'
output = err output = err