Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-14 00:17:08 +00:00
Code Issues Projects Releases Packages Wiki Activity

better stats collection and printing

Browse source
This commit is contained in:
Nick Sweeting 2019-03-22 23:00:43 -04:00
parent 4c499d77b6
commit ef865dd76a
2 changed files with 24 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
archivebox/archive_methods.py
View file

@ -85,29 +85,28 @@ def archive_link(link_dir, link):
link = load_json_link_index(link_dir, link) link = load_json_link_index(link_dir, link)
log_link_archiving_started(link_dir, link, is_new) log_link_archiving_started(link_dir, link, is_new)
skipped_entirely = True stats = {'skipped': 0, 'succeeded': 0, 'failed': 0}
for method_name, should_run, method_function in ARCHIVE_METHODS: for method_name, should_run, method_function in ARCHIVE_METHODS:
if method_name not in link['history']: if method_name not in link['history']:
link['history'][method_name] = [] link['history'][method_name] = []
if should_run(link_dir, link): if should_run(link_dir, link):
if skipped_entirely: log_archive_method_started(method_name)
skipped_entirely = False
print() result = method_function(link_dir, link)
link['history'][method_name].append(result)
stats[result['status']] += 1
log_archive_method_finished(result)
else: else:
continue stats['skipped'] += 1
log_archive_method_started(method_name) # print(' ', stats)
result = method_function(link_dir, link)
log_archive_method_finished(result)
link['history'][method_name].append(result)
write_link_index(link_dir, link) write_link_index(link_dir, link)
patch_links_index(link) patch_links_index(link)
log_link_archiving_finished(link_dir, link, is_new, stats)
log_link_archiving_finished(link_dir, link, is_new, skipped_entirely)
except Exception as err: except Exception as err:
print(' ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err)) print(' ! Failed to archive link: {}: {}'.format(err.__class__.__name__, err))

28
archivebox/logs.py
View file

@ -116,9 +116,9 @@ def log_archiving_finished(num_links):
duration, duration,
ANSI['reset'], ANSI['reset'],
)) ))
print(' - {} entries skipped'.format(_LAST_RUN_STATS['skipped'])) print(' - {} links skipped'.format(_LAST_RUN_STATS['skipped']))
print(' - {} entries updated'.format(_LAST_RUN_STATS['succeeded'])) print(' - {} links updated'.format(_LAST_RUN_STATS['succeeded']))
print(' - {} errors'.format(_LAST_RUN_STATS['failed'])) print(' - {} links had errors'.format(_LAST_RUN_STATS['failed']))
print(' To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', ''))) print(' To view your archive, open: {}/index.html'.format(OUTPUT_DIR.replace(REPO_DIR + '/', '')))
@ -135,26 +135,20 @@ def log_link_archiving_started(link_dir, link, is_new):
**ANSI, **ANSI,
)) ))
print(' {blue}{url}{reset}'.format(url=link['url'], **ANSI)) print(' {blue}{url}{reset}'.format(url=link['url'], **ANSI))
sys.stdout.write(' > {}{}'.format( print(' {} {}'.format(
'>' if is_new else '√',
pretty_path(link_dir), pretty_path(link_dir),
' (new)' if is_new else '',
)) ))
def log_link_archiving_finished(link_dir, link, is_new, skipped_entirely): def log_link_archiving_finished(link_dir, link, is_new, stats):
from util import latest_output total = sum(stats.values())
if all(output == 'succeeded' for output in latest_output(link).values()): if stats['failed'] > 0 :
_LAST_RUN_STATS['succeeded'] += 1
elif any(output == 'failed' for output in latest_output(link).values()):
_LAST_RUN_STATS['failed'] += 1 _LAST_RUN_STATS['failed'] += 1
else: elif stats['skipped'] == total:
_LAST_RUN_STATS['skipped'] += 1 _LAST_RUN_STATS['skipped'] += 1
else:
if skipped_entirely: _LAST_RUN_STATS['succeeded'] += 1
print('\r √ {}{}'.format(
pretty_path(link_dir),
' (new)' if is_new else '',
))
def log_archive_method_started(method): def log_archive_method_started(method):