From f6d6a06c7872f6c8ccdf08ff65eef06e139742ad Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 9 May 2022 21:21:26 -0700 Subject: [PATCH] always show all totals in log output --- archivebox/main.py | 9 +++++---- archivebox/parsers/__init__.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/archivebox/main.py b/archivebox/main.py index 5f672656..12016dbc 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -589,6 +589,7 @@ def add(urls: Union[str, List[str]], # save verbatim args to sources write_ahead_log = save_text_as_source('\n'.join(urls), filename='{ts}-import.txt', out_dir=out_dir) + new_links += parse_links_from_source(write_ahead_log, root_url=None, parser=parser) # If we're going one level deeper, download each link and look for more links @@ -628,16 +629,16 @@ def add(urls: Union[str, List[str]], ts = datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') if update: - stderr(f'[*] [{ts}] Archiving + updating', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving + updating {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green') archive_links(imported_links, overwrite=overwrite, **archive_kwargs) elif update_all: - stderr(f'[*] [{ts}] Archiving + updating ', len(all_links), 'URLs from entire library...', color='green') + stderr(f'[*] [{ts}] Archiving + updating {len(all_links)}/{len(all_links)}', len(all_links), 'URLs from entire library...', color='green') archive_links(all_links, overwrite=overwrite, **archive_kwargs) elif overwrite: - stderr(f'[*] [{ts}] Archiving + overwriting', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving + overwriting {len(imported_links)}/{len(all_links)}', len(imported_links), 'URLs from added set...', color='green') archive_links(imported_links, overwrite=True, **archive_kwargs) elif new_links: - stderr(f'[*] [{ts}] Archiving', len(imported_links), 'URLs from added set...', color='green') + stderr(f'[*] [{ts}] Archiving {len(new_links)}/{len(all_links)} URLs from added set...', color='green') archive_links(new_links, overwrite=False, **archive_kwargs) diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index 0ae958f2..2ce01e1a 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -8,6 +8,7 @@ For examples of supported import formats see tests/. __package__ = 'archivebox.parsers' import re +import os from io import StringIO from typing import IO, Tuple, List, Optional