From 4a5d60729684ae47723c8ab0efe899dcd9044c98 Mon Sep 17 00:00:00 2001 From: Nick Sweeting Date: Mon, 18 Nov 2024 19:08:49 -0800 Subject: [PATCH] move logging_util into archivebox.misc subfolder --- archivebox/cli/__init__.py | 2 +- archivebox/cli/archivebox_add.py | 2 +- archivebox/cli/archivebox_config.py | 2 +- archivebox/cli/archivebox_help.py | 2 +- archivebox/cli/archivebox_init.py | 2 +- archivebox/cli/archivebox_install.py | 2 +- archivebox/cli/archivebox_list.py | 4 +- archivebox/cli/archivebox_oneshot.py | 2 +- archivebox/cli/archivebox_remove.py | 2 +- archivebox/cli/archivebox_schedule.py | 2 +- archivebox/cli/archivebox_server.py | 2 +- archivebox/cli/archivebox_shell.py | 2 +- archivebox/cli/archivebox_status.py | 2 +- archivebox/cli/archivebox_update.py | 2 +- archivebox/cli/archivebox_version.py | 2 +- archivebox/config/constants.py | 2 +- archivebox/config/django.py | 4 +- archivebox/config/paths.py | 2 +- archivebox/core/admin_snapshots.py | 3 +- archivebox/core/settings_logging.py | 3 +- archivebox/core/views.py | 3 +- archivebox/extractors/__init__.py | 2 +- archivebox/index/__init__.py | 7 ++-- archivebox/index/html.py | 2 +- archivebox/main.py | 37 +++++++++---------- archivebox/misc/checks.py | 4 +- archivebox/{ => misc}/logging_util.py | 22 +++++------ archivebox/parsers/__init__.py | 2 +- .../abx_plugin_archivedotorg/archive_org.py | 2 +- .../abx_plugin_chrome/config.py | 4 +- .../abx_plugin_chrome/dom.py | 7 +--- .../abx_plugin_chrome/pdf.py | 7 +--- .../abx_plugin_chrome/screenshot.py | 2 +- .../abx_plugin_curl/headers.py | 8 +--- .../abx_plugin_favicon/favicon.py | 2 +- .../pkgs/abx-plugin-git/abx_plugin_git/git.py | 2 +- .../abx_plugin_htmltotext/htmltotext.py | 2 +- .../abx_plugin_mercury/mercury.py | 7 +--- .../abx_plugin_readability/readability.py | 6 +-- .../abx_plugin_singlefile/singlefile.py | 8 ++-- .../abx_plugin_title/extractor.py | 10 ++--- .../abx-plugin-wget/abx_plugin_wget/wget.py | 6 +-- .../abx_plugin_ytdlp/media.py | 2 +- archivebox/workers/supervisor_util.py | 2 +- 44 files changed, 89 insertions(+), 113 deletions(-) rename archivebox/{ => misc}/logging_util.py (97%) diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 3339b0b8..8649063f 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -246,7 +246,7 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st command.subcommand = 'help' if command.subcommand not in ('version',): - from ..logging_util import log_cli_command + from archivebox.misc.logging_util import log_cli_command log_cli_command( subcommand=command.subcommand, diff --git a/archivebox/cli/archivebox_add.py b/archivebox/cli/archivebox_add.py index 1e945a9b..db0bb305 100644 --- a/archivebox/cli/archivebox_add.py +++ b/archivebox/cli/archivebox_add.py @@ -18,8 +18,8 @@ from archivebox.config.common import ARCHIVING_CONFIG from archivebox.config.django import setup_django from archivebox.config.permissions import USER, HOSTNAME from archivebox.misc.checks import check_data_folder +from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr from archivebox.parsers import PARSERS -from archivebox.logging_util import SmartFormatter, accept_stdin, stderr if TYPE_CHECKING: diff --git a/archivebox/cli/archivebox_config.py b/archivebox/cli/archivebox_config.py index 51eedaea..5983dd43 100644 --- a/archivebox/cli/archivebox_config.py +++ b/archivebox/cli/archivebox_config.py @@ -11,8 +11,8 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR +from archivebox.misc.logging_util import SmartFormatter, accept_stdin from ..main import config -from ..logging_util import SmartFormatter, accept_stdin @docstring(config.__doc__) diff --git a/archivebox/cli/archivebox_help.py b/archivebox/cli/archivebox_help.py index de47e6a8..c12319dd 100755 --- a/archivebox/cli/archivebox_help.py +++ b/archivebox/cli/archivebox_help.py @@ -9,9 +9,9 @@ from pathlib import Path from typing import Optional, List, IO from archivebox.misc.util import docstring +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from archivebox.config import DATA_DIR from ..main import help -from ..logging_util import SmartFormatter, reject_stdin @docstring(help.__doc__) diff --git a/archivebox/cli/archivebox_init.py b/archivebox/cli/archivebox_init.py index 70d87e4b..42c33a37 100755 --- a/archivebox/cli/archivebox_init.py +++ b/archivebox/cli/archivebox_init.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import init diff --git a/archivebox/cli/archivebox_install.py b/archivebox/cli/archivebox_install.py index 2f68c57d..faf02fa3 100755 --- a/archivebox/cli/archivebox_install.py +++ b/archivebox/cli/archivebox_install.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import install diff --git a/archivebox/cli/archivebox_list.py b/archivebox/cli/archivebox_list.py index a9733377..e34717bc 100644 --- a/archivebox/cli/archivebox_list.py +++ b/archivebox/cli/archivebox_list.py @@ -8,8 +8,9 @@ import argparse from pathlib import Path from typing import Optional, List, IO -from archivebox.misc.util import docstring from archivebox.config import DATA_DIR +from archivebox.misc.util import docstring +from archivebox.misc.logging_util import SmartFormatter, reject_stdin, stderr from ..main import list_all from ..index import ( LINK_FILTERS, @@ -24,7 +25,6 @@ from ..index import ( get_corrupted_folders, get_unrecognized_folders, ) -from ..logging_util import SmartFormatter, reject_stdin, stderr @docstring(list_all.__doc__) diff --git a/archivebox/cli/archivebox_oneshot.py b/archivebox/cli/archivebox_oneshot.py index 61dc2bb4..75dc53a6 100644 --- a/archivebox/cli/archivebox_oneshot.py +++ b/archivebox/cli/archivebox_oneshot.py @@ -11,7 +11,7 @@ from typing import List, Optional, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, accept_stdin, stderr +from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr from ..main import oneshot diff --git a/archivebox/cli/archivebox_remove.py b/archivebox/cli/archivebox_remove.py index 9e7e434b..40c426b4 100644 --- a/archivebox/cli/archivebox_remove.py +++ b/archivebox/cli/archivebox_remove.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, accept_stdin +from archivebox.misc.logging_util import SmartFormatter, accept_stdin from ..main import remove diff --git a/archivebox/cli/archivebox_schedule.py b/archivebox/cli/archivebox_schedule.py index f6920b05..4f791421 100644 --- a/archivebox/cli/archivebox_schedule.py +++ b/archivebox/cli/archivebox_schedule.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import schedule diff --git a/archivebox/cli/archivebox_server.py b/archivebox/cli/archivebox_server.py index 3c57bf43..4f1e2a60 100644 --- a/archivebox/cli/archivebox_server.py +++ b/archivebox/cli/archivebox_server.py @@ -11,7 +11,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR from archivebox.config.common import SERVER_CONFIG -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import server @docstring(server.__doc__) diff --git a/archivebox/cli/archivebox_shell.py b/archivebox/cli/archivebox_shell.py index c904e0a2..3b035c82 100644 --- a/archivebox/cli/archivebox_shell.py +++ b/archivebox/cli/archivebox_shell.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import shell diff --git a/archivebox/cli/archivebox_status.py b/archivebox/cli/archivebox_status.py index 22e37973..017c1ea1 100644 --- a/archivebox/cli/archivebox_status.py +++ b/archivebox/cli/archivebox_status.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO from archivebox.misc.util import docstring from archivebox.config import DATA_DIR -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin from ..main import status diff --git a/archivebox/cli/archivebox_update.py b/archivebox/cli/archivebox_update.py index d816b0c0..3fc3d116 100644 --- a/archivebox/cli/archivebox_update.py +++ b/archivebox/cli/archivebox_update.py @@ -21,7 +21,7 @@ from archivebox.index import ( get_corrupted_folders, get_unrecognized_folders, ) -from archivebox.logging_util import SmartFormatter, accept_stdin +from archivebox.misc.logging_util import SmartFormatter, accept_stdin # from ..main import update def update(): diff --git a/archivebox/cli/archivebox_version.py b/archivebox/cli/archivebox_version.py index 4229ff5a..4d55227d 100755 --- a/archivebox/cli/archivebox_version.py +++ b/archivebox/cli/archivebox_version.py @@ -10,7 +10,7 @@ from typing import Optional, List, IO # from archivebox.misc.util import docstring from archivebox.config import DATA_DIR, VERSION -from ..logging_util import SmartFormatter, reject_stdin +from archivebox.misc.logging_util import SmartFormatter, reject_stdin # @docstring(version.__doc__) diff --git a/archivebox/config/constants.py b/archivebox/config/constants.py index 5124384d..80894b58 100644 --- a/archivebox/config/constants.py +++ b/archivebox/config/constants.py @@ -21,7 +21,7 @@ from collections.abc import Mapping from benedict import benedict -from ..misc.logging import DEFAULT_CLI_COLORS +from archivebox.misc.logging import DEFAULT_CLI_COLORS from .paths import ( PACKAGE_DIR, diff --git a/archivebox/config/django.py b/archivebox/config/django.py index 073cd2d4..3fb85eb5 100644 --- a/archivebox/config/django.py +++ b/archivebox/config/django.py @@ -10,10 +10,10 @@ from rich.console import Console import django -from . import CONSTANTS +from archivebox.misc import logging +from . import CONSTANTS from .common import SHELL_CONFIG -from ..misc import logging if not SHELL_CONFIG.USE_COLOR: diff --git a/archivebox/config/paths.py b/archivebox/config/paths.py index 1f582881..8edb0759 100644 --- a/archivebox/config/paths.py +++ b/archivebox/config/paths.py @@ -114,7 +114,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool: """Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)""" - from archivebox.logging_util import pretty_path + from archivebox.misc.logging_util import pretty_path try: socket_path = str(dir_path / '.test_socket.sock') diff --git a/archivebox/core/admin_snapshots.py b/archivebox/core/admin_snapshots.py index 4f6ff945..08a9b9b7 100644 --- a/archivebox/core/admin_snapshots.py +++ b/archivebox/core/admin_snapshots.py @@ -17,9 +17,8 @@ from archivebox.config import DATA_DIR from archivebox.config.common import SERVER_CONFIG from archivebox.misc.util import htmldecode, urldecode from archivebox.misc.paginators import AccelleratedPaginator +from archivebox.misc.logging_util import printable_filesize from archivebox.search.admin import SearchResultsAdminMixin - -from archivebox.logging_util import printable_filesize from archivebox.index.html import snapshot_icons from archivebox.extractors import archive_links from archivebox.main import remove diff --git a/archivebox/core/settings_logging.py b/archivebox/core/settings_logging.py index 6962ad6f..e871b7bf 100644 --- a/archivebox/core/settings_logging.py +++ b/archivebox/core/settings_logging.py @@ -11,8 +11,7 @@ import pydantic import django.template from archivebox.config import CONSTANTS - -from ..misc.logging import IS_TTY +from archivebox.misc.logging import IS_TTY IGNORABLE_URL_PATTERNS = [ diff --git a/archivebox/core/views.py b/archivebox/core/views.py index a6ad7d9e..3603b43a 100644 --- a/archivebox/core/views.py +++ b/archivebox/core/views.py @@ -31,8 +31,7 @@ from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG from archivebox.misc.util import base_url, htmlencode, ts_to_date_str from archivebox.misc.serve_static import serve_static_with_byterange_support - -from archivebox.logging_util import printable_filesize +from archivebox.misc.logging_util import printable_filesize from archivebox.search import query_search_index diff --git a/archivebox/extractors/__init__.py b/archivebox/extractors/__init__.py index 42f9d6c7..b1d4f513 100644 --- a/archivebox/extractors/__init__.py +++ b/archivebox/extractors/__init__.py @@ -17,7 +17,7 @@ from ..index import ( write_link_details, ) from archivebox.misc.util import enforce_types -from ..logging_util import ( +from archivebox.misc.logging_util import ( log_archiving_started, log_archiving_paused, log_archiving_finished, diff --git a/archivebox/index/__init__.py b/archivebox/index/__init__.py index a12f83fd..fc159ec8 100644 --- a/archivebox/index/__init__.py +++ b/archivebox/index/__init__.py @@ -12,13 +12,12 @@ from urllib.parse import urlparse from django.db.models import QuerySet, Q -from archivebox.misc.logging import stderr -from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder from archivebox.config import DATA_DIR, CONSTANTS from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG - -from ..logging_util import ( +from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder +from archivebox.misc.logging import stderr +from archivebox.misc.logging_util import ( TimedProgress, log_indexing_process_started, log_indexing_process_finished, diff --git a/archivebox/index/html.py b/archivebox/index/html.py index 24cad5c0..934845a0 100644 --- a/archivebox/index/html.py +++ b/archivebox/index/html.py @@ -21,9 +21,9 @@ from archivebox.misc.util import ( from archivebox.config import CONSTANTS, DATA_DIR, VERSION from archivebox.config.common import SERVER_CONFIG from archivebox.config.version import get_COMMIT_HASH +from archivebox.misc.logging_util import printable_filesize from .schema import Link -from ..logging_util import printable_filesize MAIN_INDEX_TEMPLATE = 'static_index.html' MINIMAL_INDEX_TEMPLATE = 'minimal_index.html' diff --git a/archivebox/main.py b/archivebox/main.py index 63dba52c..74f8813c 100755 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -18,20 +18,30 @@ from abx_pkg import Binary import abx import archivebox +from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR +from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG +from archivebox.config.permissions import SudoPermission, IN_DOCKER +from archivebox.config.collection import write_config_file, load_all_config, get_real_name from archivebox.misc.checks import check_data_folder from archivebox.misc.util import enforce_types # type: ignore from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT from archivebox.misc.system import run as run_shell from archivebox.misc.logging import stderr, hint -from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR -from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG -from archivebox.config.permissions import SudoPermission, IN_DOCKER -from archivebox.config.collection import ( - write_config_file, - load_all_config, - get_real_name, +from archivebox.misc.logging_util import ( + TimedProgress, + log_importing_started, + log_crawl_started, + log_removal_started, + log_removal_finished, + log_list_started, + log_list_finished, + printable_config, + printable_folders, + printable_filesize, + printable_folder_status, ) + from .cli import ( CLI_SUBCOMMANDS, run_subcommand, @@ -78,19 +88,6 @@ from .index.sql import ( from .index.html import generate_index_from_links from .index.csv import links_to_csv from .extractors import archive_links, archive_link, ignore_methods -from .logging_util import ( - TimedProgress, - log_importing_started, - log_crawl_started, - log_removal_started, - log_removal_finished, - log_list_started, - log_list_finished, - printable_config, - printable_folders, - printable_filesize, - printable_folder_status, -) @enforce_types diff --git a/archivebox/misc/checks.py b/archivebox/misc/checks.py index 8a2894fe..bf4fae9a 100644 --- a/archivebox/misc/checks.py +++ b/archivebox/misc/checks.py @@ -157,9 +157,9 @@ def check_data_dir_permissions(): def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True): from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir from archivebox.misc.logging import STDERR + from archivebox.misc.logging_util import pretty_path from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP from archivebox.config.common import STORAGE_CONFIG - from archivebox.logging_util import pretty_path tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR socket_file = tmp_dir.absolute().resolve() / "supervisord.sock" @@ -204,9 +204,9 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex import archivebox from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP from archivebox.misc.logging import STDERR + from archivebox.misc.logging_util import pretty_path from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir from archivebox.config.common import STORAGE_CONFIG - from archivebox.logging_util import pretty_path lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR diff --git a/archivebox/logging_util.py b/archivebox/misc/logging_util.py similarity index 97% rename from archivebox/logging_util.py rename to archivebox/misc/logging_util.py index 33ab0766..4996a188 100644 --- a/archivebox/logging_util.py +++ b/archivebox/misc/logging_util.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING if TYPE_CHECKING: - from .index.schema import Link, ArchiveResult + from ..index.schema import Link, ArchiveResult from rich import print from rich.panel import Panel @@ -303,14 +303,14 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None): print() if resume: print('[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]'.format( - start_ts.strftime('%Y-%m-%d %H:%M:%S'), - num_links, - resume, + start_ts.strftime('%Y-%m-%d %H:%M:%S'), + num_links, + resume, )) else: print('[green][▶] [{}] Starting archiving of {} snapshots in index...[/]'.format( - start_ts.strftime('%Y-%m-%d %H:%M:%S'), - num_links, + start_ts.strftime('%Y-%m-%d %H:%M:%S'), + num_links, )) def log_archiving_paused(num_links: int, idx: int, timestamp: str): @@ -400,8 +400,9 @@ def log_archive_method_started(method: str): def log_archive_method_finished(result: "ArchiveResult"): - """quote the argument with whitespace in a command so the user can - copy-paste the outputted string directly to run the cmd + """ + quote the argument with whitespace in a command so the user can + copy-paste the outputted string directly to run the cmd """ # Prettify CMD string and make it safe to copy-paste by quoting arguments quoted_cmd = ' '.join( @@ -467,7 +468,7 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str): print(' {}'.format(' '.join(filter_patterns or ()))) def log_list_finished(links): - from .index.csv import links_to_csv + from ..index.csv import links_to_csv print() print('---------------------------------------------------------------------------------------------------') print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | ')) @@ -545,8 +546,7 @@ def printable_filesize(num_bytes: Union[int, float]) -> str: @enforce_types -def printable_folders(folders: Dict[str, Optional["Link"]], - with_headers: bool=False) -> str: +def printable_folders(folders: Dict[str, Optional["Link"]], with_headers: bool=False) -> str: return '\n'.join( f'{folder} {link and link.url} "{link and link.title}"' for folder, link in folders.items() diff --git a/archivebox/parsers/__init__.py b/archivebox/parsers/__init__.py index be9623d9..d1965ee5 100644 --- a/archivebox/parsers/__init__.py +++ b/archivebox/parsers/__init__.py @@ -17,6 +17,7 @@ from archivebox.config import DATA_DIR, CONSTANTS from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG from archivebox.misc.system import atomic_write from archivebox.misc.logging import stderr, hint +from archivebox.misc.logging_util import TimedProgress, log_source_saved from archivebox.misc.util import ( basename, htmldecode, @@ -25,7 +26,6 @@ from archivebox.misc.util import ( ) from ..index.schema import Link -from ..logging_util import TimedProgress, log_source_saved from . import pocket_api from . import readwise_reader_api diff --git a/archivebox/pkgs/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py b/archivebox/pkgs/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py index c030fde5..a9102bb8 100644 --- a/archivebox/pkgs/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py +++ b/archivebox/pkgs/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py @@ -5,8 +5,8 @@ from pathlib import Path from typing import Optional, List, Dict, Tuple from collections import defaultdict -from archivebox.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError +from archivebox.misc.logging_util import TimedProgress from archivebox.misc.system import run, chmod_file from archivebox.misc.util import enforce_types, is_static_file, dedupe diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py index 2635bffb..3efea3b3 100644 --- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py +++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/config.py @@ -10,9 +10,9 @@ from abx_plugin_default_binproviders import env from archivebox.config import CONSTANTS from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG -from archivebox.misc.logging import STDERR from archivebox.misc.util import dedupe -from archivebox.logging_util import pretty_path +from archivebox.misc.logging import STDERR +from archivebox.misc.logging_util import pretty_path CHROMIUM_BINARY_NAMES_LINUX = [ diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/dom.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/dom.py index 721d7f17..cef5f146 100644 --- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/dom.py +++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/dom.py @@ -5,11 +5,8 @@ from typing import Optional from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError from archivebox.misc.system import run, chmod_file, atomic_write -from archivebox.misc.util import ( - enforce_types, - is_static_file, -) -from archivebox.logging_util import TimedProgress +from archivebox.misc.util import enforce_types, is_static_file +from archivebox.misc.logging_util import TimedProgress from .config import CHROME_CONFIG from .binaries import CHROME_BINARY diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/pdf.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/pdf.py index cb69544b..5b67ef4a 100644 --- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/pdf.py +++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/pdf.py @@ -4,12 +4,9 @@ from pathlib import Path from typing import Optional from archivebox.misc.system import run, chmod_file -from archivebox.misc.util import ( - enforce_types, - is_static_file, -) +from archivebox.misc.util import enforce_types, is_static_file from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from .config import CHROME_CONFIG from .binaries import CHROME_BINARY diff --git a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/screenshot.py b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/screenshot.py index 227d2ad5..5d157f44 100644 --- a/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/screenshot.py +++ b/archivebox/pkgs/abx-plugin-chrome/abx_plugin_chrome/screenshot.py @@ -5,8 +5,8 @@ from typing import Optional from archivebox.misc.system import run, chmod_file from archivebox.misc.util import enforce_types, is_static_file +from archivebox.misc.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError -from archivebox.logging_util import TimedProgress from .config import CHROME_CONFIG from .binaries import CHROME_BINARY diff --git a/archivebox/pkgs/abx-plugin-curl/abx_plugin_curl/headers.py b/archivebox/pkgs/abx-plugin-curl/abx_plugin_curl/headers.py index 335d5678..3e9c0d30 100644 --- a/archivebox/pkgs/abx-plugin-curl/abx_plugin_curl/headers.py +++ b/archivebox/pkgs/abx-plugin-curl/abx_plugin_curl/headers.py @@ -4,14 +4,10 @@ from pathlib import Path from typing import Optional -from archivebox.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput from archivebox.misc.system import atomic_write -from archivebox.misc.util import ( - enforce_types, - get_headers, - dedupe, -) +from archivebox.misc.util import enforce_types, get_headers, dedupe +from archivebox.misc.logging_util import TimedProgress from .binaries import CURL_BINARY from .config import CURL_CONFIG diff --git a/archivebox/pkgs/abx-plugin-favicon/abx_plugin_favicon/favicon.py b/archivebox/pkgs/abx-plugin-favicon/abx_plugin_favicon/favicon.py index e77a61ce..3be96595 100644 --- a/archivebox/pkgs/abx-plugin-favicon/abx_plugin_favicon/favicon.py +++ b/archivebox/pkgs/abx-plugin-favicon/abx_plugin_favicon/favicon.py @@ -4,8 +4,8 @@ from pathlib import Path from archivebox.misc.system import chmod_file, run from archivebox.misc.util import enforce_types, domain, dedupe +from archivebox.misc.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput -from archivebox.logging_util import TimedProgress from abx_plugin_curl.config import CURL_CONFIG from abx_plugin_curl.binaries import CURL_BINARY diff --git a/archivebox/pkgs/abx-plugin-git/abx_plugin_git/git.py b/archivebox/pkgs/abx-plugin-git/abx_plugin_git/git.py index 713b124e..b3e4d120 100644 --- a/archivebox/pkgs/abx-plugin-git/abx_plugin_git/git.py +++ b/archivebox/pkgs/abx-plugin-git/abx_plugin_git/git.py @@ -13,7 +13,7 @@ from archivebox.misc.util import ( without_query, without_fragment, ) -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError from abx_plugin_git.config import GIT_CONFIG diff --git a/archivebox/pkgs/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py b/archivebox/pkgs/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py index 6faac6b3..a901ac7c 100644 --- a/archivebox/pkgs/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py +++ b/archivebox/pkgs/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py @@ -9,7 +9,7 @@ from archivebox.config import VERSION from archivebox.config.common import ARCHIVING_CONFIG from archivebox.misc.system import atomic_write from archivebox.misc.util import enforce_types, is_static_file -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveError from abx_plugin_title.extractor import get_html diff --git a/archivebox/pkgs/abx-plugin-mercury/abx_plugin_mercury/mercury.py b/archivebox/pkgs/abx-plugin-mercury/abx_plugin_mercury/mercury.py index 5bb6fead..56ed1136 100644 --- a/archivebox/pkgs/abx-plugin-mercury/abx_plugin_mercury/mercury.py +++ b/archivebox/pkgs/abx-plugin-mercury/abx_plugin_mercury/mercury.py @@ -6,13 +6,10 @@ from subprocess import CompletedProcess from typing import Optional, List import json -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveError from archivebox.misc.system import run, atomic_write -from archivebox.misc.util import ( - enforce_types, - is_static_file, -) +from archivebox.misc.util import enforce_types, is_static_file from .config import MERCURY_CONFIG from .binaries import MERCURY_BINARY diff --git a/archivebox/pkgs/abx-plugin-readability/abx_plugin_readability/readability.py b/archivebox/pkgs/abx-plugin-readability/abx_plugin_readability/readability.py index e17349df..5de5f122 100644 --- a/archivebox/pkgs/abx-plugin-readability/abx_plugin_readability/readability.py +++ b/archivebox/pkgs/abx-plugin-readability/abx_plugin_readability/readability.py @@ -1,15 +1,15 @@ __package__ = 'abx_plugin_readability' +import json from pathlib import Path from tempfile import NamedTemporaryFile - from typing import Optional -import json from archivebox.misc.system import run, atomic_write from archivebox.misc.util import enforce_types, is_static_file from archivebox.index.schema import Link, ArchiveResult, ArchiveError -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress + from abx_plugin_title.extractor import get_html from .config import READABILITY_CONFIG diff --git a/archivebox/pkgs/abx-plugin-singlefile/abx_plugin_singlefile/singlefile.py b/archivebox/pkgs/abx-plugin-singlefile/abx_plugin_singlefile/singlefile.py index 361f996c..4bcbc002 100644 --- a/archivebox/pkgs/abx-plugin-singlefile/abx_plugin_singlefile/singlefile.py +++ b/archivebox/pkgs/abx-plugin-singlefile/abx_plugin_singlefile/singlefile.py @@ -1,17 +1,17 @@ __package__ = 'abx_plugin_singlefile' -from pathlib import Path - -from typing import Optional import json +from pathlib import Path +from typing import Optional from archivebox.index.schema import Link, ArchiveResult, ArchiveError from archivebox.misc.system import run, chmod_file from archivebox.misc.util import enforce_types, is_static_file, dedupe -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from abx_plugin_chrome.config import CHROME_CONFIG from abx_plugin_chrome.binaries import CHROME_BINARY + from .config import SINGLEFILE_CONFIG from .binaries import SINGLEFILE_BINARY diff --git a/archivebox/pkgs/abx-plugin-title/abx_plugin_title/extractor.py b/archivebox/pkgs/abx-plugin-title/abx_plugin_title/extractor.py index ec4507df..eec79bbd 100644 --- a/archivebox/pkgs/abx-plugin-title/abx_plugin_title/extractor.py +++ b/archivebox/pkgs/abx-plugin-title/abx_plugin_title/extractor.py @@ -6,13 +6,9 @@ from pathlib import Path from typing import Optional from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError -from archivebox.logging_util import TimedProgress -from archivebox.misc.util import ( - enforce_types, - download_url, - htmldecode, - dedupe, -) +from archivebox.misc.logging_util import TimedProgress +from archivebox.misc.util import enforce_types, download_url, htmldecode, dedupe) + from abx_plugin_curl.config import CURL_CONFIG from abx_plugin_curl.binaries import CURL_BINARY diff --git a/archivebox/pkgs/abx-plugin-wget/abx_plugin_wget/wget.py b/archivebox/pkgs/abx-plugin-wget/abx_plugin_wget/wget.py index db589dd8..209d844f 100644 --- a/archivebox/pkgs/abx-plugin-wget/abx_plugin_wget/wget.py +++ b/archivebox/pkgs/abx-plugin-wget/abx_plugin_wget/wget.py @@ -2,13 +2,12 @@ __package__ = 'abx_plugin_wget' import re import os -from pathlib import Path - from typing import Optional +from pathlib import Path from datetime import datetime, timezone -from archivebox.logging_util import TimedProgress from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError +from archivebox.misc.logging_util import TimedProgress from archivebox.misc.system import run, chmod_file from archivebox.misc.util import ( enforce_types, @@ -19,6 +18,7 @@ from archivebox.misc.util import ( urldecode, dedupe, ) + from .config import WGET_CONFIG from .binaries import WGET_BINARY diff --git a/archivebox/pkgs/abx-plugin-ytdlp/abx_plugin_ytdlp/media.py b/archivebox/pkgs/abx-plugin-ytdlp/abx_plugin_ytdlp/media.py index 03df93d1..262f8097 100644 --- a/archivebox/pkgs/abx-plugin-ytdlp/abx_plugin_ytdlp/media.py +++ b/archivebox/pkgs/abx-plugin-ytdlp/abx_plugin_ytdlp/media.py @@ -6,7 +6,7 @@ from typing import Optional from archivebox.misc.system import run, chmod_file from archivebox.misc.util import enforce_types, is_static_file, dedupe from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError -from archivebox.logging_util import TimedProgress +from archivebox.misc.logging_util import TimedProgress from .config import YTDLP_CONFIG from .binaries import YTDLP_BINARY diff --git a/archivebox/workers/supervisor_util.py b/archivebox/workers/supervisor_util.py index f57fc7ce..bd443569 100644 --- a/archivebox/workers/supervisor_util.py +++ b/archivebox/workers/supervisor_util.py @@ -19,7 +19,7 @@ from archivebox.config import CONSTANTS from archivebox.config.paths import get_or_create_working_tmp_dir from archivebox.config.permissions import ARCHIVEBOX_USER from archivebox.misc.logging import STDERR -from archivebox.logging_util import pretty_path +from archivebox.misc.logging_util import pretty_path LOG_FILE_NAME = "supervisord.log" CONFIG_FILE_NAME = "supervisord.conf"