move logging_util into archivebox.misc subfolder

This commit is contained in:
Nick Sweeting 2024-11-18 19:08:49 -08:00
parent e469c5a344
commit 4a5d607296
No known key found for this signature in database
44 changed files with 89 additions and 113 deletions

View file

@ -246,7 +246,7 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
command.subcommand = 'help'
if command.subcommand not in ('version',):
from ..logging_util import log_cli_command
from archivebox.misc.logging_util import log_cli_command
log_cli_command(
subcommand=command.subcommand,

View file

@ -18,8 +18,8 @@ from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.config.django import setup_django
from archivebox.config.permissions import USER, HOSTNAME
from archivebox.misc.checks import check_data_folder
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
from archivebox.parsers import PARSERS
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
if TYPE_CHECKING:

View file

@ -11,8 +11,8 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
from ..main import config
from ..logging_util import SmartFormatter, accept_stdin
@docstring(config.__doc__)

View file

@ -9,9 +9,9 @@ from pathlib import Path
from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from archivebox.config import DATA_DIR
from ..main import help
from ..logging_util import SmartFormatter, reject_stdin
@docstring(help.__doc__)

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import init

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import install

View file

@ -8,8 +8,9 @@ import argparse
from pathlib import Path
from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.misc.util import docstring
from archivebox.misc.logging_util import SmartFormatter, reject_stdin, stderr
from ..main import list_all
from ..index import (
LINK_FILTERS,
@ -24,7 +25,6 @@ from ..index import (
get_corrupted_folders,
get_unrecognized_folders,
)
from ..logging_util import SmartFormatter, reject_stdin, stderr
@docstring(list_all.__doc__)

View file

@ -11,7 +11,7 @@ from typing import List, Optional, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin, stderr
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
from ..main import oneshot

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
from ..main import remove

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import schedule

View file

@ -11,7 +11,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import server
@docstring(server.__doc__)

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import shell

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
from ..main import status

View file

@ -21,7 +21,7 @@ from archivebox.index import (
get_corrupted_folders,
get_unrecognized_folders,
)
from archivebox.logging_util import SmartFormatter, accept_stdin
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
# from ..main import update
def update():

View file

@ -10,7 +10,7 @@ from typing import Optional, List, IO
# from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR, VERSION
from ..logging_util import SmartFormatter, reject_stdin
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
# @docstring(version.__doc__)

View file

@ -21,7 +21,7 @@ from collections.abc import Mapping
from benedict import benedict
from ..misc.logging import DEFAULT_CLI_COLORS
from archivebox.misc.logging import DEFAULT_CLI_COLORS
from .paths import (
PACKAGE_DIR,

View file

@ -10,10 +10,10 @@ from rich.console import Console
import django
from . import CONSTANTS
from archivebox.misc import logging
from . import CONSTANTS
from .common import SHELL_CONFIG
from ..misc import logging
if not SHELL_CONFIG.USE_COLOR:

View file

@ -114,7 +114,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
"""Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
from archivebox.logging_util import pretty_path
from archivebox.misc.logging_util import pretty_path
try:
socket_path = str(dir_path / '.test_socket.sock')

View file

@ -17,9 +17,8 @@ from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.misc.util import htmldecode, urldecode
from archivebox.misc.paginators import AccelleratedPaginator
from archivebox.misc.logging_util import printable_filesize
from archivebox.search.admin import SearchResultsAdminMixin
from archivebox.logging_util import printable_filesize
from archivebox.index.html import snapshot_icons
from archivebox.extractors import archive_links
from archivebox.main import remove

View file

@ -11,8 +11,7 @@ import pydantic
import django.template
from archivebox.config import CONSTANTS
from ..misc.logging import IS_TTY
from archivebox.misc.logging import IS_TTY
IGNORABLE_URL_PATTERNS = [

View file

@ -31,8 +31,7 @@ from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
from archivebox.misc.serve_static import serve_static_with_byterange_support
from archivebox.logging_util import printable_filesize
from archivebox.misc.logging_util import printable_filesize
from archivebox.search import query_search_index

View file

@ -17,7 +17,7 @@ from ..index import (
write_link_details,
)
from archivebox.misc.util import enforce_types
from ..logging_util import (
from archivebox.misc.logging_util import (
log_archiving_started,
log_archiving_paused,
log_archiving_finished,

View file

@ -12,13 +12,12 @@ from urllib.parse import urlparse
from django.db.models import QuerySet, Q
from archivebox.misc.logging import stderr
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
from ..logging_util import (
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
from archivebox.misc.logging import stderr
from archivebox.misc.logging_util import (
TimedProgress,
log_indexing_process_started,
log_indexing_process_finished,

View file

@ -21,9 +21,9 @@ from archivebox.misc.util import (
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.version import get_COMMIT_HASH
from archivebox.misc.logging_util import printable_filesize
from .schema import Link
from ..logging_util import printable_filesize
MAIN_INDEX_TEMPLATE = 'static_index.html'
MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'

View file

@ -18,20 +18,30 @@ from abx_pkg import Binary
import abx
import archivebox
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
from archivebox.config.permissions import SudoPermission, IN_DOCKER
from archivebox.config.collection import write_config_file, load_all_config, get_real_name
from archivebox.misc.checks import check_data_folder
from archivebox.misc.util import enforce_types # type: ignore
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
from archivebox.misc.system import run as run_shell
from archivebox.misc.logging import stderr, hint
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
from archivebox.config.permissions import SudoPermission, IN_DOCKER
from archivebox.config.collection import (
write_config_file,
load_all_config,
get_real_name,
from archivebox.misc.logging_util import (
TimedProgress,
log_importing_started,
log_crawl_started,
log_removal_started,
log_removal_finished,
log_list_started,
log_list_finished,
printable_config,
printable_folders,
printable_filesize,
printable_folder_status,
)
from .cli import (
CLI_SUBCOMMANDS,
run_subcommand,
@ -78,19 +88,6 @@ from .index.sql import (
from .index.html import generate_index_from_links
from .index.csv import links_to_csv
from .extractors import archive_links, archive_link, ignore_methods
from .logging_util import (
TimedProgress,
log_importing_started,
log_crawl_started,
log_removal_started,
log_removal_finished,
log_list_started,
log_list_finished,
printable_config,
printable_folders,
printable_filesize,
printable_folder_status,
)
@enforce_types

View file

@ -157,9 +157,9 @@ def check_data_dir_permissions():
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.common import STORAGE_CONFIG
from archivebox.logging_util import pretty_path
tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
@ -204,9 +204,9 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
import archivebox
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
from archivebox.config.common import STORAGE_CONFIG
from archivebox.logging_util import pretty_path
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR

View file

@ -15,7 +15,7 @@ from dataclasses import dataclass
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
if TYPE_CHECKING:
from .index.schema import Link, ArchiveResult
from ..index.schema import Link, ArchiveResult
from rich import print
from rich.panel import Panel
@ -303,14 +303,14 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
print()
if resume:
print('[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]'.format(
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
resume,
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
resume,
))
else:
print('[green][▶] [{}] Starting archiving of {} snapshots in index...[/]'.format(
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
num_links,
))
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
@ -400,8 +400,9 @@ def log_archive_method_started(method: str):
def log_archive_method_finished(result: "ArchiveResult"):
"""quote the argument with whitespace in a command so the user can
copy-paste the outputted string directly to run the cmd
"""
quote the argument with whitespace in a command so the user can
copy-paste the outputted string directly to run the cmd
"""
# Prettify CMD string and make it safe to copy-paste by quoting arguments
quoted_cmd = ' '.join(
@ -467,7 +468,7 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
print(' {}'.format(' '.join(filter_patterns or ())))
def log_list_finished(links):
from .index.csv import links_to_csv
from ..index.csv import links_to_csv
print()
print('---------------------------------------------------------------------------------------------------')
print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
@ -545,8 +546,7 @@ def printable_filesize(num_bytes: Union[int, float]) -> str:
@enforce_types
def printable_folders(folders: Dict[str, Optional["Link"]],
with_headers: bool=False) -> str:
def printable_folders(folders: Dict[str, Optional["Link"]], with_headers: bool=False) -> str:
return '\n'.join(
f'{folder} {link and link.url} "{link and link.title}"'
for folder, link in folders.items()

View file

@ -17,6 +17,7 @@ from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG
from archivebox.misc.system import atomic_write
from archivebox.misc.logging import stderr, hint
from archivebox.misc.logging_util import TimedProgress, log_source_saved
from archivebox.misc.util import (
basename,
htmldecode,
@ -25,7 +26,6 @@ from archivebox.misc.util import (
)
from ..index.schema import Link
from ..logging_util import TimedProgress, log_source_saved
from . import pocket_api
from . import readwise_reader_api

View file

@ -5,8 +5,8 @@ from pathlib import Path
from typing import Optional, List, Dict, Tuple
from collections import defaultdict
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe

View file

@ -10,9 +10,9 @@ from abx_plugin_default_binproviders import env
from archivebox.config import CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
from archivebox.misc.logging import STDERR
from archivebox.misc.util import dedupe
from archivebox.logging_util import pretty_path
from archivebox.misc.logging import STDERR
from archivebox.misc.logging_util import pretty_path
CHROMIUM_BINARY_NAMES_LINUX = [

View file

@ -5,11 +5,8 @@ from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.system import run, chmod_file, atomic_write
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.logging_util import TimedProgress
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.misc.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -4,12 +4,9 @@ from pathlib import Path
from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -5,8 +5,8 @@ from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from .config import CHROME_CONFIG
from .binaries import CHROME_BINARY

View file

@ -4,14 +4,10 @@ from pathlib import Path
from typing import Optional
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
from archivebox.misc.system import atomic_write
from archivebox.misc.util import (
enforce_types,
get_headers,
dedupe,
)
from archivebox.misc.util import enforce_types, get_headers, dedupe
from archivebox.misc.logging_util import TimedProgress
from .binaries import CURL_BINARY
from .config import CURL_CONFIG

View file

@ -4,8 +4,8 @@ from pathlib import Path
from archivebox.misc.system import chmod_file, run
from archivebox.misc.util import enforce_types, domain, dedupe
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
from archivebox.logging_util import TimedProgress
from abx_plugin_curl.config import CURL_CONFIG
from abx_plugin_curl.binaries import CURL_BINARY

View file

@ -13,7 +13,7 @@ from archivebox.misc.util import (
without_query,
without_fragment,
)
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from abx_plugin_git.config import GIT_CONFIG

View file

@ -9,7 +9,7 @@ from archivebox.config import VERSION
from archivebox.config.common import ARCHIVING_CONFIG
from archivebox.misc.system import atomic_write
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from abx_plugin_title.extractor import get_html

View file

@ -6,13 +6,10 @@ from subprocess import CompletedProcess
from typing import Optional, List
import json
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.misc.system import run, atomic_write
from archivebox.misc.util import (
enforce_types,
is_static_file,
)
from archivebox.misc.util import enforce_types, is_static_file
from .config import MERCURY_CONFIG
from .binaries import MERCURY_BINARY

View file

@ -1,15 +1,15 @@
__package__ = 'abx_plugin_readability'
import json
from pathlib import Path
from tempfile import NamedTemporaryFile
from typing import Optional
import json
from archivebox.misc.system import run, atomic_write
from archivebox.misc.util import enforce_types, is_static_file
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from abx_plugin_title.extractor import get_html
from .config import READABILITY_CONFIG

View file

@ -1,17 +1,17 @@
__package__ = 'abx_plugin_singlefile'
from pathlib import Path
from typing import Optional
import json
from pathlib import Path
from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from abx_plugin_chrome.config import CHROME_CONFIG
from abx_plugin_chrome.binaries import CHROME_BINARY
from .config import SINGLEFILE_CONFIG
from .binaries import SINGLEFILE_BINARY

View file

@ -6,13 +6,9 @@ from pathlib import Path
from typing import Optional
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.util import (
enforce_types,
download_url,
htmldecode,
dedupe,
)
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.util import enforce_types, download_url, htmldecode, dedupe)
from abx_plugin_curl.config import CURL_CONFIG
from abx_plugin_curl.binaries import CURL_BINARY

View file

@ -2,13 +2,12 @@ __package__ = 'abx_plugin_wget'
import re
import os
from pathlib import Path
from typing import Optional
from pathlib import Path
from datetime import datetime, timezone
from archivebox.logging_util import TimedProgress
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.misc.logging_util import TimedProgress
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import (
enforce_types,
@ -19,6 +18,7 @@ from archivebox.misc.util import (
urldecode,
dedupe,
)
from .config import WGET_CONFIG
from .binaries import WGET_BINARY

View file

@ -6,7 +6,7 @@ from typing import Optional
from archivebox.misc.system import run, chmod_file
from archivebox.misc.util import enforce_types, is_static_file, dedupe
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.logging_util import TimedProgress
from archivebox.misc.logging_util import TimedProgress
from .config import YTDLP_CONFIG
from .binaries import YTDLP_BINARY

View file

@ -19,7 +19,7 @@ from archivebox.config import CONSTANTS
from archivebox.config.paths import get_or_create_working_tmp_dir
from archivebox.config.permissions import ARCHIVEBOX_USER
from archivebox.misc.logging import STDERR
from archivebox.logging_util import pretty_path
from archivebox.misc.logging_util import pretty_path
LOG_FILE_NAME = "supervisord.log"
CONFIG_FILE_NAME = "supervisord.conf"