mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-24 21:23:22 +00:00
move logging_util into archivebox.misc subfolder
This commit is contained in:
parent
e469c5a344
commit
4a5d607296
44 changed files with 89 additions and 113 deletions
|
@ -246,7 +246,7 @@ def main(args: List[str] | Omitted=OMITTED, stdin: IO | Omitted=OMITTED, pwd: st
|
||||||
command.subcommand = 'help'
|
command.subcommand = 'help'
|
||||||
|
|
||||||
if command.subcommand not in ('version',):
|
if command.subcommand not in ('version',):
|
||||||
from ..logging_util import log_cli_command
|
from archivebox.misc.logging_util import log_cli_command
|
||||||
|
|
||||||
log_cli_command(
|
log_cli_command(
|
||||||
subcommand=command.subcommand,
|
subcommand=command.subcommand,
|
||||||
|
|
|
@ -18,8 +18,8 @@ from archivebox.config.common import ARCHIVING_CONFIG
|
||||||
from archivebox.config.django import setup_django
|
from archivebox.config.django import setup_django
|
||||||
from archivebox.config.permissions import USER, HOSTNAME
|
from archivebox.config.permissions import USER, HOSTNAME
|
||||||
from archivebox.misc.checks import check_data_folder
|
from archivebox.misc.checks import check_data_folder
|
||||||
|
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
|
||||||
from archivebox.parsers import PARSERS
|
from archivebox.parsers import PARSERS
|
||||||
from archivebox.logging_util import SmartFormatter, accept_stdin, stderr
|
|
||||||
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
|
|
@ -11,8 +11,8 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
|
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
|
||||||
from ..main import config
|
from ..main import config
|
||||||
from ..logging_util import SmartFormatter, accept_stdin
|
|
||||||
|
|
||||||
|
|
||||||
@docstring(config.__doc__)
|
@docstring(config.__doc__)
|
||||||
|
|
|
@ -9,9 +9,9 @@ from pathlib import Path
|
||||||
from typing import Optional, List, IO
|
from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..main import help
|
from ..main import help
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
|
||||||
|
|
||||||
|
|
||||||
@docstring(help.__doc__)
|
@docstring(help.__doc__)
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import init
|
from ..main import init
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import install
|
from ..main import install
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -8,8 +8,9 @@ import argparse
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, List, IO
|
from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
|
from archivebox.misc.util import docstring
|
||||||
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin, stderr
|
||||||
from ..main import list_all
|
from ..main import list_all
|
||||||
from ..index import (
|
from ..index import (
|
||||||
LINK_FILTERS,
|
LINK_FILTERS,
|
||||||
|
@ -24,7 +25,6 @@ from ..index import (
|
||||||
get_corrupted_folders,
|
get_corrupted_folders,
|
||||||
get_unrecognized_folders,
|
get_unrecognized_folders,
|
||||||
)
|
)
|
||||||
from ..logging_util import SmartFormatter, reject_stdin, stderr
|
|
||||||
|
|
||||||
|
|
||||||
@docstring(list_all.__doc__)
|
@docstring(list_all.__doc__)
|
||||||
|
|
|
@ -11,7 +11,7 @@ from typing import List, Optional, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, accept_stdin, stderr
|
from archivebox.misc.logging_util import SmartFormatter, accept_stdin, stderr
|
||||||
from ..main import oneshot
|
from ..main import oneshot
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, accept_stdin
|
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
|
||||||
from ..main import remove
|
from ..main import remove
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import schedule
|
from ..main import schedule
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -11,7 +11,7 @@ from typing import Optional, List, IO
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from archivebox.config.common import SERVER_CONFIG
|
from archivebox.config.common import SERVER_CONFIG
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import server
|
from ..main import server
|
||||||
|
|
||||||
@docstring(server.__doc__)
|
@docstring(server.__doc__)
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import shell
|
from ..main import shell
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
from archivebox.misc.util import docstring
|
from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR
|
from archivebox.config import DATA_DIR
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
from ..main import status
|
from ..main import status
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -21,7 +21,7 @@ from archivebox.index import (
|
||||||
get_corrupted_folders,
|
get_corrupted_folders,
|
||||||
get_unrecognized_folders,
|
get_unrecognized_folders,
|
||||||
)
|
)
|
||||||
from archivebox.logging_util import SmartFormatter, accept_stdin
|
from archivebox.misc.logging_util import SmartFormatter, accept_stdin
|
||||||
# from ..main import update
|
# from ..main import update
|
||||||
|
|
||||||
def update():
|
def update():
|
||||||
|
|
|
@ -10,7 +10,7 @@ from typing import Optional, List, IO
|
||||||
|
|
||||||
# from archivebox.misc.util import docstring
|
# from archivebox.misc.util import docstring
|
||||||
from archivebox.config import DATA_DIR, VERSION
|
from archivebox.config import DATA_DIR, VERSION
|
||||||
from ..logging_util import SmartFormatter, reject_stdin
|
from archivebox.misc.logging_util import SmartFormatter, reject_stdin
|
||||||
|
|
||||||
|
|
||||||
# @docstring(version.__doc__)
|
# @docstring(version.__doc__)
|
||||||
|
|
|
@ -21,7 +21,7 @@ from collections.abc import Mapping
|
||||||
|
|
||||||
from benedict import benedict
|
from benedict import benedict
|
||||||
|
|
||||||
from ..misc.logging import DEFAULT_CLI_COLORS
|
from archivebox.misc.logging import DEFAULT_CLI_COLORS
|
||||||
|
|
||||||
from .paths import (
|
from .paths import (
|
||||||
PACKAGE_DIR,
|
PACKAGE_DIR,
|
||||||
|
|
|
@ -10,10 +10,10 @@ from rich.console import Console
|
||||||
|
|
||||||
import django
|
import django
|
||||||
|
|
||||||
from . import CONSTANTS
|
from archivebox.misc import logging
|
||||||
|
|
||||||
|
from . import CONSTANTS
|
||||||
from .common import SHELL_CONFIG
|
from .common import SHELL_CONFIG
|
||||||
from ..misc import logging
|
|
||||||
|
|
||||||
|
|
||||||
if not SHELL_CONFIG.USE_COLOR:
|
if not SHELL_CONFIG.USE_COLOR:
|
||||||
|
|
|
@ -114,7 +114,7 @@ def dir_is_writable(dir_path: Path, uid: int | None = None, gid: int | None = No
|
||||||
|
|
||||||
def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
|
def assert_dir_can_contain_unix_sockets(dir_path: Path) -> bool:
|
||||||
"""Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
|
"""Check if a given directory can contain unix sockets (e.g. /tmp/supervisord.sock)"""
|
||||||
from archivebox.logging_util import pretty_path
|
from archivebox.misc.logging_util import pretty_path
|
||||||
|
|
||||||
try:
|
try:
|
||||||
socket_path = str(dir_path / '.test_socket.sock')
|
socket_path = str(dir_path / '.test_socket.sock')
|
||||||
|
|
|
@ -17,9 +17,8 @@ from archivebox.config import DATA_DIR
|
||||||
from archivebox.config.common import SERVER_CONFIG
|
from archivebox.config.common import SERVER_CONFIG
|
||||||
from archivebox.misc.util import htmldecode, urldecode
|
from archivebox.misc.util import htmldecode, urldecode
|
||||||
from archivebox.misc.paginators import AccelleratedPaginator
|
from archivebox.misc.paginators import AccelleratedPaginator
|
||||||
|
from archivebox.misc.logging_util import printable_filesize
|
||||||
from archivebox.search.admin import SearchResultsAdminMixin
|
from archivebox.search.admin import SearchResultsAdminMixin
|
||||||
|
|
||||||
from archivebox.logging_util import printable_filesize
|
|
||||||
from archivebox.index.html import snapshot_icons
|
from archivebox.index.html import snapshot_icons
|
||||||
from archivebox.extractors import archive_links
|
from archivebox.extractors import archive_links
|
||||||
from archivebox.main import remove
|
from archivebox.main import remove
|
||||||
|
|
|
@ -11,8 +11,7 @@ import pydantic
|
||||||
import django.template
|
import django.template
|
||||||
|
|
||||||
from archivebox.config import CONSTANTS
|
from archivebox.config import CONSTANTS
|
||||||
|
from archivebox.misc.logging import IS_TTY
|
||||||
from ..misc.logging import IS_TTY
|
|
||||||
|
|
||||||
|
|
||||||
IGNORABLE_URL_PATTERNS = [
|
IGNORABLE_URL_PATTERNS = [
|
||||||
|
|
|
@ -31,8 +31,7 @@ from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION
|
||||||
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
|
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
|
||||||
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
|
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
|
||||||
from archivebox.misc.serve_static import serve_static_with_byterange_support
|
from archivebox.misc.serve_static import serve_static_with_byterange_support
|
||||||
|
from archivebox.misc.logging_util import printable_filesize
|
||||||
from archivebox.logging_util import printable_filesize
|
|
||||||
from archivebox.search import query_search_index
|
from archivebox.search import query_search_index
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -17,7 +17,7 @@ from ..index import (
|
||||||
write_link_details,
|
write_link_details,
|
||||||
)
|
)
|
||||||
from archivebox.misc.util import enforce_types
|
from archivebox.misc.util import enforce_types
|
||||||
from ..logging_util import (
|
from archivebox.misc.logging_util import (
|
||||||
log_archiving_started,
|
log_archiving_started,
|
||||||
log_archiving_paused,
|
log_archiving_paused,
|
||||||
log_archiving_finished,
|
log_archiving_finished,
|
||||||
|
|
|
@ -12,13 +12,12 @@ from urllib.parse import urlparse
|
||||||
from django.db.models import QuerySet, Q
|
from django.db.models import QuerySet, Q
|
||||||
|
|
||||||
|
|
||||||
from archivebox.misc.logging import stderr
|
|
||||||
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
|
|
||||||
|
|
||||||
from archivebox.config import DATA_DIR, CONSTANTS
|
from archivebox.config import DATA_DIR, CONSTANTS
|
||||||
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
|
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG, SEARCH_BACKEND_CONFIG
|
||||||
|
from archivebox.misc.util import scheme, enforce_types, ExtendedEncoder
|
||||||
from ..logging_util import (
|
from archivebox.misc.logging import stderr
|
||||||
|
from archivebox.misc.logging_util import (
|
||||||
TimedProgress,
|
TimedProgress,
|
||||||
log_indexing_process_started,
|
log_indexing_process_started,
|
||||||
log_indexing_process_finished,
|
log_indexing_process_finished,
|
||||||
|
|
|
@ -21,9 +21,9 @@ from archivebox.misc.util import (
|
||||||
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
|
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
|
||||||
from archivebox.config.common import SERVER_CONFIG
|
from archivebox.config.common import SERVER_CONFIG
|
||||||
from archivebox.config.version import get_COMMIT_HASH
|
from archivebox.config.version import get_COMMIT_HASH
|
||||||
|
from archivebox.misc.logging_util import printable_filesize
|
||||||
|
|
||||||
from .schema import Link
|
from .schema import Link
|
||||||
from ..logging_util import printable_filesize
|
|
||||||
|
|
||||||
MAIN_INDEX_TEMPLATE = 'static_index.html'
|
MAIN_INDEX_TEMPLATE = 'static_index.html'
|
||||||
MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'
|
MINIMAL_INDEX_TEMPLATE = 'minimal_index.html'
|
||||||
|
|
|
@ -18,20 +18,30 @@ from abx_pkg import Binary
|
||||||
|
|
||||||
import abx
|
import abx
|
||||||
import archivebox
|
import archivebox
|
||||||
|
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
|
||||||
|
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
|
||||||
|
from archivebox.config.permissions import SudoPermission, IN_DOCKER
|
||||||
|
from archivebox.config.collection import write_config_file, load_all_config, get_real_name
|
||||||
from archivebox.misc.checks import check_data_folder
|
from archivebox.misc.checks import check_data_folder
|
||||||
from archivebox.misc.util import enforce_types # type: ignore
|
from archivebox.misc.util import enforce_types # type: ignore
|
||||||
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
|
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
|
||||||
from archivebox.misc.system import run as run_shell
|
from archivebox.misc.system import run as run_shell
|
||||||
from archivebox.misc.logging import stderr, hint
|
from archivebox.misc.logging import stderr, hint
|
||||||
from archivebox.config import CONSTANTS, VERSION, DATA_DIR, ARCHIVE_DIR
|
from archivebox.misc.logging_util import (
|
||||||
from archivebox.config.common import SHELL_CONFIG, SEARCH_BACKEND_CONFIG, STORAGE_CONFIG, SERVER_CONFIG, ARCHIVING_CONFIG
|
TimedProgress,
|
||||||
from archivebox.config.permissions import SudoPermission, IN_DOCKER
|
log_importing_started,
|
||||||
from archivebox.config.collection import (
|
log_crawl_started,
|
||||||
write_config_file,
|
log_removal_started,
|
||||||
load_all_config,
|
log_removal_finished,
|
||||||
get_real_name,
|
log_list_started,
|
||||||
|
log_list_finished,
|
||||||
|
printable_config,
|
||||||
|
printable_folders,
|
||||||
|
printable_filesize,
|
||||||
|
printable_folder_status,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
from .cli import (
|
from .cli import (
|
||||||
CLI_SUBCOMMANDS,
|
CLI_SUBCOMMANDS,
|
||||||
run_subcommand,
|
run_subcommand,
|
||||||
|
@ -78,19 +88,6 @@ from .index.sql import (
|
||||||
from .index.html import generate_index_from_links
|
from .index.html import generate_index_from_links
|
||||||
from .index.csv import links_to_csv
|
from .index.csv import links_to_csv
|
||||||
from .extractors import archive_links, archive_link, ignore_methods
|
from .extractors import archive_links, archive_link, ignore_methods
|
||||||
from .logging_util import (
|
|
||||||
TimedProgress,
|
|
||||||
log_importing_started,
|
|
||||||
log_crawl_started,
|
|
||||||
log_removal_started,
|
|
||||||
log_removal_finished,
|
|
||||||
log_list_started,
|
|
||||||
log_list_finished,
|
|
||||||
printable_config,
|
|
||||||
printable_folders,
|
|
||||||
printable_filesize,
|
|
||||||
printable_folder_status,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@enforce_types
|
@enforce_types
|
||||||
|
|
|
@ -157,9 +157,9 @@ def check_data_dir_permissions():
|
||||||
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
|
def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
|
||||||
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
|
from archivebox.config.paths import assert_dir_can_contain_unix_sockets, dir_is_writable, get_or_create_working_tmp_dir
|
||||||
from archivebox.misc.logging import STDERR
|
from archivebox.misc.logging import STDERR
|
||||||
|
from archivebox.misc.logging_util import pretty_path
|
||||||
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
||||||
from archivebox.config.common import STORAGE_CONFIG
|
from archivebox.config.common import STORAGE_CONFIG
|
||||||
from archivebox.logging_util import pretty_path
|
|
||||||
|
|
||||||
tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR
|
tmp_dir = tmp_dir or STORAGE_CONFIG.TMP_DIR
|
||||||
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
|
socket_file = tmp_dir.absolute().resolve() / "supervisord.sock"
|
||||||
|
@ -204,9 +204,9 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
|
||||||
import archivebox
|
import archivebox
|
||||||
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
|
||||||
from archivebox.misc.logging import STDERR
|
from archivebox.misc.logging import STDERR
|
||||||
|
from archivebox.misc.logging_util import pretty_path
|
||||||
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
|
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
|
||||||
from archivebox.config.common import STORAGE_CONFIG
|
from archivebox.config.common import STORAGE_CONFIG
|
||||||
from archivebox.logging_util import pretty_path
|
|
||||||
|
|
||||||
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
|
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,7 @@ from dataclasses import dataclass
|
||||||
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
|
from typing import Any, Optional, List, Dict, Union, IO, TYPE_CHECKING
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from .index.schema import Link, ArchiveResult
|
from ..index.schema import Link, ArchiveResult
|
||||||
|
|
||||||
from rich import print
|
from rich import print
|
||||||
from rich.panel import Panel
|
from rich.panel import Panel
|
||||||
|
@ -303,14 +303,14 @@ def log_archiving_started(num_links: int, resume: Optional[float]=None):
|
||||||
print()
|
print()
|
||||||
if resume:
|
if resume:
|
||||||
print('[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]'.format(
|
print('[green][▶] [{}] Resuming archive updating for {} pages starting from {}...[/]'.format(
|
||||||
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
|
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
|
||||||
num_links,
|
num_links,
|
||||||
resume,
|
resume,
|
||||||
))
|
))
|
||||||
else:
|
else:
|
||||||
print('[green][▶] [{}] Starting archiving of {} snapshots in index...[/]'.format(
|
print('[green][▶] [{}] Starting archiving of {} snapshots in index...[/]'.format(
|
||||||
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
|
start_ts.strftime('%Y-%m-%d %H:%M:%S'),
|
||||||
num_links,
|
num_links,
|
||||||
))
|
))
|
||||||
|
|
||||||
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
|
def log_archiving_paused(num_links: int, idx: int, timestamp: str):
|
||||||
|
@ -400,8 +400,9 @@ def log_archive_method_started(method: str):
|
||||||
|
|
||||||
|
|
||||||
def log_archive_method_finished(result: "ArchiveResult"):
|
def log_archive_method_finished(result: "ArchiveResult"):
|
||||||
"""quote the argument with whitespace in a command so the user can
|
"""
|
||||||
copy-paste the outputted string directly to run the cmd
|
quote the argument with whitespace in a command so the user can
|
||||||
|
copy-paste the outputted string directly to run the cmd
|
||||||
"""
|
"""
|
||||||
# Prettify CMD string and make it safe to copy-paste by quoting arguments
|
# Prettify CMD string and make it safe to copy-paste by quoting arguments
|
||||||
quoted_cmd = ' '.join(
|
quoted_cmd = ' '.join(
|
||||||
|
@ -467,7 +468,7 @@ def log_list_started(filter_patterns: Optional[List[str]], filter_type: str):
|
||||||
print(' {}'.format(' '.join(filter_patterns or ())))
|
print(' {}'.format(' '.join(filter_patterns or ())))
|
||||||
|
|
||||||
def log_list_finished(links):
|
def log_list_finished(links):
|
||||||
from .index.csv import links_to_csv
|
from ..index.csv import links_to_csv
|
||||||
print()
|
print()
|
||||||
print('---------------------------------------------------------------------------------------------------')
|
print('---------------------------------------------------------------------------------------------------')
|
||||||
print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
|
print(links_to_csv(links, cols=['timestamp', 'is_archived', 'num_outputs', 'url'], header=True, ljust=16, separator=' | '))
|
||||||
|
@ -545,8 +546,7 @@ def printable_filesize(num_bytes: Union[int, float]) -> str:
|
||||||
|
|
||||||
|
|
||||||
@enforce_types
|
@enforce_types
|
||||||
def printable_folders(folders: Dict[str, Optional["Link"]],
|
def printable_folders(folders: Dict[str, Optional["Link"]], with_headers: bool=False) -> str:
|
||||||
with_headers: bool=False) -> str:
|
|
||||||
return '\n'.join(
|
return '\n'.join(
|
||||||
f'{folder} {link and link.url} "{link and link.title}"'
|
f'{folder} {link and link.url} "{link and link.title}"'
|
||||||
for folder, link in folders.items()
|
for folder, link in folders.items()
|
|
@ -17,6 +17,7 @@ from archivebox.config import DATA_DIR, CONSTANTS
|
||||||
from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG
|
from archivebox.config.common import SHELL_CONFIG, ARCHIVING_CONFIG
|
||||||
from archivebox.misc.system import atomic_write
|
from archivebox.misc.system import atomic_write
|
||||||
from archivebox.misc.logging import stderr, hint
|
from archivebox.misc.logging import stderr, hint
|
||||||
|
from archivebox.misc.logging_util import TimedProgress, log_source_saved
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import (
|
||||||
basename,
|
basename,
|
||||||
htmldecode,
|
htmldecode,
|
||||||
|
@ -25,7 +26,6 @@ from archivebox.misc.util import (
|
||||||
)
|
)
|
||||||
|
|
||||||
from ..index.schema import Link
|
from ..index.schema import Link
|
||||||
from ..logging_util import TimedProgress, log_source_saved
|
|
||||||
|
|
||||||
from . import pocket_api
|
from . import pocket_api
|
||||||
from . import readwise_reader_api
|
from . import readwise_reader_api
|
||||||
|
|
|
@ -5,8 +5,8 @@ from pathlib import Path
|
||||||
from typing import Optional, List, Dict, Tuple
|
from typing import Optional, List, Dict, Tuple
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
||||||
|
|
||||||
|
|
|
@ -10,9 +10,9 @@ from abx_plugin_default_binproviders import env
|
||||||
|
|
||||||
from archivebox.config import CONSTANTS
|
from archivebox.config import CONSTANTS
|
||||||
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
|
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
|
||||||
from archivebox.misc.logging import STDERR
|
|
||||||
from archivebox.misc.util import dedupe
|
from archivebox.misc.util import dedupe
|
||||||
from archivebox.logging_util import pretty_path
|
from archivebox.misc.logging import STDERR
|
||||||
|
from archivebox.misc.logging_util import pretty_path
|
||||||
|
|
||||||
|
|
||||||
CHROMIUM_BINARY_NAMES_LINUX = [
|
CHROMIUM_BINARY_NAMES_LINUX = [
|
||||||
|
|
|
@ -5,11 +5,8 @@ from typing import Optional
|
||||||
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
from archivebox.misc.system import run, chmod_file, atomic_write
|
from archivebox.misc.system import run, chmod_file, atomic_write
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
enforce_types,
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
is_static_file,
|
|
||||||
)
|
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
|
|
||||||
from .config import CHROME_CONFIG
|
from .config import CHROME_CONFIG
|
||||||
from .binaries import CHROME_BINARY
|
from .binaries import CHROME_BINARY
|
||||||
|
|
|
@ -4,12 +4,9 @@ from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
enforce_types,
|
|
||||||
is_static_file,
|
|
||||||
)
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
|
|
||||||
from .config import CHROME_CONFIG
|
from .config import CHROME_CONFIG
|
||||||
from .binaries import CHROME_BINARY
|
from .binaries import CHROME_BINARY
|
||||||
|
|
|
@ -5,8 +5,8 @@ from typing import Optional
|
||||||
|
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import enforce_types, is_static_file
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
|
|
||||||
from .config import CHROME_CONFIG
|
from .config import CHROME_CONFIG
|
||||||
from .binaries import CHROME_BINARY
|
from .binaries import CHROME_BINARY
|
||||||
|
|
|
@ -4,14 +4,10 @@ from pathlib import Path
|
||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
|
||||||
from archivebox.misc.system import atomic_write
|
from archivebox.misc.system import atomic_write
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import enforce_types, get_headers, dedupe
|
||||||
enforce_types,
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
get_headers,
|
|
||||||
dedupe,
|
|
||||||
)
|
|
||||||
|
|
||||||
from .binaries import CURL_BINARY
|
from .binaries import CURL_BINARY
|
||||||
from .config import CURL_CONFIG
|
from .config import CURL_CONFIG
|
||||||
|
|
|
@ -4,8 +4,8 @@ from pathlib import Path
|
||||||
|
|
||||||
from archivebox.misc.system import chmod_file, run
|
from archivebox.misc.system import chmod_file, run
|
||||||
from archivebox.misc.util import enforce_types, domain, dedupe
|
from archivebox.misc.util import enforce_types, domain, dedupe
|
||||||
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
|
|
||||||
from abx_plugin_curl.config import CURL_CONFIG
|
from abx_plugin_curl.config import CURL_CONFIG
|
||||||
from abx_plugin_curl.binaries import CURL_BINARY
|
from abx_plugin_curl.binaries import CURL_BINARY
|
||||||
|
|
|
@ -13,7 +13,7 @@ from archivebox.misc.util import (
|
||||||
without_query,
|
without_query,
|
||||||
without_fragment,
|
without_fragment,
|
||||||
)
|
)
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
|
|
||||||
from abx_plugin_git.config import GIT_CONFIG
|
from abx_plugin_git.config import GIT_CONFIG
|
||||||
|
|
|
@ -9,7 +9,7 @@ from archivebox.config import VERSION
|
||||||
from archivebox.config.common import ARCHIVING_CONFIG
|
from archivebox.config.common import ARCHIVING_CONFIG
|
||||||
from archivebox.misc.system import atomic_write
|
from archivebox.misc.system import atomic_write
|
||||||
from archivebox.misc.util import enforce_types, is_static_file
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
||||||
|
|
||||||
from abx_plugin_title.extractor import get_html
|
from abx_plugin_title.extractor import get_html
|
||||||
|
|
|
@ -6,13 +6,10 @@ from subprocess import CompletedProcess
|
||||||
from typing import Optional, List
|
from typing import Optional, List
|
||||||
import json
|
import json
|
||||||
|
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
||||||
from archivebox.misc.system import run, atomic_write
|
from archivebox.misc.system import run, atomic_write
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
enforce_types,
|
|
||||||
is_static_file,
|
|
||||||
)
|
|
||||||
from .config import MERCURY_CONFIG
|
from .config import MERCURY_CONFIG
|
||||||
from .binaries import MERCURY_BINARY
|
from .binaries import MERCURY_BINARY
|
||||||
|
|
||||||
|
|
|
@ -1,15 +1,15 @@
|
||||||
__package__ = 'abx_plugin_readability'
|
__package__ = 'abx_plugin_readability'
|
||||||
|
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
import json
|
|
||||||
|
|
||||||
from archivebox.misc.system import run, atomic_write
|
from archivebox.misc.system import run, atomic_write
|
||||||
from archivebox.misc.util import enforce_types, is_static_file
|
from archivebox.misc.util import enforce_types, is_static_file
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
|
|
||||||
from abx_plugin_title.extractor import get_html
|
from abx_plugin_title.extractor import get_html
|
||||||
|
|
||||||
from .config import READABILITY_CONFIG
|
from .config import READABILITY_CONFIG
|
||||||
|
|
|
@ -1,17 +1,17 @@
|
||||||
__package__ = 'abx_plugin_singlefile'
|
__package__ = 'abx_plugin_singlefile'
|
||||||
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from typing import Optional
|
|
||||||
import json
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveError
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
|
|
||||||
from abx_plugin_chrome.config import CHROME_CONFIG
|
from abx_plugin_chrome.config import CHROME_CONFIG
|
||||||
from abx_plugin_chrome.binaries import CHROME_BINARY
|
from abx_plugin_chrome.binaries import CHROME_BINARY
|
||||||
|
|
||||||
from .config import SINGLEFILE_CONFIG
|
from .config import SINGLEFILE_CONFIG
|
||||||
from .binaries import SINGLEFILE_BINARY
|
from .binaries import SINGLEFILE_BINARY
|
||||||
|
|
||||||
|
|
|
@ -6,13 +6,9 @@ from pathlib import Path
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import enforce_types, download_url, htmldecode, dedupe)
|
||||||
enforce_types,
|
|
||||||
download_url,
|
|
||||||
htmldecode,
|
|
||||||
dedupe,
|
|
||||||
)
|
|
||||||
from abx_plugin_curl.config import CURL_CONFIG
|
from abx_plugin_curl.config import CURL_CONFIG
|
||||||
from abx_plugin_curl.binaries import CURL_BINARY
|
from abx_plugin_curl.binaries import CURL_BINARY
|
||||||
|
|
||||||
|
|
|
@ -2,13 +2,12 @@ __package__ = 'abx_plugin_wget'
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
from pathlib import Path
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from archivebox.logging_util import TimedProgress
|
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import (
|
from archivebox.misc.util import (
|
||||||
enforce_types,
|
enforce_types,
|
||||||
|
@ -19,6 +18,7 @@ from archivebox.misc.util import (
|
||||||
urldecode,
|
urldecode,
|
||||||
dedupe,
|
dedupe,
|
||||||
)
|
)
|
||||||
|
|
||||||
from .config import WGET_CONFIG
|
from .config import WGET_CONFIG
|
||||||
from .binaries import WGET_BINARY
|
from .binaries import WGET_BINARY
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ from typing import Optional
|
||||||
from archivebox.misc.system import run, chmod_file
|
from archivebox.misc.system import run, chmod_file
|
||||||
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
from archivebox.misc.util import enforce_types, is_static_file, dedupe
|
||||||
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
from archivebox.index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
|
||||||
from archivebox.logging_util import TimedProgress
|
from archivebox.misc.logging_util import TimedProgress
|
||||||
|
|
||||||
from .config import YTDLP_CONFIG
|
from .config import YTDLP_CONFIG
|
||||||
from .binaries import YTDLP_BINARY
|
from .binaries import YTDLP_BINARY
|
||||||
|
|
|
@ -19,7 +19,7 @@ from archivebox.config import CONSTANTS
|
||||||
from archivebox.config.paths import get_or_create_working_tmp_dir
|
from archivebox.config.paths import get_or_create_working_tmp_dir
|
||||||
from archivebox.config.permissions import ARCHIVEBOX_USER
|
from archivebox.config.permissions import ARCHIVEBOX_USER
|
||||||
from archivebox.misc.logging import STDERR
|
from archivebox.misc.logging import STDERR
|
||||||
from archivebox.logging_util import pretty_path
|
from archivebox.misc.logging_util import pretty_path
|
||||||
|
|
||||||
LOG_FILE_NAME = "supervisord.log"
|
LOG_FILE_NAME = "supervisord.log"
|
||||||
CONFIG_FILE_NAME = "supervisord.conf"
|
CONFIG_FILE_NAME = "supervisord.conf"
|
||||||
|
|
Loading…
Reference in a new issue