Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-21 19:53:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

move abx plugins inside vendor dir

Browse source
This commit is contained in:
Nick Sweeting 2024-10-28 04:07:35 -07:00
parent 5d9a32c364
commit b3c1cb716e
No known key found for this signature in database
243 changed files with 2162 additions and 2709 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
archivebox/__init__.py
View file

@ -15,7 +15,7 @@ import os
import sys
from pathlib import Path
from typing import cast
ASCII_LOGO = """
█████╗ ██████╗ ██████╗██╗ ██╗██╗██╗ ██╗███████╗ ██████╗ ██████╗ ██╗ ██╗
██╔══██╗██╔══██╗██╔════╝██║ ██║██║██║ ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
@ -52,6 +52,50 @@ load_vendored_libs()
# print('DONE LOADING VENDORED LIBRARIES')
import abx # noqa
import abx_spec_archivebox # noqa
import abx_spec_config # noqa
import abx_spec_pydantic_pkgr # noqa
import abx_spec_django # noqa
import abx_spec_searchbackend # noqa
abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
abx.pm.register(abx_spec_config.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
abx.pm.register(abx_spec_django.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
pm = abx.pm
# Load all installed ABX-compatible plugins
ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
# Load all ArchiveBox-specific plugins
ARCHIVEBOX_BUILTIN_PLUGINS = {
'config': PACKAGE_DIR / 'config',
'core': PACKAGE_DIR / 'core',
# 'search': PACKAGE_DIR / 'search',
# 'core': PACKAGE_DIR / 'core',
}
# Load all user-defined ArchiveBox plugins
USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
# Merge all plugins together
ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
# Load ArchiveBox plugins
LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
from .config.constants import CONSTANTS # noqa
from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa
from .config.version import VERSION # noqa

30
archivebox/config/__init__.py
View file

@ -1,4 +1,5 @@
__package__ = 'archivebox.config'
__package__ = 'config'
__order__ = 200
from .paths import (
PACKAGE_DIR, # noqa
@ -9,30 +10,3 @@ from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHI
from .version import VERSION # noqa
import abx
# @abx.hookimpl
# def get_INSTALLED_APPS():
# return ['config']
@abx.hookimpl
def get_CONFIG():
from .common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
'SHELL_CONFIG': SHELL_CONFIG,
'STORAGE_CONFIG': STORAGE_CONFIG,
'GENERAL_CONFIG': GENERAL_CONFIG,
'SERVER_CONFIG': SERVER_CONFIG,
'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
}

18
archivebox/config/configfile.py → archivebox/config/collection.py
View file

@ -9,6 +9,8 @@ from configparser import ConfigParser
from benedict import benedict
import archivebox
from archivebox.config.constants import CONSTANTS
from archivebox.misc.logging import stderr
@ -16,9 +18,9 @@ from archivebox.misc.logging import stderr
def get_real_name(key: str) -> str:
"""get the up-to-date canonical name for a given old alias or current key"""
from django.conf import settings
CONFIGS = archivebox.pm.hook.get_CONFIGS()
for section in settings.CONFIGS.values():
for section in CONFIGS.values():
try:
return section.aliases[key]
except KeyError:
@ -115,17 +117,15 @@ def load_config_file() -> Optional[benedict]:
def section_for_key(key: str) -> Any:
from django.conf import settings
for config_section in settings.CONFIGS.values():
for config_section in archivebox.pm.hook.get_CONFIGS().values():
if hasattr(config_section, key):
return config_section
return None
raise ValueError(f'No config section found for key: {key}')
def write_config_file(config: Dict[str, str]) -> benedict:
"""load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
import abx.archivebox.reads
from archivebox.misc.system import atomic_write
CONFIG_HEADER = (
@ -175,7 +175,7 @@ def write_config_file(config: Dict[str, str]) -> benedict:
updated_config = {}
try:
# validate the updated_config by attempting to re-parse it
updated_config = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
updated_config = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
except BaseException: # lgtm [py/catch-base-exception]
# something went horribly wrong, revert to the previous version
with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
@ -233,11 +233,11 @@ def load_config(defaults: Dict[str, Any],
return benedict(extended_config)
def load_all_config():
import abx.archivebox.reads
import abx
flat_config = benedict()
for config_section in abx.archivebox.reads.get_CONFIGS().values():
for config_section in abx.pm.hook.get_CONFIGS().values():
config_section.__init__()
flat_config.update(config_section.model_dump())

11
archivebox/config/common.py
View file

@ -7,10 +7,10 @@ from typing import Dict, Optional, List
from pathlib import Path
from rich import print
from pydantic import Field, field_validator, computed_field
from pydantic import Field, field_validator
from django.utils.crypto import get_random_string
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
from .constants import CONSTANTS
from .version import get_COMMIT_HASH, get_BUILD_TIME
@ -31,22 +31,19 @@ class ShellConfig(BaseConfigSet):
ANSI: Dict[str, str] = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
VERSIONS_AVAILABLE: bool = False # .check_for_update.get_versions_available_on_github(c)},
CAN_UPGRADE: bool = False # .check_for_update.can_upgrade(c)},
# VERSIONS_AVAILABLE: bool = False # .check_for_update.get_versions_available_on_github(c)},
# CAN_UPGRADE: bool = False # .check_for_update.can_upgrade(c)},
@computed_field
@property
def TERM_WIDTH(self) -> int:
if not self.IS_TTY:
return 200
return shutil.get_terminal_size((140, 10)).columns
@computed_field
@property
def COMMIT_HASH(self) -> Optional[str]:
return get_COMMIT_HASH()
@computed_field
@property
def BUILD_TIME(self) -> str:
return get_BUILD_TIME()

2
archivebox/config/django.py
View file

@ -97,7 +97,7 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
except Exception as e:
bump_startup_progress_bar(advance=1000)
is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version', 'init'))
is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version'))
if not is_using_meta_cmd:
# show error message to user only if they're not running a meta command / just trying to get help
STDERR.print()

28
archivebox/config/views.py
View file

@ -14,8 +14,8 @@ from django.utils.html import format_html, mark_safe
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
import abx.archivebox.reads
import abx
import archivebox
from archivebox.config import CONSTANTS
from archivebox.misc.util import parse_date
@ -65,7 +65,7 @@ def obj_to_yaml(obj: Any, indent: int=0) -> str:
@render_with_table_view
def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
rows = {
@ -81,12 +81,11 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
relevant_configs = {
key: val
for key, val in settings.FLAT_CONFIG.items()
for key, val in FLAT_CONFIG.items()
if '_BINARY' in key or '_VERSION' in key
}
for plugin_id, plugin in abx.archivebox.reads.get_PLUGINS().items():
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
if not plugin.hooks.get('get_BINARIES'):
continue
@ -131,17 +130,16 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
binary = None
plugin = None
for plugin_id in abx.archivebox.reads.get_PLUGINS().keys():
loaded_plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
try:
for loaded_binary in loaded_plugin.hooks.get_BINARIES().values():
for loaded_binary in plugin['hooks'].get_BINARIES().values():
if loaded_binary.name == key:
binary = loaded_binary
plugin = loaded_plugin
plugin = plugin
# break # last write wins
except Exception as e:
print(e)
@ -161,7 +159,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
"name": binary.name,
"description": binary.abspath,
"fields": {
'plugin': plugin.package,
'plugin': plugin['package'],
'binprovider': binary.loaded_binprovider,
'abspath': binary.loaded_abspath,
'version': binary.loaded_version,
@ -215,9 +213,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
return color
return 'black'
for plugin_id in settings.PLUGINS.keys():
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
plugin.hooks.get_BINPROVIDERS = plugin.hooks.get('get_BINPROVIDERS', lambda: {})
plugin.hooks.get_BINARIES = plugin.hooks.get('get_BINARIES', lambda: {})
plugin.hooks.get_CONFIG = plugin.hooks.get('get_CONFIG', lambda: {})
@ -263,7 +259,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert plugin_id, f'Could not find a plugin matching the specified name: {key}'
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
plugin = abx.get_plugin(plugin_id)
return ItemContext(
slug=key,

29
archivebox/core/__init__.py
View file

@ -1,2 +1,31 @@
__package__ = 'archivebox.core'
import abx
@abx.hookimpl
def register_admin(admin_site):
"""Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
from core.admin import register_admin
register_admin(admin_site)
@abx.hookimpl
def get_CONFIG():
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
'SHELL_CONFIG': SHELL_CONFIG,
'STORAGE_CONFIG': STORAGE_CONFIG,
'GENERAL_CONFIG': GENERAL_CONFIG,
'SERVER_CONFIG': SERVER_CONFIG,
'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
}

4
archivebox/core/admin_site.py
View file

@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
from django.contrib import admin
import abx.django.use
import archivebox
class ArchiveBoxAdmin(admin.AdminSite):
site_header = 'ArchiveBox'
@ -37,6 +37,6 @@ def register_admin_site():
sites.site = archivebox_admin
# register all plugins admin classes
abx.django.use.register_admin(archivebox_admin)
archivebox.pm.hook.register_admin(admin_site=archivebox_admin)
return archivebox_admin

13
archivebox/core/apps.py
View file

@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
from django.apps import AppConfig
import abx
import archivebox
class CoreConfig(AppConfig):
@ -10,16 +10,11 @@ class CoreConfig(AppConfig):
def ready(self):
"""Register the archivebox.core.admin_site as the main django admin site"""
from django.conf import settings
archivebox.pm.hook.ready(settings=settings)
from core.admin_site import register_admin_site
register_admin_site()
abx.pm.hook.ready()
@abx.hookimpl
def register_admin(admin_site):
"""Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
from core.admin import register_admin
register_admin(admin_site)

25
archivebox/core/settings.py
View file

@ -9,10 +9,12 @@ from pathlib import Path
from django.utils.crypto import get_random_string
import abx
import archivebox
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS # noqa
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG # noqa
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@ -22,24 +24,8 @@ IS_GETTING_VERSION_OR_HELP = 'version' in sys.argv or 'help' in sys.argv or '--v
### ArchiveBox Plugin Settings
################################################################################
PLUGIN_HOOKSPECS = [
'abx_spec_django',
'abx_spec_pydantic_pkgr',
'abx_spec_config',
'abx_spec_archivebox',
]
abx.register_hookspecs(PLUGIN_HOOKSPECS)
SYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
USER_PLUGINS = abx.find_plugins_in_dir(DATA_DIR / 'user_plugins')
ALL_PLUGINS = {**SYSTEM_PLUGINS, **USER_PLUGINS}
# Load ArchiveBox plugins
abx.load_plugins(ALL_PLUGINS)
# # Load ArchiveBox config from plugins
ALL_PLUGINS = archivebox.ALL_PLUGINS
LOADED_PLUGINS = archivebox.LOADED_PLUGINS
################################################################################
### Django Core Settings
@ -101,6 +87,7 @@ INSTALLED_APPS = [
MIDDLEWARE = [
'core.middleware.TimezoneMiddleware',
'django.middleware.security.SecurityMiddleware',

5
archivebox/core/settings_logging.py
View file

@ -163,11 +163,6 @@ SETTINGS_LOGGING = {
"level": "DEBUG",
"propagate": False,
},
"plugins_extractor": {
"handlers": ["default", "logfile"],
"level": "DEBUG",
"propagate": False,
},
"httpx": {
"handlers": ["outbound_webhooks"],
"level": "INFO",

38
archivebox/core/views.py
View file

@ -21,6 +21,7 @@ from django.utils.decorators import method_decorator
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
import archivebox
from core.models import Snapshot
from core.forms import AddLinkForm
@ -32,9 +33,8 @@ from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
from archivebox.misc.serve_static import serve_static_with_byterange_support
from ..plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
from ..logging_util import printable_filesize
from ..search import query_search_index
from archivebox.logging_util import printable_filesize
from archivebox.search import query_search_index
class HomepageView(View):
@ -154,7 +154,7 @@ class SnapshotView(View):
'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'warc_path': warc_path,
'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
'SAVE_ARCHIVE_DOT_ORG': archivebox.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG,
'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
'archiveresults': sorted(archiveresults.values(), key=lambda r: all_types.index(r['name']) if r['name'] in all_types else -r['size']),
'best_result': best_result,
@ -500,21 +500,25 @@ class HealthCheckView(View):
def find_config_section(key: str) -> str:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
if key in CONSTANTS_CONFIG:
return 'CONSTANT'
matching_sections = [
section_id for section_id, section in settings.CONFIGS.items() if key in section.model_fields
section_id for section_id, section in CONFIGS.items() if key in section.model_fields
]
section = matching_sections[0] if matching_sections else 'DYNAMIC'
return section
def find_config_default(key: str) -> str:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
if key in CONSTANTS_CONFIG:
return str(CONSTANTS_CONFIG[key])
default_val = None
for config in settings.CONFIGS.values():
for config in CONFIGS.values():
if key in config.model_fields:
default_val = config.model_fields[key].default
break
@ -530,7 +534,9 @@ def find_config_default(key: str) -> str:
return default_val
def find_config_type(key: str) -> str:
for config in settings.CONFIGS.values():
CONFIGS = archivebox.pm.hook.get_CONFIGS()
for config in CONFIGS.values():
if hasattr(config, key):
type_hints = get_type_hints(config)
try:
@ -547,7 +553,8 @@ def key_is_safe(key: str) -> bool:
@render_with_table_view
def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
rows = {
@ -560,7 +567,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
# "Aliases": [],
}
for section_id, section in reversed(list(settings.CONFIGS.items())):
for section_id, section in reversed(list(CONFIGS.items())):
for key, field in section.model_fields.items():
rows['Section'].append(section_id) # section.replace('_', ' ').title().replace(' Config', '')
rows['Key'].append(ItemLink(key, key=key))
@ -570,7 +577,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
# rows['Aliases'].append(', '.join(find_config_aliases(key)))
section = 'CONSTANT'
for key in CONSTANTS_CONFIG.keys():
rows['Section'].append(section) # section.replace('_', ' ').title().replace(' Config', '')
@ -589,7 +595,9 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
# aliases = USER_CONFIG.get(key, {}).get("aliases", [])
@ -597,7 +605,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
if key in CONSTANTS_CONFIG:
section_header = mark_safe(f'[CONSTANTS] &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>')
elif key in settings.FLAT_CONFIG:
elif key in FLAT_CONFIG:
section_header = mark_safe(f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}] &nbsp; <b><code style="color: lightgray">{key}</code></b>')
else:
section_header = mark_safe(f'[DYNAMIC CONFIG] &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>')
@ -613,7 +621,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
"fields": {
'Key': key,
'Type': find_config_type(key),
'Value': settings.FLAT_CONFIG.get(key, settings.CONFIGS.get(key, None)) if key_is_safe(key) else '********',
'Value': FLAT_CONFIG.get(key, CONFIGS.get(key, None)) if key_is_safe(key) else '********',
},
"help_texts": {
'Key': mark_safe(f'''
@ -635,13 +643,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
<code>{find_config_default(key) or '↗️ See in ArchiveBox source code...'}</code>
</a>
<br/><br/>
<p style="display: {"block" if key in settings.FLAT_CONFIG else "none"}">
<p style="display: {"block" if key in FLAT_CONFIG else "none"}">
<i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
<br/><br/>
<code>archivebox config --set {key}="{
val.strip("'")
if (val := find_config_default(key)) else
(repr(settings.FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
(repr(FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
}"</code>
</p>
'''),

46
archivebox/extractors/__init__.py
View file

@ -27,43 +27,29 @@ from ..logging_util import (
log_archive_method_finished,
)
from .title import should_save_title, save_title
from .favicon import should_save_favicon, save_favicon
from .wget import should_save_wget, save_wget
from .singlefile import should_save_singlefile, save_singlefile
from .readability import should_save_readability, save_readability
from .mercury import should_save_mercury, save_mercury
from .htmltotext import should_save_htmltotext, save_htmltotext
from .pdf import should_save_pdf, save_pdf
from .screenshot import should_save_screenshot, save_screenshot
from .dom import should_save_dom, save_dom
from .git import should_save_git, save_git
from .media import should_save_media, save_media
from .archive_org import should_save_archive_dot_org, save_archive_dot_org
from .headers import should_save_headers, save_headers
ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
def get_default_archive_methods() -> List[ArchiveMethodEntry]:
# TODO: move to abx.pm.hook.get_EXTRACTORS()
return [
('favicon', should_save_favicon, save_favicon),
('headers', should_save_headers, save_headers),
('singlefile', should_save_singlefile, save_singlefile),
('pdf', should_save_pdf, save_pdf),
('screenshot', should_save_screenshot, save_screenshot),
('dom', should_save_dom, save_dom),
('wget', should_save_wget, save_wget),
# keep title, readability, and htmltotext below wget and singlefile, as they depend on them
('title', should_save_title, save_title),
('readability', should_save_readability, save_readability),
('mercury', should_save_mercury, save_mercury),
('htmltotext', should_save_htmltotext, save_htmltotext),
('git', should_save_git, save_git),
('media', should_save_media, save_media),
('archive_org', should_save_archive_dot_org, save_archive_dot_org),
# ('favicon', should_save_favicon, save_favicon),
# ('headers', should_save_headers, save_headers),
# ('singlefile', should_save_singlefile, save_singlefile),
# ('pdf', should_save_pdf, save_pdf),
# ('screenshot', should_save_screenshot, save_screenshot),
# ('dom', should_save_dom, save_dom),
# ('wget', should_save_wget, save_wget),
# # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
# ('title', should_save_title, save_title),
# ('readability', should_save_readability, save_readability),
# ('mercury', should_save_mercury, save_mercury),
# ('htmltotext', should_save_htmltotext, save_htmltotext),
# ('git', should_save_git, save_git),
# ('media', should_save_media, save_media),
# ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
]
ARCHIVE_METHODS_INDEXING_PRECEDENCE = [

11
archivebox/index/html.py
View file

@ -8,6 +8,8 @@ from typing import List, Optional, Iterator, Mapping
from django.utils.html import format_html, mark_safe # type: ignore
from django.core.cache import cache
import abx
from archivebox.misc.system import atomic_write
from archivebox.misc.util import (
enforce_types,
@ -19,7 +21,6 @@ from archivebox.misc.util import (
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.version import get_COMMIT_HASH
from archivebox.plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
from .schema import Link
from ..logging_util import printable_filesize
@ -79,8 +80,10 @@ def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
@enforce_types
def link_details_template(link: Link) -> str:
from ..extractors.wget import wget_output_path
from abx_plugin_wget_extractor.wget import wget_output_path
SAVE_ARCHIVE_DOT_ORG = abx.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG
link_info = link._asdict(extended=True)
@ -102,7 +105,7 @@ def link_details_template(link: Link) -> str:
'status': 'archived' if link.is_archived else 'not yet archived',
'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
})

4
archivebox/index/json.py
View file

@ -8,7 +8,7 @@ from pathlib import Path
from datetime import datetime, timezone
from typing import List, Optional, Iterator, Any, Union
import abx.archivebox.reads
import abx
from archivebox.config import VERSION, DATA_DIR, CONSTANTS
from archivebox.config.common import SERVER_CONFIG, SHELL_CONFIG
@ -33,7 +33,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
'source': 'https://github.com/ArchiveBox/ArchiveBox',
'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
'dependencies': dict(abx.archivebox.reads.get_BINARIES()),
'dependencies': dict(abx.pm.hook.get_BINARIES()),
},
}

9
archivebox/index/schema.py
View file

@ -17,9 +17,9 @@ from dataclasses import dataclass, asdict, field, fields
from django.utils.functional import cached_property
from archivebox.config import ARCHIVE_DIR, CONSTANTS
import abx
from plugins_extractor.favicon.config import FAVICON_CONFIG
from archivebox.config import ARCHIVE_DIR, CONSTANTS
from archivebox.misc.system import get_dir_size
from archivebox.misc.util import ts_to_date_str, parse_date
@ -426,7 +426,10 @@ class Link:
def canonical_outputs(self) -> Dict[str, Optional[str]]:
"""predict the expected output paths that should be present after archiving"""
from ..extractors.wget import wget_output_path
from abx_plugin_wget.wget import wget_output_path
FAVICON_CONFIG = abx.pm.hook.get_CONFIGS().favicon
# TODO: banish this awful duplication from the codebase and import these
# from their respective extractor files
canonical = {

29
archivebox/machine/models.py
View file

@ -8,9 +8,10 @@ from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
import abx.archivebox.reads
import abx
import archivebox
from abx.archivebox.base_binary import BaseBinary, BaseBinProvider
from pydantic_pkgr import Binary, BinProvider
from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
@ -180,7 +181,7 @@ class NetworkInterface(ABIDModel, ModelWithHealthStats):
class InstalledBinaryManager(models.Manager):
def get_from_db_or_cache(self, binary: BaseBinary) -> 'InstalledBinary':
def get_from_db_or_cache(self, binary: Binary) -> 'InstalledBinary':
"""Get or create an InstalledBinary record for a Binary on the local machine"""
global _CURRENT_BINARIES
@ -216,7 +217,7 @@ class InstalledBinaryManager(models.Manager):
# if binary was not yet loaded from filesystem, do it now
# this is expensive, we have to find it's abspath, version, and sha256, but it's necessary
# to make sure we have a good, up-to-date record of it in the DB & in-memroy cache
binary = binary.load(fresh=True)
binary = archivebox.pm.hook.binary_load(binary=binary, fresh=True)
assert binary.loaded_binprovider and binary.loaded_abspath and binary.loaded_version and binary.loaded_sha256, f'Failed to load binary {binary.name} abspath, version, and sha256'
@ -291,8 +292,8 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
if not hasattr(self, 'machine'):
self.machine = Machine.objects.current()
if not self.binprovider:
all_known_binproviders = list(abx.archivebox.reads.get_BINPROVIDERS().values())
binary = BaseBinary(name=self.name, binproviders=all_known_binproviders).load(fresh=True)
all_known_binproviders = list(abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values())
binary = archivebox.pm.hook.binary_load(binary=Binary(name=self.name, binproviders=all_known_binproviders), fresh=True)
self.binprovider = binary.loaded_binprovider.name if binary.loaded_binprovider else None
if not self.abspath:
self.abspath = self.BINPROVIDER.get_abspath(self.name)
@ -304,16 +305,16 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
super().clean(*args, **kwargs)
@cached_property
def BINARY(self) -> BaseBinary:
for binary in abx.archivebox.reads.get_BINARIES().values():
def BINARY(self) -> Binary:
for binary in abx.as_dict(archivebox.pm.hook.get_BINARIES()).values():
if binary.name == self.name:
return binary
raise Exception(f'Orphaned InstalledBinary {self.name} {self.binprovider} was found in DB, could not find any plugin that defines it')
# TODO: we could technically reconstruct it from scratch, but why would we ever want to do that?
@cached_property
def BINPROVIDER(self) -> BaseBinProvider:
for binprovider in abx.archivebox.reads.get_BINPROVIDERS().values():
def BINPROVIDER(self) -> BinProvider:
for binprovider in abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values():
if binprovider.name == self.binprovider:
return binprovider
raise Exception(f'Orphaned InstalledBinary(name={self.name}) was found in DB, could not find any plugin that defines BinProvider(name={self.binprovider})')
@ -321,7 +322,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
# maybe not a good idea to provide this? Binary in DB is a record of the binary's config
# whereas a loaded binary is a not-yet saved instance that may not have the same config
# why would we want to load a binary record from the db when it could be freshly loaded?
def load_from_db(self) -> BaseBinary:
def load_from_db(self) -> Binary:
# TODO: implement defaults arg in pydantic_pkgr
# return self.BINARY.load(defaults={
# 'binprovider': self.BINPROVIDER,
@ -330,7 +331,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
# 'sha256': self.sha256,
# })
return BaseBinary.model_validate({
return Binary.model_validate({
**self.BINARY.model_dump(),
'abspath': self.abspath and Path(self.abspath),
'version': self.version,
@ -340,5 +341,5 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
'overrides': self.BINARY.overrides,
})
def load_fresh(self) -> BaseBinary:
return self.BINARY.load(fresh=True)
def load_fresh(self) -> Binary:
return archivebox.pm.hook.binary_load(binary=self.BINARY, fresh=True)

60
archivebox/main.py
View file

@ -14,6 +14,10 @@ from crontab import CronTab, CronSlices
from django.db.models import QuerySet
from django.utils import timezone
from pydantic_pkgr import Binary
import abx
import archivebox
from archivebox.misc.checks import check_data_folder
from archivebox.misc.util import enforce_types # type: ignore
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
@ -197,13 +201,13 @@ def version(quiet: bool=False,
from django.conf import settings
from abx.archivebox.base_binary import BaseBinary, apt, brew, env
from abx_plugin_default_binproviders import apt, brew, env
from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID
from archivebox.config.paths import get_data_locations, get_code_locations
from plugins_auth.ldap.config import LDAP_CONFIG
LDAP_ENABLED = archivebox.pm.hook.get_FLAT_CONFIG().LDAP_ENABLED
# 0.7.1
@ -242,7 +246,7 @@ def version(quiet: bool=False,
f'SUDO={CONSTANTS.IS_ROOT}',
f'ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}',
f'SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}',
f'LDAP={LDAP_CONFIG.LDAP_ENABLED}',
f'LDAP={LDAP_ENABLED}',
#f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', # add this if we have more useful info to show eventually
)
prnt()
@ -264,7 +268,8 @@ def version(quiet: bool=False,
prnt('[pale_green1][i] Binary Dependencies:[/pale_green1]')
failures = []
for name, binary in list(settings.BINARIES.items()):
BINARIES = abx.as_dict(archivebox.pm.hook.get_BINARIES())
for name, binary in list(BINARIES.items()):
if binary.name == 'archivebox':
continue
@ -295,14 +300,15 @@ def version(quiet: bool=False,
prnt()
prnt('[gold3][i] Package Managers:[/gold3]')
for name, binprovider in list(settings.BINPROVIDERS.items()):
BINPROVIDERS = abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS())
for name, binprovider in list(BINPROVIDERS.items()):
err = None
if binproviders and binprovider.name not in binproviders:
continue
# TODO: implement a BinProvider.BINARY() method that gets the loaded binary for a binprovider's INSTALLER_BIN
loaded_bin = binprovider.INSTALLER_BINARY or BaseBinary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
loaded_bin = binprovider.INSTALLER_BINARY or Binary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
abspath = None
if loaded_bin.abspath:
@ -1050,10 +1056,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
# - recommend user re-run with sudo if any deps need to be installed as root
from rich import print
from django.conf import settings
import abx.archivebox.reads
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.paths import get_or_create_working_lib_dir
@ -1076,11 +1079,11 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
package_manager_names = ', '.join(
f'[yellow]{binprovider.name}[/yellow]'
for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values()))
for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values()))
if not binproviders or (binproviders and binprovider.name in binproviders)
)
print(f'[+] Setting up package managers {package_manager_names}...')
for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values())):
for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values())):
if binproviders and binprovider.name not in binproviders:
continue
try:
@ -1093,7 +1096,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
print()
for binary in reversed(list(abx.archivebox.reads.get_BINARIES().values())):
for binary in reversed(list(abx.as_dict(abx.pm.hook.get_BINARIES()).values())):
if binary.name in ('archivebox', 'django', 'sqlite', 'python'):
# obviously must already be installed if we are running
continue
@ -1123,7 +1126,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
result = binary.install(binproviders=[binprovider_name], dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
sys.stderr.write("\033[00m\n") # reset
else:
result = binary.load_or_install(binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False)
result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
if result and result['loaded_version']:
break
except Exception as e:
@ -1134,7 +1138,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
binary.install(dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
sys.stderr.write("\033[00m\n") # reset
else:
binary.load_or_install(fresh=True, dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, fresh=True, dry_run=dry_run)
result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
if IS_ROOT and LIB_DIR:
with SudoPermission(uid=0):
if ARCHIVEBOX_USER == 0:
@ -1158,7 +1163,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
print('\n[green][√] Set up ArchiveBox and its dependencies successfully.[/green]\n', file=sys.stderr)
from plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
extra_args = []
if binproviders:
@ -1184,8 +1189,6 @@ def config(config_options_str: Optional[str]=None,
out_dir: Path=DATA_DIR) -> None:
"""Get and set your ArchiveBox project configuration values"""
import abx.archivebox.reads
from rich import print
check_data_folder()
@ -1199,7 +1202,8 @@ def config(config_options_str: Optional[str]=None,
elif config_options_str:
config_options = config_options_str.split('\n')
from django.conf import settings
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
CONFIGS = archivebox.pm.hook.get_CONFIGS()
config_options = config_options or []
@ -1209,8 +1213,8 @@ def config(config_options_str: Optional[str]=None,
if search:
if config_options:
config_options = [get_real_name(key) for key in config_options]
matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
for config_section in settings.CONFIGS.values():
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
for config_section in CONFIGS.values():
aliases = config_section.aliases
for search_key in config_options:
@ -1229,15 +1233,15 @@ def config(config_options_str: Optional[str]=None,
elif get or no_args:
if config_options:
config_options = [get_real_name(key) for key in config_options]
matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
failed_config = [key for key in config_options if key not in settings.FLAT_CONFIG]
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
failed_config = [key for key in config_options if key not in FLAT_CONFIG]
if failed_config:
stderr()
stderr('[X] These options failed to get', color='red')
stderr(' {}'.format('\n '.join(config_options)))
raise SystemExit(1)
else:
matching_config = settings.FLAT_CONFIG
matching_config = FLAT_CONFIG
print(printable_config(matching_config))
raise SystemExit(not matching_config)
@ -1258,20 +1262,20 @@ def config(config_options_str: Optional[str]=None,
if key != raw_key:
stderr(f'[i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.', color='lightyellow')
if key in settings.FLAT_CONFIG:
if key in FLAT_CONFIG:
new_config[key] = val.strip()
else:
failed_options.append(line)
if new_config:
before = settings.FLAT_CONFIG
before = FLAT_CONFIG
matching_config = write_config_file(new_config)
after = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
after = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
print(printable_config(matching_config))
side_effect_changes = {}
for key, val in after.items():
if key in settings.FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
side_effect_changes[key] = after[key]
# import ipdb; ipdb.set_trace()
@ -1313,7 +1317,7 @@ def schedule(add: bool=False,
"""Set ArchiveBox to regularly import URLs at specific times using cron"""
check_data_folder()
from archivebox.plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
from archivebox.config.permissions import USER
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)

3
archivebox/misc/checks.py
View file

@ -201,6 +201,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True):
import archivebox
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.misc.logging import STDERR
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
@ -209,6 +210,8 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
assert lib_dir == archivebox.pm.hook.get_LIB_DIR(), "lib_dir is not the same as the one in the flat config"
if not must_exist and not os.path.isdir(lib_dir):
return True

3
archivebox/misc/shell_welcome_message.py
View file

@ -23,7 +23,7 @@ from archivebox import CONSTANTS # noqa
from ..main import * # noqa
from ..cli import CLI_SUBCOMMANDS
CONFIG = settings.FLAT_CONFIG
CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
CLI_COMMAND_NAMES = ", ".join(CLI_SUBCOMMANDS.keys())
if __name__ == '__main__':
@ -55,6 +55,5 @@ if __name__ == '__main__':
prnt(' add[blink][deep_sky_blue4]?[/deep_sky_blue4][/blink] [grey53]# add ? after anything to get help[/]')
prnt(' add("https://example.com/some/new/url") [grey53]# call CLI methods from the shell[/]')
prnt(' snap = Snapshot.objects.filter(url__contains="https://example.com").last() [grey53]# query for individual snapshots[/]')
prnt(' archivebox.plugins_extractor.wget.apps.WGET_EXTRACTOR.extract(snap.id) [grey53]# call an extractor directly[/]')
prnt(' snap.archiveresult_set.all() [grey53]# see extractor results[/]')
prnt(' bool(re.compile(CONFIG.URL_DENYLIST).search("https://example.com/abc.exe")) [grey53]# test out a config change[/]')

11
archivebox/parsers/pocket_api.py
View file

@ -6,8 +6,7 @@ import re
from typing import IO, Iterable, Optional
from configparser import ConfigParser
from pocket import Pocket
import archivebox
from archivebox.config import CONSTANTS
from archivebox.misc.util import enforce_types
from archivebox.misc.system import atomic_write
@ -22,7 +21,7 @@ API_DB_PATH = CONSTANTS.SOURCES_DIR / 'pocket_api.db'
_BROKEN_PROTOCOL_RE = re.compile('^(http[s]?)(:/(?!/))')
def get_pocket_articles(api: Pocket, since=None, page=0):
def get_pocket_articles(api, since=None, page=0):
body, headers = api.get(
state='archive',
sort='oldest',
@ -94,7 +93,9 @@ def should_parse_as_pocket_api(text: str) -> bool:
def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
"""Parse bookmarks from the Pocket API"""
from archivebox.plugins_extractor.pocket.config import POCKET_CONFIG
from pocket import Pocket
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
input_buffer.seek(0)
pattern = re.compile(r"^pocket:\/\/(\w+)")
@ -102,7 +103,7 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
if should_parse_as_pocket_api(line):
username = pattern.search(line).group(1)
api = Pocket(POCKET_CONFIG.POCKET_CONSUMER_KEY, POCKET_CONFIG.POCKET_ACCESS_TOKENS[username])
api = Pocket(FLAT_CONFIG.POCKET_CONSUMER_KEY, FLAT_CONFIG.POCKET_ACCESS_TOKENS[username])
api.last_since = None
for article in get_pocket_articles(api, since=read_since(username)):

25
archivebox/parsers/readwise_reader_api.py
View file

@ -8,9 +8,10 @@ from datetime import datetime
from typing import IO, Iterable, Optional
from configparser import ConfigParser
import abx
from archivebox.misc.util import enforce_types
from archivebox.misc.system import atomic_write
from archivebox.plugins_extractor.readwise.config import READWISE_CONFIG
from ..index.schema import Link
@ -62,26 +63,30 @@ def link_from_article(article: dict, sources: list):
def write_cursor(username: str, since: str):
if not READWISE_CONFIG.READWISE_DB_PATH.exists():
atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
if not READWISE_DB_PATH.exists():
atomic_write(READWISE_DB_PATH, "")
since_file = ConfigParser()
since_file.optionxform = str
since_file.read(READWISE_CONFIG.READWISE_DB_PATH)
since_file.read(READWISE_DB_PATH)
since_file[username] = {"since": since}
with open(READWISE_CONFIG.READWISE_DB_PATH, "w+") as new:
with open(READWISE_DB_PATH, "w+") as new:
since_file.write(new)
def read_cursor(username: str) -> Optional[str]:
if not READWISE_CONFIG.READWISE_DB_PATH.exists():
atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
if not READWISE_DB_PATH.exists():
atomic_write(READWISE_DB_PATH, "")
config_file = ConfigParser()
config_file.optionxform = str
config_file.read(READWISE_CONFIG.READWISE_DB_PATH)
config_file.read(READWISE_DB_PATH)
return config_file.get(username, "since", fallback=None)
@ -97,12 +102,14 @@ def should_parse_as_readwise_reader_api(text: str) -> bool:
def parse_readwise_reader_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
"""Parse bookmarks from the Readwise Reader API"""
READWISE_READER_TOKENS = abx.pm.hook.get_CONFIG().READWISE_READER_TOKENS
input_buffer.seek(0)
pattern = re.compile(r"^readwise-reader:\/\/(\w+)")
for line in input_buffer:
if should_parse_as_readwise_reader_api(line):
username = pattern.search(line).group(1)
api = ReadwiseReaderAPI(READWISE_CONFIG.READWISE_READER_TOKENS[username], cursor=read_cursor(username))
api = ReadwiseReaderAPI(READWISE_READER_TOKENS[username], cursor=read_cursor(username))
for article in get_readwise_reader_articles(api):
yield link_from_article(article, sources=[line])

6
archivebox/search/__init__.py
View file

@ -6,8 +6,8 @@ from typing import List, Union
from django.db.models import QuerySet
from django.conf import settings
import abx.archivebox.reads
import abx
import archivebox
from archivebox.index.schema import Link
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import stderr
@ -57,7 +57,7 @@ def get_indexable_content(results: QuerySet):
def import_backend():
for backend in abx.archivebox.reads.get_SEARCHBACKENDS().values():
for backend in abx.as_dict(archivebox.pm.hook.get_SEARCHBACKENDS()).values():
if backend.name == SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE:
return backend
raise Exception(f'Could not load {SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE} as search backend')

28
archivebox/vendor/__init__.py vendored
View file

@ -4,23 +4,27 @@ from pathlib import Path
VENDOR_DIR = Path(__file__).parent
VENDORED_LIBS = {
# sys.path dir: library name
#'python-atomicwrites': 'atomicwrites',
#'django-taggit': 'taggit',
# 'pydantic-pkgr': 'pydantic_pkgr',
# 'pocket': 'pocket',
#'base32-crockford': 'base32_crockford',
}
VENDORED_LIBS = [
'abx',
'pydantic-pkgr',
'pocket',
]
for subdir in reversed(sorted(VENDOR_DIR.iterdir())):
if subdir.is_dir() and subdir.name not in VENDORED_LIBS and not subdir.name.startswith('_'):
VENDORED_LIBS.append(subdir.name)
def load_vendored_libs():
for lib_subdir, lib_name in VENDORED_LIBS.items():
lib_dir = VENDOR_DIR / lib_subdir
assert lib_dir.is_dir(), 'Expected vendor libary {lib_name} could not be found in {lib_dir}'
if str(VENDOR_DIR) not in sys.path:
sys.path.append(str(VENDOR_DIR))
for lib_name in VENDORED_LIBS:
lib_dir = VENDOR_DIR / lib_name
assert lib_dir.is_dir(), f'Expected vendor libary {lib_name} could not be found in {lib_dir}'
try:
lib = importlib.import_module(lib_name)
# print(f"Successfully imported lib from environment {lib_name}: {inspect.getfile(lib)}")
# print(f"Successfully imported lib from environment {lib_name}")
except ImportError:
sys.path.append(str(lib_dir))
try:

0
packages/abx-plugin-archivedotorg-extractor/README.md → archivebox/vendor/abx-plugin-archivedotorg/README.md vendored
View file

21
archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/__init__.py vendored Normal file
View file

@ -0,0 +1,21 @@
__label__ = 'Archive.org'
__homepage__ = 'https://archive.org'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import ARCHIVEDOTORG_CONFIG
return {
'ARCHIVEDOTORG_CONFIG': ARCHIVEDOTORG_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import ARCHIVEDOTORG_EXTRACTOR
#
# return {
# 'archivedotorg': ARCHIVEDOTORG_EXTRACTOR,
# }

0
archivebox/extractors/archive_org.py → archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py vendored
View file

5
packages/abx-plugin-archivedotorg-extractor/config.py → archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/config.py vendored
View file

@ -1,7 +1,4 @@
__package__ = 'plugins_extractor.archivedotorg'
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
class ArchivedotorgConfig(BaseConfigSet):

18
archivebox/vendor/abx-plugin-archivedotorg/pyproject.toml vendored Normal file
View file

@ -0,0 +1,18 @@
[project]
name = "abx-plugin-archivedotorg"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-plugin-curl>=2024.10.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_archivedotorg = "abx_plugin_archivedotorg"

0
packages/abx-plugin-chrome-extractor/README.md → archivebox/vendor/abx-plugin-chrome/README.md vendored
View file

34
archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/__init__.py vendored Normal file
View file

@ -0,0 +1,34 @@
__label__ = 'Chrome'
__author__ = 'ArchiveBox'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import CHROME_CONFIG
return {
'CHROME_CONFIG': CHROME_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import CHROME_BINARY
return {
'chrome': CHROME_BINARY,
}
@abx.hookimpl
def ready():
from .config import CHROME_CONFIG
CHROME_CONFIG.validate()
# @abx.hookimpl
# def get_EXTRACTORS():
# return {
# 'pdf': PDF_EXTRACTOR,
# 'screenshot': SCREENSHOT_EXTRACTOR,
# 'dom': DOM_EXTRACTOR,
# }

15
packages/abx-plugin-chrome-extractor/binaries.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/binaries.py vendored
View file

@ -1,5 +1,3 @@
__package__ = 'plugins_extractor.chrome'
import os
import platform
from pathlib import Path
@ -7,17 +5,18 @@ from typing import List, Optional
from pydantic import InstanceOf
from pydantic_pkgr import (
Binary,
BinProvider,
BinName,
BinaryOverrides,
bin_abspath,
)
import abx.archivebox.reads
from abx.archivebox.base_binary import BaseBinary, env, apt, brew
import abx
from abx_puppeteer_binprovider.binproviders import PUPPETEER_BINPROVIDER
from abx_playwright_binprovider.binproviders import PLAYWRIGHT_BINPROVIDER
from abx_plugin_default_binproviders import apt, brew, env
from abx_plugin_puppeteer.binproviders import PUPPETEER_BINPROVIDER
from abx_plugin_playwright.binproviders import PLAYWRIGHT_BINPROVIDER
from .config import CHROME_CONFIG
@ -81,7 +80,7 @@ def create_macos_app_symlink(target: Path, shortcut: Path):
###################### Config ##########################
class ChromeBinary(BaseBinary):
class ChromeBinary(Binary):
name: BinName = CHROME_CONFIG.CHROME_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [PUPPETEER_BINPROVIDER, env, PLAYWRIGHT_BINPROVIDER, apt, brew]
@ -105,7 +104,7 @@ class ChromeBinary(BaseBinary):
@staticmethod
def symlink_to_lib(binary, bin_dir=None) -> None:
bin_dir = bin_dir or abx.archivebox.reads.get_CONFIGS().STORAGE_CONFIG.LIB_DIR / 'bin'
bin_dir = bin_dir or abx.pm.hook.get_BIN_DIR()
if not (binary.abspath and os.path.isfile(binary.abspath)):
return

6
packages/abx-plugin-chrome-extractor/config.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/config.py vendored
View file

@ -1,5 +1,3 @@
__package__ = 'plugins_extractor.chrome'
import os
from pathlib import Path
from typing import List, Optional
@ -7,8 +5,8 @@ from typing import List, Optional
from pydantic import Field
from pydantic_pkgr import bin_abspath
from abx.archivebox.base_configset import BaseConfigSet
from abx.archivebox.base_binary import env
from abx_spec_config.base_configset import BaseConfigSet
from abx_plugin_default_binproviders import env
from archivebox.config import CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG

0
archivebox/extractors/dom.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/dom.py vendored
View file

0
archivebox/extractors/pdf.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/pdf.py vendored
View file

0
archivebox/extractors/screenshot.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/screenshot.py vendored
View file

18
archivebox/vendor/abx-plugin-chrome/pyproject.toml vendored Normal file
View file

@ -0,0 +1,18 @@
[project]
name = "abx-plugin-chrome"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_chrome = "abx_plugin_chrome"

0
packages/abx-plugin-curl-extractor/README.md → archivebox/vendor/abx-plugin-curl/README.md vendored
View file

18
archivebox/vendor/abx-plugin-curl/abx_plugin_curl/__init__.py vendored Normal file
View file

@ -0,0 +1,18 @@
import abx
@abx.hookimpl
def get_CONFIG():
from .config import CURL_CONFIG
return {
'curl': CURL_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import CURL_BINARY
return {
'curl': CURL_BINARY,
}

8
packages/abx-plugin-curl-extractor/binaries.py → archivebox/vendor/abx-plugin-curl/abx_plugin_curl/binaries.py vendored
View file

@ -1,17 +1,17 @@
__package__ = 'plugins_extractor.curl'
__package__ = 'abx_plugin_curl'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName
from pydantic_pkgr import BinProvider, BinName, Binary
from abx.archivebox.base_binary import BaseBinary, env, apt, brew
from abx_plugin_default_binproviders import apt, brew, env
from .config import CURL_CONFIG
class CurlBinary(BaseBinary):
class CurlBinary(Binary):
name: BinName = CURL_CONFIG.CURL_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]

4
packages/abx-plugin-curl-extractor/config.py → archivebox/vendor/abx-plugin-curl/abx_plugin_curl/config.py vendored
View file

@ -1,11 +1,11 @@
__package__ = 'plugins_extractor.curl'
__package__ = 'abx_plugin_curl'
from typing import List, Optional
from pathlib import Path
from pydantic import Field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG

0
archivebox/extractors/headers.py → archivebox/vendor/abx-plugin-curl/abx_plugin_curl/headers.py vendored
View file

18
archivebox/vendor/abx-plugin-curl/pyproject.toml vendored Normal file
View file

@ -0,0 +1,18 @@
[project]
name = "abx-plugin-curl"
version = "2024.10.24"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_curl = "abx_plugin_curl"

0
packages/abx-plugin-default-binproviders/README.md → archivebox/vendor/abx-plugin-default-binproviders/README.md vendored
View file

1
packages/abx-plugin-default-binproviders/abx_plugin_default_binproviders.py → archivebox/vendor/abx-plugin-default-binproviders/abx_plugin_default_binproviders.py vendored
View file

@ -16,7 +16,6 @@ env = ENV_BINPROVIDER = EnvProvider()
@abx.hookimpl(tryfirst=True)
def get_BINPROVIDERS() -> Dict[str, BinProvider]:
return {
'apt': APT_BINPROVIDER,
'brew': BREW_BINPROVIDER,

0
packages/abx-plugin-default-binproviders/pyproject.toml → archivebox/vendor/abx-plugin-default-binproviders/pyproject.toml vendored
View file

0
packages/abx-plugin-favicon-extractor/README.md → archivebox/vendor/abx-plugin-favicon/README.md vendored
View file

29
archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/__init__.py vendored Normal file
View file

@ -0,0 +1,29 @@
__label__ = 'Favicon'
__version__ = '2024.10.24'
__author__ = 'ArchiveBox'
__homepage__ = 'https://github.com/ArchiveBox/archivebox'
__dependencies__ = [
'abx>=0.1.0',
'abx-spec-config>=0.1.0',
'abx-plugin-curl-extractor>=2024.10.24',
]
import abx
@abx.hookimpl
def get_CONFIG():
from .config import FAVICON_CONFIG
return {
'FAVICON_CONFIG': FAVICON_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import FAVICON_EXTRACTOR
# return {
# 'favicon': FAVICON_EXTRACTOR,
# }

5
packages/abx-plugin-favicon-extractor/config.py → archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/config.py vendored
View file

@ -1,7 +1,4 @@
__package__ = 'plugins_extractor.favicon'
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
class FaviconConfig(BaseConfigSet):

0
archivebox/extractors/favicon.py → archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/favicon.py vendored
View file

18
archivebox/vendor/abx-plugin-favicon/pyproject.toml vendored Normal file
View file

@ -0,0 +1,18 @@
[project]
name = "abx-plugin-favicon"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-plugin-curl>=2024.10.28",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_favicon = "abx_plugin_favicon"

0
packages/abx-plugin-git-extractor/README.md → archivebox/vendor/abx-plugin-git/README.md vendored
View file

29
archivebox/vendor/abx-plugin-git/abx_plugin_git/__init__.py vendored Normal file
View file

@ -0,0 +1,29 @@
__package__ = 'abx_plugin_git'
__label__ = 'Git'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import GIT_CONFIG
return {
'GIT_CONFIG': GIT_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import GIT_BINARY
return {
'git': GIT_BINARY,
}
@abx.hookimpl
def get_EXTRACTORS():
from .extractors import GIT_EXTRACTOR
return {
'git': GIT_EXTRACTOR,
}

8
packages/abx-plugin-git-extractor/binaries.py → archivebox/vendor/abx-plugin-git/abx_plugin_git/binaries.py vendored
View file

@ -1,17 +1,17 @@
__package__ = 'plugins_extractor.git'
__package__ = 'abx_plugin_git'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName
from pydantic_pkgr import BinProvider, BinName, Binary
from abx.archivebox.base_binary import BaseBinary, env, apt, brew
from abx_plugin_default_binproviders import apt, brew, env
from .config import GIT_CONFIG
class GitBinary(BaseBinary):
class GitBinary(Binary):
name: BinName = GIT_CONFIG.GIT_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]

4
packages/abx-plugin-git-extractor/config.py → archivebox/vendor/abx-plugin-git/abx_plugin_git/config.py vendored
View file

@ -1,10 +1,10 @@
__package__ = 'plugins_extractor.git'
__package__ = 'abx_plugin_git'
from typing import List
from pydantic import Field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG

15
archivebox/vendor/abx-plugin-git/abx_plugin_git/extractors.py vendored Normal file
View file

@ -0,0 +1,15 @@
__package__ = 'abx_plugin_git'
# from pathlib import Path
# from .binaries import GIT_BINARY
# class GitExtractor(BaseExtractor):
# name: ExtractorName = 'git'
# binary: str = GIT_BINARY.name
# def get_output_path(self, snapshot) -> Path | None:
# return snapshot.as_link() / 'git'
# GIT_EXTRACTOR = GitExtractor()

4
archivebox/extractors/git.py → archivebox/vendor/abx-plugin-git/abx_plugin_git/git.py vendored
View file

@ -16,8 +16,8 @@ from archivebox.misc.util import (
from ..logging_util import TimedProgress
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.plugins_extractor.git.config import GIT_CONFIG
from archivebox.plugins_extractor.git.binaries import GIT_BINARY
from abx_plugin_git.config import GIT_CONFIG
from abx_plugin_git.binaries import GIT_BINARY
def get_output_path():

19
archivebox/vendor/abx-plugin-git/pyproject.toml vendored Normal file
View file

@ -0,0 +1,19 @@
[project]
name = "abx-plugin-git"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
"abx-plugin-default-binproviders>=2024.10.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_git = "abx_plugin_git"

0
packages/abx-plugin-htmltotext-extractor/README.md → archivebox/vendor/abx-plugin-htmltotext/README.md vendored
View file

22
archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/__init__.py vendored Normal file
View file

@ -0,0 +1,22 @@
__package__ = 'abx_plugin_htmltotext'
__label__ = 'HTML-to-Text'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import HTMLTOTEXT_CONFIG
return {
'HTMLTOTEXT_CONFIG': HTMLTOTEXT_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import FAVICON_EXTRACTOR
# return {
# 'htmltotext': FAVICON_EXTRACTOR,
# }

5
packages/abx-plugin-htmltotext-extractor/config.py → archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/config.py vendored
View file

@ -1,7 +1,4 @@
__package__ = 'plugins_extractor.htmltotext'
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
class HtmltotextConfig(BaseConfigSet):

0
archivebox/extractors/htmltotext.py → archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py vendored
View file

17
archivebox/vendor/abx-plugin-htmltotext/pyproject.toml vendored Normal file
View file

@ -0,0 +1,17 @@
[project]
name = "abx-plugin-htmltotext"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_htmltotext = "abx_plugin_htmltotext"

0
packages/abx-plugin-ldap-auth/README.md → archivebox/vendor/abx-plugin-ldap-auth/README.md vendored
View file

33
packages/abx-plugin-ldap-auth/__init__.py → archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/__init__.py vendored
View file

@ -1,36 +1,15 @@
__package__ = 'plugins_auth.ldap'
__id__ = 'ldap'
__package__ = 'abx_plugin_ldap_auth'
__label__ = 'LDAP'
__version__ = '2024.10.14'
__author__ = 'ArchiveBox'
__homepage__ = 'https://github.com/django-auth-ldap/django-auth-ldap'
__dependencies__ = ['pip']
import abx
@abx.hookimpl
def get_PLUGIN():
return {
__id__: {
'id': __id__,
'package': __package__,
'label': __label__,
'version': __version__,
'author': __author__,
'homepage': __homepage__,
'dependencies': __dependencies__,
}
}
@abx.hookimpl
def get_CONFIG():
from .config import LDAP_CONFIG
return {
__id__: LDAP_CONFIG
'LDAP_CONFIG': LDAP_CONFIG
}
@abx.hookimpl
@ -48,12 +27,12 @@ def create_superuser_from_ldap_user(sender, user=None, ldap_user=None, **kwargs)
ArchiveBox requires staff/superuser status to view the admin at all, so we must create a user
+ set staff and superuser when LDAP authenticates a new person.
"""
from django.conf import settings
from .config import LDAP_CONFIG
if user is None:
return # not authenticated at all
if not user.id and settings.CONFIGS.ldap.LDAP_CREATE_SUPERUSER:
if not user.id and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
user.is_superuser = True # authenticated via LDAP, but user is not set up in DB yet
user.is_staff = True
@ -69,9 +48,7 @@ def ready():
LDAP_CONFIG.validate()
from django.conf import settings
if settings.CONFIGS.ldap.LDAP_ENABLED:
if LDAP_CONFIG.LDAP_ENABLED:
# tell django-auth-ldap to call our function when a user is authenticated via LDAP
import django_auth_ldap.backend
django_auth_ldap.backend.populate_user.connect(create_superuser_from_ldap_user)

15
packages/abx-plugin-ldap-auth/binaries.py → archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/binaries.py vendored
View file

@ -1,5 +1,4 @@
__package__ = 'plugins_auth.ldap'
__package__ = 'abx_plugin_ldap_auth'
import inspect
@ -7,12 +6,10 @@ from typing import List
from pathlib import Path
from pydantic import InstanceOf
from pydantic_pkgr import BinaryOverrides, SemVer
from pydantic_pkgr import BinaryOverrides, SemVer, Binary, BinProvider
from abx.archivebox.base_binary import BaseBinary, BaseBinProvider, apt
from plugins_pkg.pip.binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
from abx_plugin_default_binproviders import apt
from abx_plugin_pip.binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
from .config import get_ldap_lib
@ -39,10 +36,10 @@ def get_LDAP_LIB_version():
return LDAP_LIB and SemVer(LDAP_LIB.__version__)
class LdapBinary(BaseBinary):
class LdapBinary(Binary):
name: str = 'ldap'
description: str = 'LDAP Authentication'
binproviders_supported: List[InstanceOf[BaseBinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
overrides: BinaryOverrides = {
LIB_PIP_BINPROVIDER.name: {

6
packages/abx-plugin-ldap-auth/config.py → archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/config.py vendored
View file

@ -1,11 +1,11 @@
__package__ = 'plugins_auth.ldap'
__package__ = 'abx_plugin_ldap_auth'
import sys
from typing import Dict, List, Optional
from pydantic import Field, model_validator, computed_field
from pydantic import Field, computed_field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
LDAP_LIB = None
LDAP_SEARCH = None

20
archivebox/vendor/abx-plugin-ldap-auth/pyproject.toml vendored Normal file
View file

@ -0,0 +1,20 @@
[project]
name = "abx-plugin-ldap-auth"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-django>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_ldap_auth = "abx_plugin_ldap_auth"

0
packages/abx-plugin-mercury-extractor/README.md → archivebox/vendor/abx-plugin-mercury/README.md vendored
View file

29
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/__init__.py vendored Normal file
View file

@ -0,0 +1,29 @@
__package__ = 'abx_plugin_mercury'
__label__ = 'Postlight Parser'
__homepage__ = 'https://github.com/postlight/mercury-parser'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import MERCURY_CONFIG
return {
'MERCURY_CONFIG': MERCURY_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import MERCURY_BINARY
return {
'mercury': MERCURY_BINARY,
}
@abx.hookimpl
def get_EXTRACTORS():
from .extractors import MERCURY_EXTRACTOR
return {
'mercury': MERCURY_EXTRACTOR,
}

10
packages/abx-plugin-mercury-extractor/binaries.py → archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/binaries.py vendored
View file

@ -1,18 +1,18 @@
__package__ = 'plugins_extractor.mercury'
__package__ = 'abx_plugin_mercury'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath, Binary
from abx.archivebox.base_binary import BaseBinary, env
from abx_plugin_default_binproviders import env
from archivebox.plugins_pkg.npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
from abx_plugin_npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
from .config import MERCURY_CONFIG
class MercuryBinary(BaseBinary):
class MercuryBinary(Binary):
name: BinName = MERCURY_CONFIG.MERCURY_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]

4
packages/abx-plugin-mercury-extractor/config.py → archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/config.py vendored
View file

@ -1,11 +1,11 @@
__package__ = 'plugins_extractor.mercury'
__package__ = 'abx_plugin_mercury'
from typing import List, Optional
from pathlib import Path
from pydantic import Field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG

17
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/extractors.py vendored Normal file
View file

@ -0,0 +1,17 @@
__package__ = 'abx_plugin_mercury'
# from pathlib import Path
# from .binaries import MERCURY_BINARY
# class MercuryExtractor(BaseExtractor):
# name: ExtractorName = 'mercury'
# binary: str = MERCURY_BINARY.name
# def get_output_path(self, snapshot) -> Path | None:
# return snapshot.link_dir / 'mercury' / 'content.html'
# MERCURY_EXTRACTOR = MercuryExtractor()

0
archivebox/extractors/mercury.py → archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/mercury.py vendored
View file

17
archivebox/vendor/abx-plugin-mercury/pyproject.toml vendored Normal file
View file

@ -0,0 +1,17 @@
[project]
name = "abx-plugin-mercury"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_mercury = "abx_plugin_mercury"

0
packages/abx-plugin-npm-binprovider/README.md → archivebox/vendor/abx-plugin-npm/README.md vendored
View file

5
packages/abx-plugin-npm-binprovider/abx_plugin_npm_binprovider/__init__.py → archivebox/vendor/abx-plugin-npm/abx_plugin_npm/__init__.py vendored
View file

@ -1,5 +1,3 @@
__package__ = 'abx_plugin_npm_binprovider'
__id__ = 'npm'
__label__ = 'NPM'
__author__ = 'ArchiveBox'
__homepage__ = 'https://www.npmjs.com/'
@ -10,9 +8,8 @@ import abx
@abx.hookimpl
def get_CONFIG():
from .config import NPM_CONFIG
return {
__id__: NPM_CONFIG,
'NPM_CONFIG': NPM_CONFIG,
}
@abx.hookimpl

0
packages/abx-plugin-npm-binprovider/abx_plugin_npm_binprovider/binaries.py → archivebox/vendor/abx-plugin-npm/abx_plugin_npm/binaries.py vendored
View file

3
packages/abx-plugin-npm-binprovider/abx_plugin_npm_binprovider/binproviders.py → archivebox/vendor/abx-plugin-npm/abx_plugin_npm/binproviders.py vendored
View file

@ -26,8 +26,7 @@ class LibNpmBinProvider(NpmProvider):
def setup(self) -> None:
# update paths from config at runtime
LIB_DIR = abx.pm.hook.get_CONFIG().LIB_DIR
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.npm_prefix = LIB_DIR / 'npm'
self.PATH = f'{LIB_DIR / "npm" / "node_modules" / ".bin"}:{NEW_NODE_BIN_PATH}:{OLD_NODE_BIN_PATH}'

0
packages/abx-plugin-npm-binprovider/abx_plugin_npm_binprovider/config.py → archivebox/vendor/abx-plugin-npm/abx_plugin_npm/config.py vendored
View file

4
packages/abx-plugin-npm-binprovider/pyproject.toml → archivebox/vendor/abx-plugin-npm/pyproject.toml vendored
View file

@ -1,5 +1,5 @@
[project]
name = "abx-plugin-npm-binprovider"
name = "abx-plugin-npm"
version = "2024.10.24"
description = "NPM binary provider plugin for ABX"
readme = "README.md"
@ -17,4 +17,4 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_npm_binprovider = "abx_plugin_npm_binprovider"
abx_plugin_npm = "abx_plugin_npm"

0
packages/abx-plugin-pip-binprovider/README.md → archivebox/vendor/abx-plugin-pip/README.md vendored
View file

0
packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/.plugin_order → archivebox/vendor/abx-plugin-pip/abx_plugin_pip/.plugin_order vendored
View file

5
packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/__init__.py → archivebox/vendor/abx-plugin-pip/abx_plugin_pip/__init__.py vendored
View file

@ -1,5 +1,4 @@
__package__ = 'abx_plugin_pip_binprovider'
__id__ = 'pip'
__package__ = 'abx_plugin_pip'
__label__ = 'PIP'
import abx
@ -10,7 +9,7 @@ def get_CONFIG():
from .config import PIP_CONFIG
return {
__id__: PIP_CONFIG
'PIP_CONFIG': PIP_CONFIG
}
@abx.hookimpl(tryfirst=True)

2
packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/binaries.py → archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binaries.py vendored
View file

@ -1,4 +1,4 @@
__package__ = 'abx_plugin_pip_binprovider'
__package__ = 'abx_plugin_pip'
import sys
from pathlib import Path

2
packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/binproviders.py → archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binproviders.py vendored
View file

@ -58,7 +58,7 @@ class LibPipBinProvider(PipProvider):
def setup(self) -> None:
# update venv path to match most up-to-date LIB_DIR based on runtime config
LIB_DIR = abx.pm.hook.get_FLAT_CONFIG().LIB_DIR
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.pip_venv = LIB_DIR / 'pip' / 'venv'
super().setup()

2
packages/abx-plugin-pip-binprovider/abx_plugin_pip_binprovider/config.py → archivebox/vendor/abx-plugin-pip/abx_plugin_pip/config.py vendored
View file

@ -3,7 +3,7 @@ __package__ = 'pip'
from typing import List, Optional
from pydantic import Field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
class PipDependencyConfigs(BaseConfigSet):

4
packages/abx-plugin-pip-binprovider/pyproject.toml → archivebox/vendor/abx-plugin-pip/pyproject.toml vendored
View file

@ -1,5 +1,5 @@
[project]
name = "abx-plugin-pip-binprovider"
name = "abx-plugin-pip"
version = "2024.10.24"
description = "Add your description here"
readme = "README.md"
@ -19,4 +19,4 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_pip_binprovider = "abx_plugin_pip_binprovider"
abx_plugin_pip = "abx_plugin_pip"

0
packages/abx-plugin-playwright-binprovider/README.md → archivebox/vendor/abx-plugin-playwright/README.md vendored
View file

6
packages/abx-plugin-playwright-binprovider/abx_plugin_playwright_binprovider/__init__.py → archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/__init__.py vendored
View file

@ -1,7 +1,4 @@
__package__ = 'abx_plugin_playwright_binprovider'
__id__ = 'playwright'
__label__ = 'Playwright'
__author__ = 'ArchiveBox'
__homepage__ = 'https://github.com/microsoft/playwright-python'
import abx
@ -10,9 +7,8 @@ import abx
@abx.hookimpl
def get_CONFIG():
from .config import PLAYWRIGHT_CONFIG
return {
__id__: PLAYWRIGHT_CONFIG
'PLAYWRIGHT_CONFIG': PLAYWRIGHT_CONFIG
}
@abx.hookimpl

4
packages/abx-plugin-playwright-binprovider/abx_plugin_playwright_binprovider/binaries.py → archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binaries.py vendored
View file

@ -1,4 +1,4 @@
__package__ = 'abx_plugin_playwright_binprovider'
__package__ = 'abx_plugin_playwright'
from typing import List
@ -6,7 +6,7 @@ from pydantic import InstanceOf
from pydantic_pkgr import BinName, BinProvider, Binary
from abx_plugin_pip_binprovider.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
from abx_plugin_pip.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
from abx_plugin_default_binproviders import env
from .config import PLAYWRIGHT_CONFIG

4
packages/abx-plugin-playwright-binprovider/abx_plugin_playwright_binprovider/binproviders.py → archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binproviders.py vendored
View file

@ -1,4 +1,4 @@
__package__ = 'abx_plugin_playwright_binprovider'
__package__ = 'abx_plugin_playwright'
import os
import shutil
@ -59,7 +59,7 @@ class PlaywrightBinProvider(BinProvider):
def setup(self) -> None:
# update paths from config at runtime
LIB_DIR = abx.pm.hook.get_FLAT_CONFIG().LIB_DIR
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.PATH = f"{LIB_DIR / 'bin'}:{DEFAULT_ENV_PATH}"

0
packages/abx-plugin-playwright-binprovider/abx_plugin_playwright_binprovider/config.py → archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/config.py vendored
View file

6
packages/abx-plugin-playwright-binprovider/pyproject.toml → archivebox/vendor/abx-plugin-playwright/pyproject.toml vendored
View file

@ -1,6 +1,6 @@
[project]
name = "abx-plugin-playwright-binprovider"
version = "2024.10.24"
name = "abx-plugin-playwright"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
@ -17,4 +17,4 @@ requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_playwright_binprovider = "abx_plugin_playwright_binprovider"
abx_plugin_playwright = "abx_plugin_playwright"

0
packages/abx-plugin-pocket-extractor/README.md → archivebox/vendor/abx-plugin-pocket/README.md vendored
View file

18
archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/__init__.py vendored Normal file
View file

@ -0,0 +1,18 @@
__package__ = 'abx_plugin_pocket'
__label__ = 'Pocket'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import POCKET_CONFIG
return {
'POCKET_CONFIG': POCKET_CONFIG
}
@abx.hookimpl
def ready():
from .config import POCKET_CONFIG
POCKET_CONFIG.validate()

4
packages/abx-plugin-pocket-extractor/config.py → archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/config.py vendored
View file

@ -1,10 +1,10 @@
__package__ = 'plugins_extractor.pocket'
__package__ = 'abx_plugin_pocket'
from typing import Dict
from pydantic import Field
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
class PocketConfig(BaseConfigSet):

18
archivebox/vendor/abx-plugin-pocket/pyproject.toml vendored Normal file
View file

@ -0,0 +1,18 @@
[project]
name = "abx-plugin-pocket"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"pocket>=0.3.6",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_pocket = "abx_plugin_pocket"

0
packages/abx-plugin-puppeteer-binprovider/README.md → archivebox/vendor/abx-plugin-puppeteer/README.md vendored
View file

30
archivebox/vendor/abx-plugin-puppeteer/abx_plugin_puppeteer/__init__.py vendored Normal file
View file

@ -0,0 +1,30 @@
__package__ = 'abx_plugin_puppeteer'
__label__ = 'Puppeteer'
__homepage__ = 'https://github.com/puppeteer/puppeteer'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import PUPPETEER_CONFIG
return {
'PUPPETEER_CONFIG': PUPPETEER_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import PUPPETEER_BINARY
return {
'puppeteer': PUPPETEER_BINARY,
}
@abx.hookimpl
def get_BINPROVIDERS():
from .binproviders import PUPPETEER_BINPROVIDER
return {
'puppeteer': PUPPETEER_BINPROVIDER,
}

Some files were not shown because too many files have changed in this diff Show more