Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-10 06:34:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

BasePlugin system expanded and registration system improved

Browse source
This commit is contained in:
Nick Sweeting 2024-09-03 00:58:50 -07:00
parent f1579bfdcd
commit 9af260df16
No known key found for this signature in database
50 changed files with 1062 additions and 973 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

83
archivebox/builtin_plugins/base/apps.py
View file

@ -1,83 +0,0 @@
import sys
import inspect
from typing import List, Dict, Any, Optional
from pathlib import Path
import django
from django.apps import AppConfig
from django.core.checks import Tags, Warning, register
from django.db.backends.sqlite3.base import Database as sqlite3
from pydantic import (
Field,
SerializeAsAny,
)
from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
from plugantic.extractors import Extractor, ExtractorName
from plugantic.plugins import Plugin
from plugantic.configs import ConfigSet, ConfigSectionName
from plugantic.replayers import Replayer
class PythonBinary(Binary):
name: BinName = 'python'
providers_supported: List[BinProvider] = [EnvProvider()]
provider_overrides: Dict[str, Any] = {
'env': {
'subdeps': \
lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
'abspath': \
lambda: sys.executable,
'version': \
lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
},
}
class SqliteBinary(Binary):
name: BinName = 'sqlite'
providers_supported: List[BinProvider] = [EnvProvider()]
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'env': {
'abspath': \
lambda: Path(inspect.getfile(sqlite3)),
'version': \
lambda: SemVer(sqlite3.version),
},
}
class DjangoBinary(Binary):
name: BinName = 'django'
providers_supported: List[BinProvider] = [EnvProvider()]
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'env': {
'abspath': \
lambda: inspect.getfile(django),
'version': \
lambda: django.VERSION[:3],
},
}
class BasicReplayer(Replayer):
name: str = 'basic'
class BasePlugin(Plugin):
name: str = 'base'
configs: List[SerializeAsAny[ConfigSet]] = []
binaries: List[SerializeAsAny[Binary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
extractors: List[SerializeAsAny[Extractor]] = []
replayers: List[SerializeAsAny[Replayer]] = [BasicReplayer()]
PLUGINS = [BasePlugin()]
class BaseConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'builtin_plugins.base'

0
archivebox/builtin_plugins/base/__init__.py → archivebox/builtin_plugins/npm/__init__.py
View file

66
archivebox/builtin_plugins/npm/apps.py Normal file
View file

@ -0,0 +1,66 @@
__package__ = 'archivebox.builtin_plugins.npm'
from pathlib import Path
from typing import List, Dict, Optional
from pydantic import InstanceOf, Field
from django.apps import AppConfig
from django.conf import settings
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
from plugantic.base_configset import ConfigSectionName
from pkg.settings import env, apt, brew
from ...config import CONFIG
###################### Config ##########################
class NpmDependencyConfigs(BaseConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
USE_NPM: bool = True
NPM_BINARY: str = Field(default='npm')
NPM_ARGS: Optional[List[str]] = Field(default=None)
NPM_EXTRA_ARGS: List[str] = []
NPM_DEFAULT_ARGS: List[str] = []
DEFAULT_GLOBAL_CONFIG = {
}
NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
class NpmProvider(NpmProvider, BaseBinProvider):
PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
class NpmBinary(BaseBinary):
name: BinName = 'npm'
binproviders_supported: List[InstanceOf[BinProvider]] = [env, apt, brew]
NPM_BINARY = NpmBinary()
class NpmPlugin(BasePlugin):
name: str = 'builtin_plugins.npm'
app_label: str = 'npm'
verbose_name: str = 'NPM'
configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
binaries: List[InstanceOf[BaseBinary]] = [NPM_BINARY]
PLUGIN = NpmPlugin()
DJANGO_APP = PLUGIN.AppConfig
# CONFIGS = PLUGIN.configs
# BINARIES = PLUGIN.binaries
# EXTRACTORS = PLUGIN.extractors
# REPLAYERS = PLUGIN.replayers
# CHECKS = PLUGIN.checks

0
archivebox/builtin_plugins/base/migrations/__init__.py → archivebox/builtin_plugins/pip/__init__.py
View file

66
archivebox/builtin_plugins/pip/apps.py Normal file
View file

@ -0,0 +1,66 @@
import sys
from pathlib import Path
from typing import List, Dict, Optional
from pydantic import InstanceOf, Field
from django.apps import AppConfig
from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
from plugantic.base_configset import ConfigSectionName
from pkg.settings import env, apt, brew
###################### Config ##########################
class PipDependencyConfigs(BaseConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
USE_PIP: bool = True
PIP_BINARY: str = Field(default='pip')
PIP_ARGS: Optional[List[str]] = Field(default=None)
PIP_EXTRA_ARGS: List[str] = []
PIP_DEFAULT_ARGS: List[str] = []
DEFAULT_GLOBAL_CONFIG = {
}
PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
class PipProvider(PipProvider, BaseBinProvider):
PATH: PATHStr = str(Path(sys.executable).parent)
pip = PipProvider(PATH=str(Path(sys.executable).parent))
class PipBinary(BaseBinary):
name: BinName = 'pip'
binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
PIP_BINARY = PipBinary()
class PipPlugin(BasePlugin):
name: str = 'builtin_plugins.pip'
app_label: str = 'pip'
verbose_name: str = 'PIP'
configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY]
PLUGIN = PipPlugin()
DJANGO_APP = PLUGIN.AppConfig
# CONFIGS = PLUGIN.configs
# BINARIES = PLUGIN.binaries
# EXTRACTORS = PLUGIN.extractors
# REPLAYERS = PLUGIN.replayers
# CHECKS = PLUGIN.checks

123
archivebox/builtin_plugins/singlefile/apps.py
View file

@ -1,42 +1,31 @@
from typing import List, Optional, Dict
from pathlib import Path
from typing import List, Dict, Optional
from django.apps import AppConfig
from django.core.checks import Tags, Warning, register
from pydantic import (
Field,
SerializeAsAny,
)
from pydantic_pkgr import BinProvider, BinName, Binary, EnvProvider, NpmProvider
# Depends on other PyPI/vendor packages:
from pydantic import InstanceOf, Field
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
from pydantic_pkgr.binprovider import bin_abspath
from pydantic_pkgr.binary import BinProviderName, ProviderLookupDict
from plugantic.extractors import Extractor, ExtractorName
from plugantic.plugins import Plugin
from plugantic.configs import ConfigSet, ConfigSectionName
# Depends on other Django apps:
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
from plugantic.base_configset import ConfigSectionName
# Depends on Other Plugins:
from pkg.settings import env
from builtin_plugins.npm.apps import npm
###################### Config ##########################
class SinglefileToggleConfig(ConfigSet):
class SinglefileToggleConfigs(BaseConfigSet):
section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
SAVE_SINGLEFILE: bool = True
class SinglefileDependencyConfig(ConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
SINGLEFILE_BINARY: str = Field(default='wget')
SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
SINGLEFILE_EXTRA_ARGS: List[str] = []
SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
class SinglefileOptionsConfig(ConfigSet):
class SinglefileOptionsConfigs(BaseConfigSet):
section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
# loaded from shared config
@ -47,67 +36,83 @@ class SinglefileOptionsConfig(ConfigSet):
SINGLEFILE_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
class SinglefileDependencyConfigs(BaseConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
DEFAULT_CONFIG = {
SINGLEFILE_BINARY: str = Field(default='wget')
SINGLEFILE_ARGS: Optional[List[str]] = Field(default=None)
SINGLEFILE_EXTRA_ARGS: List[str] = []
SINGLEFILE_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
class SinglefileConfigs(SinglefileToggleConfigs, SinglefileOptionsConfigs, SinglefileDependencyConfigs):
# section: ConfigSectionName = 'ALL_CONFIGS'
pass
DEFAULT_GLOBAL_CONFIG = {
'CHECK_SSL_VALIDITY': False,
'SAVE_SINGLEFILE': True,
'TIMEOUT': 120,
}
PLUGIN_CONFIG = [
SinglefileToggleConfig(**DEFAULT_CONFIG),
SinglefileDependencyConfig(**DEFAULT_CONFIG),
SinglefileOptionsConfig(**DEFAULT_CONFIG),
SINGLEFILE_CONFIGS = [
SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
]
###################### Binaries ############################
min_version: str = "1.1.54"
max_version: str = "2.0.0"
class SinglefileBinary(Binary):
name: BinName = 'single-file'
providers_supported: List[BinProvider] = [NpmProvider()]
def get_singlefile_abspath() -> Optional[Path]:
return
class SinglefileBinary(BaseBinary):
name: BinName = 'single-file'
binproviders_supported: List[InstanceOf[BinProvider]] = [env, npm]
provider_overrides: Dict[BinProviderName, ProviderLookupDict] ={
'env': {
'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
},
'npm': {
# 'abspath': lambda: bin_abspath('single-file', PATH=NpmProvider().PATH) or bin_abspath('single-file', PATH=env.PATH),
'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
},
# 'env': {
# 'abspath': lambda: bin_abspath('single-file-node.js', PATH=env.PATH) or bin_abspath('single-file', PATH=env.PATH),
# },
# 'npm': {
# 'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
# 'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
# },
}
SINGLEFILE_BINARY = SinglefileBinary()
###################### Extractors ##########################
PLUGIN_BINARIES = [SINGLEFILE_BINARY]
class SinglefileExtractor(Extractor):
name: ExtractorName = 'singlefile'
binary: Binary = SinglefileBinary()
class SinglefileExtractor(BaseExtractor):
name: str = 'singlefile'
binary: BinName = SINGLEFILE_BINARY.name
def get_output_path(self, snapshot) -> Path:
return Path(snapshot.link_dir) / 'singlefile.html'
###################### Plugins #############################
SINGLEFILE_BINARY = SinglefileBinary()
SINGLEFILE_EXTRACTOR = SinglefileExtractor()
class SinglefilePlugin(BasePlugin):
name: str = 'builtin_plugins.singlefile'
app_label: str ='singlefile'
verbose_name: str = 'SingleFile'
configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
class SinglefilePlugin(Plugin):
name: str = 'singlefile'
configs: List[SerializeAsAny[ConfigSet]] = [*PLUGIN_CONFIG]
binaries: List[SerializeAsAny[Binary]] = [SinglefileBinary()]
extractors: List[SerializeAsAny[Extractor]] = [SinglefileExtractor()]
PLUGINS = [SinglefilePlugin()]
###################### Django Apps #########################
class SinglefileConfig(AppConfig):
name = 'builtin_plugins.singlefile'
verbose_name = 'SingleFile'
def ready(self):
pass
# print('Loaded singlefile plugin')
PLUGIN = SinglefilePlugin()
DJANGO_APP = PLUGIN.AppConfig
# CONFIGS = PLUGIN.configs
# BINARIES = PLUGIN.binaries
# EXTRACTORS = PLUGIN.extractors
# REPLAYERS = PLUGIN.replayers
# CHECKS = PLUGIN.checks

66
archivebox/builtin_plugins/singlefile/config.yaml
View file

@ -1,66 +0,0 @@
name: singlefile
plugin_version: '0.0.1'
plugin_spec: '0.0.1'
binaries:
singlefile:
providers:
- env
- npm
commands:
- singlefile.exec
- singlefile.extract
- singlefile.should_extract
- singlefile.get_output_path
extractors:
singlefile:
binary: singlefile
test: singlefile.should_extract
extract: singlefile.extract
output_files:
- singlefile.html
configs:
ARCHIVE_METHOD_TOGGLES:
SAVE_SINGLEFILE:
type: bool
default: true
DEPENDENCY_CONFIG:
SINGLEFILE_BINARY:
type: str
default: wget
SINGLEFILE_ARGS:
type: Optional[List[str]]
default: null
SINGLEFILE_EXTRA_ARGS:
type: List[str]
default: []
SINGLEFILE_DEFAULT_ARGS:
type: List[str]
default:
- "--timeout={TIMEOUT-10}"
ARCHIVE_METHOD_OPTIONS:
SINGLEFILE_USER_AGENT:
type: str
default: ""
alias: USER_AGENT
SINGLEFILE_TIMEOUT:
type: int
default: 60
alias: TIMEOUT
SINGLEFILE_CHECK_SSL_VALIDITY:
type: bool
default: true
alias: CHECK_SSL_VALIDITY
SINGLEFILE_RESTRICT_FILE_NAMES:
type: str
default: windows
alias: RESTRICT_FILE_NAMES
SINGLEFILE_COOKIES_FILE:
type: Optional[Path]
default: null
alias: COOKIES_FILE

3
archivebox/builtin_plugins/singlefile/tests.py
View file

@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.

0
archivebox/builtin_plugins/systempython/__init__.py Normal file
View file

0
archivebox/builtin_plugins/base/admin.py → archivebox/builtin_plugins/systempython/admin.py
View file

116
archivebox/builtin_plugins/systempython/apps.py Normal file
View file

@ -0,0 +1,116 @@
__package__ = 'archivebox.builtin_plugins.systempython'
import os
import sys
import inspect
from typing import List, Dict, Any, Callable, ClassVar
from pathlib import Path
import django
from django.apps import AppConfig
from django.core.checks import Tags, Warning, register
from django.utils.functional import classproperty
from django.db.backends.sqlite3.base import Database as sqlite3
from django.core.checks import Tags, Error, register
from pydantic import InstanceOf, Field
from pydantic_pkgr import SemVer, BinProvider, BinProviderName, ProviderLookupDict, BinName, Binary, EnvProvider, NpmProvider
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider, BaseExtractor, BaseReplayer
from plugantic.base_check import BaseCheck
from pkg.settings import env, apt, brew
from builtin_plugins.pip.apps import pip
class PythonBinary(BaseBinary):
name: BinName = 'python'
binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'apt': {
'subdeps': \
lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
'abspath': \
lambda: sys.executable,
'version': \
lambda: '{}.{}.{}'.format(*sys.version_info[:3]),
},
}
class SqliteBinary(BaseBinary):
name: BinName = 'sqlite'
binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'pip': {
'abspath': \
lambda: Path(inspect.getfile(sqlite3)),
'version': \
lambda: SemVer(sqlite3.version),
},
}
class DjangoBinary(BaseBinary):
name: BinName = 'django'
binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'pip': {
'abspath': \
lambda: inspect.getfile(django),
'version': \
lambda: django.VERSION[:3],
},
}
class BasicReplayer(BaseReplayer):
name: str = 'basic'
class CheckUserIsNotRoot(BaseCheck):
label: str = 'CheckUserIsNotRoot'
tag = Tags.database
@staticmethod
def check(settings, logger) -> List[Warning]:
errors = []
if getattr(settings, "USER", None) == 'root' or getattr(settings, "PUID", None) == 0:
errors.append(
Error(
"Cannot run as root!",
id="core.S001",
hint=f'Run ArchiveBox as a non-root user with a UID greater than 500. (currently running as UID {os.getuid()}).',
)
)
logger.debug('[√] UID is not root')
return errors
class SystemPythonPlugin(BasePlugin):
name: str = 'builtin_plugins.systempython'
app_label: str = 'systempython'
verbose_name: str = 'System Python'
configs: List[InstanceOf[BaseConfigSet]] = []
binaries: List[InstanceOf[BaseBinary]] = [PythonBinary(), SqliteBinary(), DjangoBinary()]
extractors: List[InstanceOf[BaseExtractor]] = []
replayers: List[InstanceOf[BaseReplayer]] = [BasicReplayer()]
checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
PLUGIN = SystemPythonPlugin()
DJANGO_APP = PLUGIN.AppConfig
# CONFIGS = PLUGIN.configs
# BINARIES = PLUGIN.binaries
# EXTRACTORS = PLUGIN.extractors
# REPLAYERS = PLUGIN.replayers
# PARSERS = PLUGIN.parsers
# DAEMONS = PLUGIN.daemons
# MODELS = PLUGIN.models
# CHECKS = PLUGIN.checks

0
archivebox/builtin_plugins/systempython/migrations/__init__.py Normal file
View file

0
archivebox/builtin_plugins/base/models.py → archivebox/builtin_plugins/systempython/models.py
View file

0
archivebox/builtin_plugins/base/tests.py → archivebox/builtin_plugins/systempython/tests.py
View file

0
archivebox/builtin_plugins/base/views.py → archivebox/builtin_plugins/systempython/views.py
View file

0
archivebox/builtin_plugins/ytdlp/__init__.py Normal file
View file

48
archivebox/builtin_plugins/ytdlp/apps.py Normal file
View file

@ -0,0 +1,48 @@
import sys
from pathlib import Path
from typing import List, Dict, Optional
from pydantic import InstanceOf, Field
from django.apps import AppConfig
from pydantic_pkgr import BinProvider, BinName, PATHStr
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
from plugantic.base_configset import ConfigSectionName
from pkg.settings import env, apt, brew
from builtin_plugins.pip.apps import pip
###################### Config ##########################
class YtdlpDependencyConfigs(BaseConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
USE_YTDLP: bool = True
YTDLP_BINARY: str = Field(default='yt-dlp')
DEFAULT_GLOBAL_CONFIG = {}
YTDLP_CONFIG = YtdlpDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
class YtdlpBinary(BaseBinary):
name: BinName = YTDLP_CONFIG.YTDLP_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [env, pip, apt, brew]
YTDLP_BINARY = YtdlpBinary()
class YtdlpPlugin(BasePlugin):
name: str = 'builtin_plugins.ytdlp'
app_label: str = 'ytdlp'
verbose_name: str = 'YTDLP'
configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG]
binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY]
PLUGIN = YtdlpPlugin()
DJANGO_APP = PLUGIN.AppConfig

149
archivebox/core/settings.py
View file

@ -19,6 +19,46 @@ IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
################################################################################
### ArchiveBox Plugin Settings
################################################################################
BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins' # /app/archivebox/builtin_plugins
USERDATA_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins' # /data/user_plugins
def find_plugins_in_dir(plugins_dir, prefix: str) -> Dict[str, Path]:
return {
f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
for plugin_entrypoint in sorted(plugins_dir.glob('*/apps.py'))
}
INSTALLED_PLUGINS = {
**find_plugins_in_dir(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
**find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'),
}
### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
PLUGINS = AttrDict({})
CONFIGS = AttrDict({})
BINPROVIDERS = AttrDict({})
BINARIES = AttrDict({})
EXTRACTORS = AttrDict({})
REPLAYERS = AttrDict({})
CHECKS = AttrDict({})
ADMINDATAVIEWS = AttrDict({})
PLUGIN_KEYS = AttrDict({
'CONFIGS': CONFIGS,
'BINPROVIDERS': BINPROVIDERS,
'BINARIES': BINARIES,
'EXTRACTORS': EXTRACTORS,
'REPLAYERS': REPLAYERS,
'CHECKS': CHECKS,
'ADMINDATAVIEWS': ADMINDATAVIEWS,
})
################################################################################
### Django Core Settings
################################################################################
@ -35,52 +75,35 @@ APPEND_SLASH = True
DEBUG = CONFIG.DEBUG or ('--debug' in sys.argv)
BUILTIN_PLUGINS_DIR = CONFIG.PACKAGE_DIR / 'builtin_plugins'
USER_PLUGINS_DIR = CONFIG.OUTPUT_DIR / 'user_plugins'
def find_plugins(plugins_dir, prefix: str) -> Dict[str, Any]:
plugins = {
f'{prefix}.{plugin_entrypoint.parent.name}': plugin_entrypoint.parent
for plugin_entrypoint in plugins_dir.glob('*/apps.py')
}
# print(f'Found {prefix} plugins:\n', '\n '.join(plugins.keys()))
return plugins
INSTALLED_PLUGINS = {
**find_plugins(BUILTIN_PLUGINS_DIR, prefix='builtin_plugins'),
**find_plugins(USER_PLUGINS_DIR, prefix='user_plugins'),
}
INSTALLED_APPS = [
# Django default apps
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.admin',
'django_jsonform',
# 3rd-party apps from PyPI
'django_jsonform', # handles rendering Pydantic models to Django HTML widgets/forms
'signal_webhooks', # handles REST API outbound webhooks
'signal_webhooks',
'abid_utils',
'plugantic',
'core',
'api',
'pkg',
# our own apps
'abid_utils', # handles ABID ID creation, handling, and models
'plugantic', # ArchiveBox plugin API definition + finding/registering/calling interface
'core', # core django model with Snapshot, ArchiveResult, etc.
'api', # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
'pkg', # ArchiveBox runtime package management interface for subdependencies
*INSTALLED_PLUGINS.keys(),
# ArchiveBox plugins
*INSTALLED_PLUGINS.keys(), # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins
'admin_data_views',
'django_extensions',
# 3rd-party apps from PyPI that need to be loaded last
'admin_data_views', # handles rendering some convenient automatic read-only views of data in Django admin
'django_extensions', # provides Django Debug Toolbar (and other non-debug helpers)
]
# For usage with https://www.jetadmin.io/integrations/django
# INSTALLED_APPS += ['jet_django']
# JET_PROJECT = 'archivebox'
# JET_TOKEN = 'some-api-token-here'
MIDDLEWARE = [
'core.middleware.TimezoneMiddleware',
'django.middleware.security.SecurityMiddleware',
@ -371,8 +394,11 @@ LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
"console": {
"level": "DEBUG",
"filters": [],
'formatter': 'simple',
"class": "logging.StreamHandler",
},
'logfile': {
'level': 'ERROR',
@ -380,14 +406,57 @@ LOGGING = {
'filename': ERROR_LOG,
'maxBytes': 1024 * 1024 * 25, # 25 MB
'backupCount': 10,
'formatter': 'verbose',
},
# "mail_admins": {
# "level": "ERROR",
# "filters": ["require_debug_false"],
# "class": "django.utils.log.AdminEmailHandler",
# },
},
'filters': {
'noisyrequestsfilter': {
'()': NoisyRequestsFilter,
}
},
"require_debug_false": {
"()": "django.utils.log.RequireDebugFalse",
},
"require_debug_true": {
"()": "django.utils.log.RequireDebugTrue",
},
},
'formatters': {
'verbose': {
'format': '{name} {levelname} {asctime} {module} {process:d} {thread:d} {message}',
'style': '{',
},
'simple': {
'format': '{name} {message}',
'style': '{',
},
"django.server": {
"()": "django.utils.log.ServerFormatter",
"format": "[{server_time}] {message}",
"style": "{",
},
},
'loggers': {
'api': {
'handlers': ['console', 'logfile'],
'level': 'DEBUG',
},
'checks': {
'handlers': ['console', 'logfile'],
'level': 'DEBUG',
},
'core': {
'handlers': ['console', 'logfile'],
'level': 'DEBUG',
},
'builtin_plugins': {
'handlers': ['console', 'logfile'],
'level': 'DEBUG',
},
'django': {
'handlers': ['console', 'logfile'],
'level': 'INFO',
@ -397,6 +466,8 @@ LOGGING = {
'handlers': ['console', 'logfile'],
'level': 'INFO',
'filters': ['noisyrequestsfilter'],
'propagate': False,
"formatter": "django.server",
}
},
}
@ -541,3 +612,9 @@ if DEBUG_REQUESTS_TRACKER:
# https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
DEBUG_LOGFIRE = False
DEBUG_LOGFIRE = DEBUG_LOGFIRE and (Path(CONFIG.OUTPUT_DIR) / '.logfire').is_dir()
# For usage with https://www.jetadmin.io/integrations/django
# INSTALLED_APPS += ['jet_django']
# JET_PROJECT = 'archivebox'
# JET_TOKEN = 'some-api-token-here'

2
archivebox/main.py
View file

@ -318,7 +318,7 @@ def init(force: bool=False, quick: bool=False, setup: bool=False, out_dir: Path=
print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
elif existing_index:
# TODO: properly detect and print the existing version in current index as well
print('{green}[^] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
print('{green}[*] Verifying and updating existing ArchiveBox collection to v{}...{reset}'.format(VERSION, **ANSI))
print('{green}----------------------------------------------------------------------{reset}'.format(**ANSI))
else:
if force:

71
archivebox/pkg/settings.py
View file

@ -10,77 +10,24 @@ import django
from django.conf import settings
from django.db.backends.sqlite3.base import Database as sqlite3
from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
from pydantic_pkgr import Binary, BinProvider, BrewProvider, PipProvider, NpmProvider, AptProvider, EnvProvider, SemVer
from pydantic_pkgr.binprovider import bin_abspath
from ..config import NODE_BIN_PATH, bin_path
env = EnvProvider(PATH=NODE_BIN_PATH + ':' + os.environ.get('PATH', '/bin'))
apt = AptProvider()
brew = BrewProvider()
env = EnvProvider(PATH=os.environ.get('PATH', '/bin'))
# Defined in their own plugins:
#pip = PipProvider(PATH=str(Path(sys.executable).parent))
#npm = NpmProvider(PATH=NODE_BIN_PATH)
LOADED_DEPENDENCIES = {}
for bin_key, dependency in settings.CONFIG.DEPENDENCIES.items():
# 'PYTHON_BINARY': {
# 'path': bin_path(config['PYTHON_BINARY']),
# 'version': config['PYTHON_VERSION'],
# 'hash': bin_hash(config['PYTHON_BINARY']),
# 'enabled': True,
# 'is_valid': bool(config['PYTHON_VERSION']),
# },
bin_name = settings.CONFIG[bin_key]
if bin_name.endswith('django/__init__.py'):
binary_spec = Binary(name='django', providers=[env], provider_overrides={
'env': {
'abspath': lambda: Path(inspect.getfile(django)),
'version': lambda: SemVer('{}.{}.{} {} ({})'.format(*django.VERSION)),
}
})
elif bin_name.endswith('sqlite3/dbapi2.py'):
binary_spec = Binary(name='sqlite3', providers=[env], provider_overrides={
'env': {
'abspath': lambda: Path(inspect.getfile(sqlite3)),
'version': lambda: SemVer(sqlite3.version),
}
})
elif bin_name.endswith('archivebox'):
binary_spec = Binary(name='archivebox', providers=[env], provider_overrides={
'env': {
'abspath': lambda: shutil.which(str(Path('archivebox').expanduser())),
'version': lambda: settings.CONFIG.VERSION,
}
})
elif bin_name.endswith('postlight/parser/cli.js'):
binary_spec = Binary(name='postlight-parser', providers=[env], provider_overrides={
'env': {
'abspath': lambda: bin_path('postlight-parser'),
'version': lambda: SemVer('1.0.0'),
}
})
else:
binary_spec = Binary(name=bin_name, providers=[env])
for bin_name, binary_spec in settings.BINARIES.items():
try:
binary = binary_spec.load()
settings.BINARIES[bin_name] = binary_spec.load()
except Exception as e:
# print(f"- ❌ Binary {bin_name} failed to load with error: {e}")
continue
assert isinstance(binary.loaded_version, SemVer)
try:
assert str(binary.loaded_version) == dependency['version'], f"Expected {bin_name} version {dependency['version']}, got {binary.loaded_version}"
assert str(binary.loaded_respath) == str(bin_abspath(dependency['path']).resolve()), f"Expected {bin_name} abspath {bin_abspath(dependency['path']).resolve()}, got {binary.loaded_respath}"
assert binary.is_valid == dependency['is_valid'], f"Expected {bin_name} is_valid={dependency['is_valid']}, got {binary.is_valid}"
except Exception as e:
pass
# print(f"WARNING: Error loading {bin_name}: {e}")
# import ipdb; ipdb.set_trace()
# print(f"- ✅ Binary {bin_name} loaded successfully")
LOADED_DEPENDENCIES[bin_key] = binary

19
archivebox/plugantic/__init__.py
View file

@ -1,16 +1,9 @@
__package__ = 'archivebox.plugantic'
from .binaries import Binary
from .extractors import Extractor
from .replayers import Replayer
from .configs import ConfigSet
from .plugins import Plugin
from .base_plugin import BasePlugin
from .base_configset import BaseConfigSet
from .base_binary import BaseBinary
from .base_extractor import BaseExtractor
from .base_replayer import BaseReplayer
from .base_check import BaseCheck
# __all__ = [
# 'BinProvider',
# 'Binary',
# 'Extractor',
# 'Replayer',
# 'ConfigSet',
# 'Plugin',
# ]

14
archivebox/plugantic/apps.py
View file

@ -1,6 +1,9 @@
import importlib
from django.apps import AppConfig
__package__ = 'archivebox.plugantic'
import json
import importlib
from django.apps import AppConfig
class PluganticConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
@ -8,10 +11,5 @@ class PluganticConfig(AppConfig):
def ready(self) -> None:
from django.conf import settings
from .plugins import PLUGINS
for plugin_name in settings.INSTALLED_PLUGINS.keys():
lib = importlib.import_module(f'{plugin_name}.apps')
if hasattr(lib, 'PLUGINS'):
for plugin_instance in lib.PLUGINS:
PLUGINS.append(plugin_instance)
print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')

34
archivebox/plugantic/base_admindataview.py Normal file
View file

@ -0,0 +1,34 @@
from typing import List, Type, Any, Dict
from pydantic_core import core_schema
from pydantic import GetCoreSchemaHandler, BaseModel
from django.utils.functional import classproperty
from django.core.checks import Warning, Tags, register
class BaseAdminDataView(BaseModel):
name: str = 'NPM Installed Packages'
route: str = '/npm/installed/'
view: str = 'builtin_plugins.npm.admin.installed_list_view'
items: Dict[str, str] = {
"name": "installed_npm_pkg",
'route': '<str:key>/',
'view': 'builtin_plugins.npm.admin.installed_detail_view',
}
def as_route(self) -> Dict[str, str | Dict[str, str]]:
return {
'route': self.route,
'view': self.view,
'name': self.name,
'items': self.items,
}
def register(self, settings, parent_plugin=None):
"""Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime"""
self._plugin = parent_plugin # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
route = self.as_route()
if route not in settings.ADMIN_DATA_VIEWS.URLS:
settings.ADMIN_DATA_VIEWS.URLS += [route] # append our route (update in place)

99
archivebox/plugantic/base_binary.py Normal file
View file

@ -0,0 +1,99 @@
__package__ = 'archivebox.plugantic'
import sys
import inspect
import importlib
from pathlib import Path
from typing import Any, Optional, Dict, List
from typing_extensions import Self
from subprocess import run, PIPE
from pydantic import Field, InstanceOf
from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
from pydantic_pkgr.binprovider import HostBinPath
import django
from django.core.cache import cache
from django.db.backends.sqlite3.base import Database as sqlite3
class BaseBinProvider(BinProvider):
# def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
# Class = super()
# get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
# # return cache.get_or_set(f'bin:abspath:{bin_name}', get_abspath_func)
# return get_abspath_func()
# def on_get_version(self, bin_name: BinName, abspath: Optional[HostBinPath]=None, **context) -> SemVer | None:
# Class = super()
# get_version_func = lambda: Class.on_get_version(bin_name, abspath, **context)
# # return cache.get_or_set(f'bin:version:{bin_name}:{abspath}', get_version_func)
# return get_version_func()
def register(self, settings, parent_plugin=None):
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
self._plugin = parent_plugin # for debugging only, never rely on this!
settings.BINPROVIDERS[self.name] = self
class BaseBinary(Binary):
binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
def register(self, settings, parent_plugin=None):
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
self._plugin = parent_plugin # for debugging only, never rely on this!
settings.BINARIES[self.name] = self
# def get_ytdlp_version() -> str:
# import yt_dlp
# return yt_dlp.version.__version__
# class YtdlpBinary(Binary):
# name: BinName = 'yt-dlp'
# providers_supported: List[BinProvider] = [
# EnvProvider(),
# PipProvider(),
# BrewProvider(),
# AptProvider(),
# ]
# provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
# 'pip': {
# 'version': get_ytdlp_version,
# },
# 'brew': {
# 'subdeps': lambda: 'yt-dlp ffmpeg',
# },
# 'apt': {
# 'subdeps': lambda: 'yt-dlp ffmpeg',
# }
# }
# class WgetBinary(Binary):
# name: BinName = 'wget'
# providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
# if __name__ == '__main__':
# PYTHON_BINARY = PythonBinary()
# SQLITE_BINARY = SqliteBinary()
# DJANGO_BINARY = DjangoBinary()
# WGET_BINARY = WgetBinary()
# YTDLP_BINARY = YtdlpPBinary()
# print('-------------------------------------DEFINING BINARIES---------------------------------')
# print(PYTHON_BINARY)
# print(SQLITE_BINARY)
# print(DJANGO_BINARY)
# print(WGET_BINARY)
# print(YTDLP_BINARY)

55
archivebox/plugantic/base_check.py Normal file
View file

@ -0,0 +1,55 @@
from typing import List, Type, Any
from pydantic_core import core_schema
from pydantic import GetCoreSchemaHandler
from django.utils.functional import classproperty
from django.core.checks import Warning, Tags, register
class BaseCheck:
label: str = ''
tag = Tags.database
@classmethod
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
return core_schema.typed_dict_schema(
{
'name': core_schema.typed_dict_field(core_schema.str_schema()),
'tag': core_schema.typed_dict_field(core_schema.str_schema()),
},
)
@classproperty
def name(cls) -> str:
return cls.label or cls.__name__
@staticmethod
def check(settings, logger) -> List[Warning]:
"""Override this method to implement your custom runtime check."""
errors = []
# if not hasattr(settings, 'SOME_KEY'):
# errors.extend(Error(
# 'Missing settings.SOME_KEY after django_setup(), did SOME_KEY get loaded?',
# id='core.C001',
# hint='Make sure to run django_setup() is able to load settings.SOME_KEY.',
# ))
# logger.debug('[√] Loaded settings.PLUGINS succesfully.')
return errors
def register(self, settings, parent_plugin=None):
# Regsiter in ArchiveBox plugins runtime settings
self._plugin = parent_plugin
settings.CHECKS[self.name] = self
# Register using Django check framework
def run_check(app_configs, **kwargs) -> List[Warning]:
from django.conf import settings
import logging
settings = settings
logger = logging.getLogger('checks')
return self.check(settings, logger)
run_check.__name__ = self.label or self.__class__.__name__
run_check.tags = [self.tag]
register(self.tag)(run_check)

81
archivebox/plugantic/base_configset.py Normal file
View file

@ -0,0 +1,81 @@
__package__ = 'archivebox.plugantic'
from typing import Optional, List, Literal
from pathlib import Path
from pydantic import BaseModel, Field, ConfigDict, computed_field
ConfigSectionName = Literal[
'GENERAL_CONFIG',
'ARCHIVE_METHOD_TOGGLES',
'ARCHIVE_METHOD_OPTIONS',
'DEPENDENCY_CONFIG',
]
ConfigSectionNames: List[ConfigSectionName] = [
'GENERAL_CONFIG',
'ARCHIVE_METHOD_TOGGLES',
'ARCHIVE_METHOD_OPTIONS',
'DEPENDENCY_CONFIG',
]
class BaseConfigSet(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
section: ConfigSectionName = 'GENERAL_CONFIG'
@computed_field
@property
def name(self) -> str:
return self.__class__.__name__
def register(self, settings, parent_plugin=None):
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
self._plugin = parent_plugin # for debugging only, never rely on this!
settings.CONFIGS[self.name] = self
# class WgetToggleConfig(ConfigSet):
# section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
# SAVE_WGET: bool = True
# SAVE_WARC: bool = True
# class WgetDependencyConfig(ConfigSet):
# section: ConfigSectionName = 'DEPENDENCY_CONFIG'
# WGET_BINARY: str = Field(default='wget')
# WGET_ARGS: Optional[List[str]] = Field(default=None)
# WGET_EXTRA_ARGS: List[str] = []
# WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
# class WgetOptionsConfig(ConfigSet):
# section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
# # loaded from shared config
# WGET_AUTO_COMPRESSION: bool = Field(default=True)
# SAVE_WGET_REQUISITES: bool = Field(default=True)
# WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
# WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
# WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
# WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
# WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
# CONFIG = {
# 'CHECK_SSL_VALIDITY': False,
# 'SAVE_WARC': False,
# 'TIMEOUT': 999,
# }
# WGET_CONFIG = [
# WgetToggleConfig(**CONFIG),
# WgetDependencyConfig(**CONFIG),
# WgetOptionsConfig(**CONFIG),
# ]

67
archivebox/plugantic/extractors.py → archivebox/plugantic/base_extractor.py
View file

@ -6,13 +6,14 @@ from typing_extensions import Self
from abc import ABC
from pathlib import Path
from pydantic import BaseModel, model_validator, field_serializer, AfterValidator
from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field
from pydantic_pkgr import BinName
from .binaries import (
Binary,
YtdlpBinary,
WgetBinary,
)
# from .binaries import (
# Binary,
# YtdlpBinary,
# WgetBinary,
# )
# stubs
@ -37,9 +38,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
class Extractor(ABC, BaseModel):
class BaseExtractor(ABC, BaseModel):
name: ExtractorName
binary: Binary
binary: BinName
output_path_func: HandlerFuncStr = 'self.get_output_path'
should_extract_func: HandlerFuncStr = 'self.should_extract'
@ -55,10 +56,14 @@ class Extractor(ABC, BaseModel):
if self.args is None:
self.args = [*self.default_args, *self.extra_args]
return self
def register(self, settings, parent_plugin=None):
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
@field_serializer('binary', when_used='json')
def dump_binary(binary) -> str:
return binary.name
self._plugin = parent_plugin # for debugging only, never rely on this!
settings.EXTRACTORS[self.name] = self
def get_output_path(self, snapshot) -> Path:
return Path(self.name)
@ -86,33 +91,37 @@ class Extractor(ABC, BaseModel):
'returncode': proc.returncode,
}
def exec(self, args: CmdArgsList, pwd: Optional[Path]=None):
def exec(self, args: CmdArgsList, pwd: Optional[Path]=None, settings=None):
pwd = pwd or Path('.')
assert self.binary.loaded_provider
return self.binary.exec(args, pwd=pwd)
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
binary = settings.BINARIES[self.binary]
return binary.exec(args, pwd=pwd)
class YtdlpExtractor(Extractor):
name: ExtractorName = 'media'
binary: Binary = YtdlpBinary()
# class YtdlpExtractor(Extractor):
# name: ExtractorName = 'media'
# binary: Binary = YtdlpBinary()
def get_output_path(self, snapshot) -> Path:
return Path(self.name)
# def get_output_path(self, snapshot) -> Path:
# return Path(self.name)
class WgetExtractor(Extractor):
name: ExtractorName = 'wget'
binary: Binary = WgetBinary()
# class WgetExtractor(Extractor):
# name: ExtractorName = 'wget'
# binary: Binary = WgetBinary()
def get_output_path(self, snapshot) -> Path:
return get_wget_output_path(snapshot)
# def get_output_path(self, snapshot) -> Path:
# return get_wget_output_path(snapshot)
class WarcExtractor(Extractor):
name: ExtractorName = 'warc'
binary: Binary = WgetBinary()
# class WarcExtractor(Extractor):
# name: ExtractorName = 'warc'
# binary: Binary = WgetBinary()
def get_output_path(self, snapshot) -> Path:
return get_wget_output_path(snapshot)
# def get_output_path(self, snapshot) -> Path:
# return get_wget_output_path(snapshot)

202
archivebox/plugantic/base_plugin.py Normal file
View file

@ -0,0 +1,202 @@
__package__ = 'archivebox.plugantic'
import json
from django.apps import AppConfig
from django.core.checks import register
from typing import List, ClassVar, Type, Dict
from typing_extensions import Self
from pydantic import (
BaseModel,
ConfigDict,
Field,
model_validator,
InstanceOf,
computed_field,
validate_call,
)
from .base_configset import BaseConfigSet
from .base_binary import BaseBinProvider, BaseBinary
from .base_extractor import BaseExtractor
from .base_replayer import BaseReplayer
from .base_check import BaseCheck
from .base_admindataview import BaseAdminDataView
from ..config import ANSI, AttrDict
class BasePlugin(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
# Required by AppConfig:
name: str = Field() # e.g. 'builtin_plugins.singlefile'
app_label: str = Field() # e.g. 'singlefile'
verbose_name: str = Field() # e.g. 'SingleFile'
default_auto_field: ClassVar[str] = 'django.db.models.AutoField'
# Required by Plugantic:
configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
extractors: List[InstanceOf[BaseExtractor]] = Field(default=[])
replayers: List[InstanceOf[BaseReplayer]] = Field(default=[])
checks: List[InstanceOf[BaseCheck]] = Field(default=[])
admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[])
@model_validator(mode='after')
def validate(self) -> Self:
"""Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
@property
def AppConfig(plugin_self) -> Type[AppConfig]:
"""Generate a Django AppConfig class for this plugin."""
class PluginAppConfig(AppConfig):
name = plugin_self.name
app_label = plugin_self.app_label
verbose_name = plugin_self.verbose_name
def ready(self):
from django.conf import settings
plugin_self.validate()
plugin_self.register(settings)
return PluginAppConfig
@computed_field
@property
def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]:
return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders})
@computed_field
@property
def BINARIES(self) -> Dict[str, BaseBinary]:
return AttrDict({binary.python_name: binary for binary in self.binaries})
@computed_field
@property
def CONFIGS(self) -> Dict[str, BaseConfigSet]:
return AttrDict({config.name: config for config in self.configs})
@computed_field
@property
def EXTRACTORS(self) -> Dict[str, BaseExtractor]:
return AttrDict({extractor.name: extractor for extractor in self.extractors})
@computed_field
@property
def REPLAYERS(self) -> Dict[str, BaseReplayer]:
return AttrDict({replayer.name: replayer for replayer in self.replayers})
@computed_field
@property
def CHECKS(self) -> Dict[str, BaseCheck]:
return AttrDict({check.name: check for check in self.checks})
@computed_field
@property
def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
@computed_field
@property
def PLUGIN_KEYS(self) -> List[str]:
return
def register(self, settings=None):
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.'
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).'
### Mutate django.conf.settings... values in-place to include plugin-provided overrides
settings.PLUGINS[self.app_label] = self
for config in self.CONFIGS.values():
config.register(settings, parent_plugin=self)
for binprovider in self.BINPROVIDERS.values():
binprovider.register(settings, parent_plugin=self)
for binary in self.BINARIES.values():
binary.register(settings, parent_plugin=self)
for extractor in self.EXTRACTORS.values():
extractor.register(settings, parent_plugin=self)
for replayer in self.REPLAYERS.values():
replayer.register(settings, parent_plugin=self)
for check in self.CHECKS.values():
check.register(settings, parent_plugin=self)
for admindataview in self.ADMINDATAVIEWS.values():
admindataview.register(settings, parent_plugin=self)
# TODO: add parsers? custom templates? persona fixtures?
plugin_prefix, plugin_shortname = self.name.split('.', 1)
print(
f' > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' +
f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' +
f'{ANSI.black}CONFIGSx{len(self.configs)} BINARIESx{len(self.binaries)} EXTRACTORSx{len(self.extractors)} REPLAYERSx{len(self.replayers)} CHECKSx{len(self.CHECKS)} ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}'
)
# @validate_call
# def install_binaries(self) -> Self:
# new_binaries = []
# for idx, binary in enumerate(self.binaries):
# new_binaries.append(binary.install() or binary)
# return self.model_copy(update={
# 'binaries': new_binaries,
# })
@validate_call
def load_binaries(self, cache=True) -> Self:
new_binaries = []
for idx, binary in enumerate(self.binaries):
new_binaries.append(binary.load(cache=cache) or binary)
return self.model_copy(update={
'binaries': new_binaries,
})
# @validate_call
# def load_or_install_binaries(self, cache=True) -> Self:
# new_binaries = []
# for idx, binary in enumerate(self.binaries):
# new_binaries.append(binary.load_or_install(cache=cache) or binary)
# return self.model_copy(update={
# 'binaries': new_binaries,
# })
# class YtdlpPlugin(BasePlugin):
# name: str = 'ytdlp'
# configs: List[SerializeAsAny[BaseConfigSet]] = []
# binaries: List[SerializeAsAny[BaseBinary]] = [YtdlpBinary()]
# extractors: List[SerializeAsAny[BaseExtractor]] = [YtdlpExtractor()]
# replayers: List[SerializeAsAny[BaseReplayer]] = [MEDIA_REPLAYER]
# class WgetPlugin(BasePlugin):
# name: str = 'wget'
# configs: List[SerializeAsAny[BaseConfigSet]] = [*WGET_CONFIG]
# binaries: List[SerializeAsAny[BaseBinary]] = [WgetBinary()]
# extractors: List[SerializeAsAny[BaseExtractor]] = [WgetExtractor(), WarcExtractor()]

17
archivebox/plugantic/replayers.py → archivebox/plugantic/base_replayer.py
View file

@ -3,10 +3,9 @@ __package__ = 'archivebox.plugantic'
from pydantic import BaseModel
# from .binproviders import LazyImportStr
class Replayer(BaseModel):
class BaseReplayer(BaseModel):
"""Describes how to render an ArchiveResult in several contexts"""
name: str = 'GenericReplayer'
url_pattern: str = '*'
@ -21,5 +20,17 @@ class Replayer(BaseModel):
# icon_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
# thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
def register(self, settings, parent_plugin=None):
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
MEDIA_REPLAYER = Replayer(name='media')
self._plugin = parent_plugin # for debugging only, never rely on this!
settings.REPLAYERS[self.name] = self
# class MediaReplayer(BaseReplayer):
# name: str = 'MediaReplayer'
# MEDIA_REPLAYER = MediaReplayer()

65
archivebox/plugantic/binaries.py
View file

@ -1,65 +0,0 @@
__package__ = 'archivebox.plugantic'
import sys
import inspect
import importlib
from pathlib import Path
from typing import Any, Optional, Dict, List
from typing_extensions import Self
from subprocess import run, PIPE
from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
import django
from django.db.backends.sqlite3.base import Database as sqlite3
def get_ytdlp_version() -> str:
import yt_dlp
return yt_dlp.version.__version__
class YtdlpBinary(Binary):
name: BinName = 'yt-dlp'
providers_supported: List[BinProvider] = [
EnvProvider(),
PipProvider(),
BrewProvider(),
AptProvider(),
]
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
'pip': {
'version': get_ytdlp_version,
},
'brew': {
'subdeps': lambda: 'yt-dlp ffmpeg',
},
'apt': {
'subdeps': lambda: 'yt-dlp ffmpeg',
}
}
class WgetBinary(Binary):
name: BinName = 'wget'
providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
# if __name__ == '__main__':
# PYTHON_BINARY = PythonBinary()
# SQLITE_BINARY = SqliteBinary()
# DJANGO_BINARY = DjangoBinary()
# WGET_BINARY = WgetBinary()
# YTDLP_BINARY = YtdlpPBinary()
# print('-------------------------------------DEFINING BINARIES---------------------------------')
# print(PYTHON_BINARY)
# print(SQLITE_BINARY)
# print(DJANGO_BINARY)
# print(WGET_BINARY)
# print(YTDLP_BINARY)

53
archivebox/plugantic/configs.py
View file

@ -1,53 +0,0 @@
__package__ = 'archivebox.plugantic'
from typing import Optional, List, Literal
from pathlib import Path
from pydantic import BaseModel, Field
ConfigSectionName = Literal['GENERAL_CONFIG', 'ARCHIVE_METHOD_TOGGLES', 'ARCHIVE_METHOD_OPTIONS', 'DEPENDENCY_CONFIG']
class ConfigSet(BaseModel):
section: ConfigSectionName = 'GENERAL_CONFIG'
class WgetToggleConfig(ConfigSet):
section: ConfigSectionName = 'ARCHIVE_METHOD_TOGGLES'
SAVE_WGET: bool = True
SAVE_WARC: bool = True
class WgetDependencyConfig(ConfigSet):
section: ConfigSectionName = 'DEPENDENCY_CONFIG'
WGET_BINARY: str = Field(default='wget')
WGET_ARGS: Optional[List[str]] = Field(default=None)
WGET_EXTRA_ARGS: List[str] = []
WGET_DEFAULT_ARGS: List[str] = ['--timeout={TIMEOUT-10}']
class WgetOptionsConfig(ConfigSet):
section: ConfigSectionName = 'ARCHIVE_METHOD_OPTIONS'
# loaded from shared config
WGET_AUTO_COMPRESSION: bool = Field(default=True)
SAVE_WGET_REQUISITES: bool = Field(default=True)
WGET_USER_AGENT: str = Field(default='', alias='USER_AGENT')
WGET_TIMEOUT: int = Field(default=60, alias='TIMEOUT')
WGET_CHECK_SSL_VALIDITY: bool = Field(default=True, alias='CHECK_SSL_VALIDITY')
WGET_RESTRICT_FILE_NAMES: str = Field(default='windows', alias='RESTRICT_FILE_NAMES')
WGET_COOKIES_FILE: Optional[Path] = Field(default=None, alias='COOKIES_FILE')
CONFIG = {
'CHECK_SSL_VALIDITY': False,
'SAVE_WARC': False,
'TIMEOUT': 999,
}
WGET_CONFIG = [
WgetToggleConfig(**CONFIG),
WgetDependencyConfig(**CONFIG),
WgetOptionsConfig(**CONFIG),
]

38
archivebox/plugantic/migrations/0001_initial.py
View file

@ -1,38 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 00:16
import abid_utils.models
import archivebox.plugantic.plugins
import charidfield.fields
import django.core.serializers.json
import django.db.models.deletion
import django_pydantic_field.fields
import uuid
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Plugin',
fields=[
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(auto_now=True)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('uuid', models.UUIDField(blank=True, null=True, unique=True)),
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
('schema', django_pydantic_field.fields.PydanticSchemaField(config=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin)),
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
),
]

21
archivebox/plugantic/migrations/0002_alter_plugin_schema.py
View file

@ -1,21 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:16
import archivebox.plugantic.plugins
import django.core.serializers.json
import django_pydantic_field.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='plugin',
name='schema',
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=None, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.plugins.Plugin),
),
]

21
archivebox/plugantic/migrations/0003_alter_plugin_schema.py
View file

@ -1,21 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:25
import archivebox.plugantic.replayers
import django.core.serializers.json
import django_pydantic_field.fields
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0002_alter_plugin_schema'),
]
operations = [
migrations.AlterField(
model_name='plugin',
name='schema',
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default={'embed_template': 'plugins/generic_replayer/templates/embed.html', 'fullpage_template': 'plugins/generic_replayer/templates/fullpage.html', 'name': 'GenericReplayer', 'row_template': 'plugins/generic_replayer/templates/row.html', 'url_pattern': '*'}, encoder=django.core.serializers.json.DjangoJSONEncoder, schema=archivebox.plugantic.replayers.Replayer),
),
]

32
archivebox/plugantic/migrations/0004_remove_plugin_schema_plugin_configs_plugin_name.py
View file

@ -1,32 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:28
import archivebox.plugantic.configs
import django.core.serializers.json
import django_pydantic_field.compat.django
import django_pydantic_field.fields
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0003_alter_plugin_schema'),
]
operations = [
migrations.RemoveField(
model_name='plugin',
name='schema',
),
migrations.AddField(
model_name='plugin',
name='configs',
field=django_pydantic_field.fields.PydanticSchemaField(config=None, default=[], encoder=django.core.serializers.json.DjangoJSONEncoder, schema=django_pydantic_field.compat.django.GenericContainer(list, (archivebox.plugantic.configs.ConfigSet,))),
),
migrations.AddField(
model_name='plugin',
name='name',
field=models.CharField(default='name', max_length=64, unique=True),
preserve_default=False,
),
]

39
archivebox/plugantic/migrations/0005_customplugin_delete_plugin.py
View file

@ -1,39 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:42
import abid_utils.models
import charidfield.fields
import django.db.models.deletion
import pathlib
import uuid
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0004_remove_plugin_schema_plugin_configs_plugin_name'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='CustomPlugin',
fields=[
('created', models.DateTimeField(auto_now_add=True)),
('modified', models.DateTimeField(auto_now=True)),
('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
('uuid', models.UUIDField(blank=True, null=True, unique=True)),
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='plg_', unique=True)),
('name', models.CharField(max_length=64, unique=True)),
('path', models.FilePathField(path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'))),
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'abstract': False,
},
),
migrations.DeleteModel(
name='Plugin',
),
]

19
archivebox/plugantic/migrations/0006_alter_customplugin_path.py
View file

@ -1,19 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:45
import pathlib
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0005_customplugin_delete_plugin'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/archivebox/plugins'), recursive=True),
),
]

19
archivebox/plugantic/migrations/0007_alter_customplugin_path.py
View file

@ -1,19 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:46
import pathlib
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0006_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins'), recursive=True),
),
]

19
archivebox/plugantic/migrations/0008_alter_customplugin_path.py
View file

@ -1,19 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:47
import pathlib
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0007_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, path=pathlib.PurePosixPath('/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data'), recursive=True),
),
]

18
archivebox/plugantic/migrations/0009_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0008_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
),
]

18
archivebox/plugantic/migrations/0010_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0009_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, match='/plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
),
]

18
archivebox/plugantic/migrations/0011_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:48
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0010_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
),
]

18
archivebox/plugantic/migrations/0012_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0011_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, default='example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
),
]

18
archivebox/plugantic/migrations/0013_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:49
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0012_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='plugins/*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data', recursive=True),
),
]

18
archivebox/plugantic/migrations/0014_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0013_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, default='/plugins/example_plugin', match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
),
]

18
archivebox/plugantic/migrations/0015_alter_customplugin_path.py
View file

@ -1,18 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:51
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0014_alter_customplugin_path'),
]
operations = [
migrations.AlterField(
model_name='customplugin',
name='path',
field=models.FilePathField(allow_files=False, allow_folders=True, match='*', path='/Volumes/NVME/Users/squash/Local/Code/archiveboxes/ArchiveBox/data/plugins', recursive=True),
),
]

16
archivebox/plugantic/migrations/0016_delete_customplugin.py
View file

@ -1,16 +0,0 @@
# Generated by Django 5.0.6 on 2024-05-18 01:57
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('plugantic', '0015_alter_customplugin_path'),
]
operations = [
migrations.DeleteModel(
name='CustomPlugin',
),
]

122
archivebox/plugantic/plugins.py
View file

@ -1,122 +0,0 @@
__package__ = 'archivebox.plugantic'
from typing import List
from typing_extensions import Self
from pydantic import (
BaseModel,
ConfigDict,
Field,
model_validator,
validate_call,
SerializeAsAny,
)
from .binaries import (
Binary,
WgetBinary,
YtdlpBinary,
)
from .extractors import (
Extractor,
YtdlpExtractor,
WgetExtractor,
WarcExtractor,
)
from .replayers import (
Replayer,
MEDIA_REPLAYER,
)
from .configs import (
ConfigSet,
WGET_CONFIG,
)
class Plugin(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
name: str = Field(default='baseplugin') # e.g. media
description: str = Field(default='') # e.g. get media using yt-dlp
configs: List[SerializeAsAny[ConfigSet]] = Field(default=[])
binaries: List[SerializeAsAny[Binary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
extractors: List[SerializeAsAny[Extractor]] = Field(default=[])
replayers: List[SerializeAsAny[Replayer]] = Field(default=[])
@model_validator(mode='after')
def validate(self):
self.description = self.description or self.name
@validate_call
def install(self) -> Self:
new_binaries = []
for idx, binary in enumerate(self.binaries):
new_binaries.append(binary.install() or binary)
return self.model_copy(update={
'binaries': new_binaries,
})
@validate_call
def load(self, cache=True) -> Self:
new_binaries = []
for idx, binary in enumerate(self.binaries):
new_binaries.append(binary.load(cache=cache) or binary)
return self.model_copy(update={
'binaries': new_binaries,
})
@validate_call
def load_or_install(self, cache=True) -> Self:
new_binaries = []
for idx, binary in enumerate(self.binaries):
new_binaries.append(binary.load_or_install(cache=cache) or binary)
return self.model_copy(update={
'binaries': new_binaries,
})
class YtdlpPlugin(Plugin):
name: str = 'ytdlp'
configs: List[SerializeAsAny[ConfigSet]] = []
binaries: List[SerializeAsAny[Binary]] = [YtdlpBinary()]
extractors: List[SerializeAsAny[Extractor]] = [YtdlpExtractor()]
replayers: List[SerializeAsAny[Replayer]] = [MEDIA_REPLAYER]
class WgetPlugin(Plugin):
name: str = 'wget'
configs: List[SerializeAsAny[ConfigSet]] = [*WGET_CONFIG]
binaries: List[SerializeAsAny[Binary]] = [WgetBinary()]
extractors: List[SerializeAsAny[Extractor]] = [WgetExtractor(), WarcExtractor()]
YTDLP_PLUGIN = YtdlpPlugin()
WGET_PLUGIN = WgetPlugin()
PLUGINS = [
YTDLP_PLUGIN,
WGET_PLUGIN,
]
LOADED_PLUGINS = PLUGINS
import json
for plugin in PLUGINS:
try:
json.dumps(plugin.model_json_schema(), indent=4)
# print(json.dumps(plugin.model_json_schema(), indent=4))
except Exception as err:
print(f'Failed to generate JSON schema for {plugin.name}')
raise
# print('-------------------------------------BEFORE INSTALL---------------------------------')
# for plugin in PLUGINS:
# print(plugin.model_dump_json(indent=4))
# print('-------------------------------------DURING LOAD/INSTALL---------------------------------')
# for plugin in PLUGINS:
# LOADED_PLUGINS.append(plugin.install())
# print('-------------------------------------AFTER INSTALL---------------------------------')
# for plugin in LOADED_PLUGINS:
# print(plugin.model_dump_json(indent=4))

62
archivebox/plugantic/views.py
View file

@ -4,17 +4,19 @@ import inspect
from typing import Any
from django.http import HttpRequest
from django.conf import settings
from django.utils.html import format_html, mark_safe
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
from plugantic.plugins import LOADED_PLUGINS
from django.conf import settings
def obj_to_yaml(obj: Any, indent: int=0) -> str:
indent_str = " " * indent
if indent == 0:
indent_str = '\n' # put extra newline between top-level entries
if isinstance(obj, dict):
if not obj:
@ -74,22 +76,34 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
if '_BINARY' in key or '_VERSION' in key
}
for plugin in LOADED_PLUGINS:
for plugin in settings.PLUGINS.values():
for binary in plugin.binaries:
binary = binary.load_or_install()
try:
binary = binary.load()
except Exception as e:
print(e)
rows['Binary'].append(ItemLink(binary.name, key=binary.name))
rows['Found Version'].append(binary.loaded_version)
rows['Found Version'].append(f'✅ {binary.loaded_version}' if binary.loaded_version else '❌ missing')
rows['From Plugin'].append(plugin.name)
rows['Provided By'].append(binary.loaded_provider)
rows['Found Abspath'].append(binary.loaded_abspath)
rows['Provided By'].append(
', '.join(
f'[{binprovider.name}]' if binprovider.name == getattr(binary.loaded_binprovider, 'name', None) else binprovider.name
for binprovider in binary.binproviders_supported
if binprovider
)
# binary.loaded_binprovider.name
# if binary.loaded_binprovider else
# ', '.join(getattr(provider, 'name', str(provider)) for provider in binary.binproviders_supported)
)
rows['Found Abspath'].append(binary.loaded_abspath or '❌ missing')
rows['Related Configuration'].append(mark_safe(', '.join(
f'<a href="/admin/environment/config/{config_key}/">{config_key}</a>'
for config_key, config_value in relevant_configs.items()
if binary.name.lower().replace('-', '').replace('_', '').replace('ytdlp', 'youtubedl') in config_key.lower()
# or binary.name.lower().replace('-', '').replace('_', '') in str(config_value).lower()
)))
rows['Overrides'].append(obj_to_yaml(binary.provider_overrides))
rows['Overrides'].append(str(obj_to_yaml(binary.provider_overrides))[:200])
# rows['Description'].append(binary.description)
return TableContext(
@ -104,7 +118,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
binary = None
plugin = None
for loaded_plugin in LOADED_PLUGINS:
for loaded_plugin in settings.PLUGINS.values():
for loaded_binary in loaded_plugin.binaries:
if loaded_binary.name == key:
binary = loaded_binary
@ -112,7 +126,10 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert plugin and binary, f'Could not find a binary matching the specified name: {key}'
binary = binary.load_or_install()
try:
binary = binary.load()
except Exception as e:
print(e)
return ItemContext(
slug=key,
@ -120,14 +137,14 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
data=[
{
"name": binary.name,
"description": binary.description,
"description": binary.abspath,
"fields": {
'plugin': plugin.name,
'binprovider': binary.loaded_provider,
'binprovider': binary.loaded_binprovider,
'abspath': binary.loaded_abspath,
'version': binary.loaded_version,
'overrides': obj_to_yaml(binary.provider_overrides),
'providers': obj_to_yaml(binary.providers_supported),
'providers': obj_to_yaml(binary.binproviders_supported),
},
"help_texts": {
# TODO
@ -148,12 +165,15 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
"extractors": [],
"replayers": [],
"configs": [],
"description": [],
"verbose_name": [],
}
for plugin in LOADED_PLUGINS:
plugin = plugin.load_or_install()
for plugin in settings.PLUGINS.values():
try:
plugin = plugin.load_binaries()
except Exception as e:
print(e)
rows['Name'].append(ItemLink(plugin.name, key=plugin.name))
rows['binaries'].append(mark_safe(', '.join(
@ -168,7 +188,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
for config_key in configset.__fields__.keys()
if config_key != 'section' and config_key in settings.CONFIG
)))
rows['description'].append(str(plugin.description))
rows['verbose_name'].append(str(plugin.verbose_name))
return TableContext(
title="Installed plugins",
@ -181,13 +201,16 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
plugin = None
for loaded_plugin in LOADED_PLUGINS:
for loaded_plugin in settings.PLUGINS.values():
if loaded_plugin.name == key:
plugin = loaded_plugin
assert plugin, f'Could not find a plugin matching the specified name: {key}'
plugin = plugin.load_or_install()
try:
plugin = plugin.load_binaries()
except Exception as e:
print(e)
return ItemContext(
slug=key,
@ -195,12 +218,13 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
data=[
{
"name": plugin.name,
"description": plugin.description,
"description": plugin.verbose_name,
"fields": {
'configs': plugin.configs,
'binaries': plugin.binaries,
'extractors': plugin.extractors,
'replayers': plugin.replayers,
'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', settings.PLUGIN_KEYS.keys()))),
},
"help_texts": {
# TODO

2
archivebox/vendor/pydantic-pkgr vendored

@ -1 +1 @@
Subproject commit c97de57f8df5f36a0f8cd1e51645f114e74bffb0
Subproject commit 36aaa4f9098e5987e23394398aa56154582bd2d2