mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 06:34:16 +00:00
cleanup plugantic and pkg apps, make BaseHook actually create its own settings
This commit is contained in:
Nick Sweeting
parent
0e79a8b683
commit
b56b1cac35
29 changed files with 272 additions and 466 deletions
|
|
@ -89,6 +89,8 @@ class ABIDModel(models.Model):
|
|||
# created_at = AutoDateTimeField(default=None, null=False, db_index=True)
|
||||
# modified_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
_prefetched_objects_cache: Dict[str, Any]
|
||||
|
||||
class Meta(TypedModelMeta):
|
||||
abstract = True
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,14 @@
|
|||
__package__ = 'archivebox.builtin_plugins.npm'
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from typing import List, Optional
|
||||
from pydantic import InstanceOf, Field
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.conf import settings
|
||||
|
||||
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
|
||||
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
|
||||
from plugantic.base_configset import ConfigSectionName
|
||||
|
||||
from pkg.settings import env, apt, brew
|
||||
from plugantic.base_plugin import BasePlugin
|
||||
from plugantic.base_configset import BaseConfigSet, ConfigSectionName
|
||||
from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
|
||||
from plugantic.base_hook import BaseHook
|
||||
|
||||
from ...config import CONFIG
|
||||
|
||||
|
|
@ -33,10 +30,11 @@ DEFAULT_GLOBAL_CONFIG = {
|
|||
NPM_CONFIG = NpmDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
|
||||
|
||||
|
||||
class NpmProvider(NpmProvider, BaseBinProvider):
|
||||
class CustomNpmProvider(NpmProvider, BaseBinProvider):
|
||||
PATH: PATHStr = str(CONFIG.NODE_BIN_PATH)
|
||||
|
||||
npm = NpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
|
||||
NPM_BINPROVIDER = CustomNpmProvider(PATH=str(CONFIG.NODE_BIN_PATH))
|
||||
npm = NPM_BINPROVIDER
|
||||
|
||||
class NpmBinary(BaseBinary):
|
||||
name: BinName = 'npm'
|
||||
|
|
@ -55,19 +53,16 @@ NODE_BINARY = NodeBinary()
|
|||
|
||||
|
||||
class NpmPlugin(BasePlugin):
|
||||
name: str = 'builtin_plugins.npm'
|
||||
app_label: str = 'npm'
|
||||
verbose_name: str = 'NPM'
|
||||
|
||||
configs: List[InstanceOf[BaseConfigSet]] = [NPM_CONFIG]
|
||||
binproviders: List[InstanceOf[BaseBinProvider]] = [npm]
|
||||
binaries: List[InstanceOf[BaseBinary]] = [NODE_BINARY, NPM_BINARY]
|
||||
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
NPM_CONFIG,
|
||||
NPM_BINPROVIDER,
|
||||
NODE_BINARY,
|
||||
NPM_BINARY,
|
||||
]
|
||||
|
||||
|
||||
PLUGIN = NpmPlugin()
|
||||
DJANGO_APP = PLUGIN.AppConfig
|
||||
# CONFIGS = PLUGIN.configs
|
||||
# BINARIES = PLUGIN.binaries
|
||||
# EXTRACTORS = PLUGIN.extractors
|
||||
# REPLAYERS = PLUGIN.replayers
|
||||
# CHECKS = PLUGIN.checks
|
||||
|
|
|
|||
|
|
@ -6,17 +6,16 @@ from typing import List, Dict, Optional
|
|||
from pydantic import InstanceOf, Field
|
||||
|
||||
import django
|
||||
from django.apps import AppConfig
|
||||
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
from django.core.checks import Error, Tags, register
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type]
|
||||
from django.core.checks import Error, Tags
|
||||
|
||||
from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
|
||||
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
|
||||
from plugantic.base_configset import ConfigSectionName
|
||||
from plugantic.base_plugin import BasePlugin
|
||||
from plugantic.base_configset import BaseConfigSet, ConfigSectionName
|
||||
from plugantic.base_check import BaseCheck
|
||||
|
||||
from pkg.settings import env, apt, brew
|
||||
from plugantic.base_binary import BaseBinary, BaseBinProvider, env, apt, brew
|
||||
from plugantic.base_hook import BaseHook
|
||||
|
||||
|
||||
###################### Config ##########################
|
||||
|
|
@ -36,15 +35,17 @@ DEFAULT_GLOBAL_CONFIG = {
|
|||
}
|
||||
PIP_CONFIG = PipDependencyConfigs(**DEFAULT_GLOBAL_CONFIG)
|
||||
|
||||
class PipProvider(PipProvider, BaseBinProvider):
|
||||
class CustomPipProvider(PipProvider, BaseBinProvider):
|
||||
PATH: PATHStr = str(Path(sys.executable).parent)
|
||||
|
||||
pip = PipProvider(PATH=str(Path(sys.executable).parent))
|
||||
|
||||
PIP_BINPROVIDER = CustomPipProvider(PATH=str(Path(sys.executable).parent))
|
||||
pip = PIP_BINPROVIDER
|
||||
|
||||
class PipBinary(BaseBinary):
|
||||
name: BinName = 'pip'
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
|
||||
|
||||
PIP_BINARY = PipBinary()
|
||||
|
||||
|
||||
|
|
@ -57,8 +58,8 @@ class PythonBinary(BaseBinary):
|
|||
binproviders_supported: List[InstanceOf[BinProvider]] = [pip, apt, brew, env]
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
'apt': {
|
||||
'subdeps': \
|
||||
lambda: 'python3 python3-minimal python3-pip python3-virtualenv',
|
||||
'packages': \
|
||||
lambda: 'python3 python3-minimal python3-pip python3-setuptools python3-virtualenv',
|
||||
'abspath': \
|
||||
lambda: sys.executable,
|
||||
'version': \
|
||||
|
|
@ -66,6 +67,8 @@ class PythonBinary(BaseBinary):
|
|||
},
|
||||
}
|
||||
|
||||
PYTHON_BINARY = PythonBinary()
|
||||
|
||||
class SqliteBinary(BaseBinary):
|
||||
name: BinName = 'sqlite'
|
||||
binproviders_supported: List[InstanceOf[BaseBinProvider]] = Field(default=[pip])
|
||||
|
|
@ -78,6 +81,8 @@ class SqliteBinary(BaseBinary):
|
|||
},
|
||||
}
|
||||
|
||||
SQLITE_BINARY = SqliteBinary()
|
||||
|
||||
|
||||
class DjangoBinary(BaseBinary):
|
||||
name: BinName = 'django'
|
||||
|
|
@ -92,12 +97,12 @@ class DjangoBinary(BaseBinary):
|
|||
},
|
||||
}
|
||||
|
||||
|
||||
DJANGO_BINARY = DjangoBinary()
|
||||
|
||||
|
||||
class CheckUserIsNotRoot(BaseCheck):
|
||||
label: str = 'CheckUserIsNotRoot'
|
||||
tag = Tags.database
|
||||
tag: str = Tags.database
|
||||
|
||||
@staticmethod
|
||||
def check(settings, logger) -> List[Warning]:
|
||||
|
|
@ -114,23 +119,22 @@ class CheckUserIsNotRoot(BaseCheck):
|
|||
return errors
|
||||
|
||||
|
||||
USER_IS_NOT_ROOT_CHECK = CheckUserIsNotRoot()
|
||||
|
||||
|
||||
class PipPlugin(BasePlugin):
|
||||
name: str = 'builtin_plugins.pip'
|
||||
app_label: str = 'pip'
|
||||
verbose_name: str = 'PIP'
|
||||
|
||||
configs: List[InstanceOf[BaseConfigSet]] = [PIP_CONFIG]
|
||||
binproviders: List[InstanceOf[BaseBinProvider]] = [pip]
|
||||
binaries: List[InstanceOf[BaseBinary]] = [PIP_BINARY, PythonBinary(), SqliteBinary(), DjangoBinary()]
|
||||
checks: List[InstanceOf[BaseCheck]] = [CheckUserIsNotRoot()]
|
||||
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
PIP_CONFIG,
|
||||
PIP_BINPROVIDER,
|
||||
PIP_BINARY,
|
||||
PYTHON_BINARY,
|
||||
SQLITE_BINARY,
|
||||
DJANGO_BINARY,
|
||||
USER_IS_NOT_ROOT_CHECK,
|
||||
]
|
||||
|
||||
PLUGIN = PipPlugin()
|
||||
DJANGO_APP = PLUGIN.AppConfig
|
||||
# CONFIGS = PLUGIN.configs
|
||||
# BINARIES = PLUGIN.binaries
|
||||
# EXTRACTORS = PLUGIN.extractors
|
||||
# REPLAYERS = PLUGIN.replayers
|
||||
# CHECKS = PLUGIN.checks
|
||||
|
|
|
|||
|
|
@ -1,19 +1,18 @@
|
|||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
# Depends on other PyPI/vendor packages:
|
||||
from pydantic import InstanceOf, Field
|
||||
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
|
||||
from pydantic_pkgr.binprovider import bin_abspath
|
||||
|
||||
# Depends on other Django apps:
|
||||
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseExtractor, BaseReplayer
|
||||
from plugantic.base_configset import ConfigSectionName
|
||||
from plugantic.base_plugin import BasePlugin
|
||||
from plugantic.base_configset import BaseConfigSet, ConfigSectionName
|
||||
from plugantic.base_binary import BaseBinary, env
|
||||
from plugantic.base_extractor import BaseExtractor
|
||||
from plugantic.base_hook import BaseHook
|
||||
|
||||
# Depends on Other Plugins:
|
||||
from pkg.settings import env
|
||||
from builtin_plugins.npm.apps import npm
|
||||
|
||||
|
||||
|
|
@ -54,11 +53,7 @@ DEFAULT_GLOBAL_CONFIG = {
|
|||
'TIMEOUT': 120,
|
||||
}
|
||||
|
||||
SINGLEFILE_CONFIGS = [
|
||||
SinglefileToggleConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
SinglefileDependencyConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
SinglefileOptionsConfigs(**DEFAULT_GLOBAL_CONFIG),
|
||||
]
|
||||
SINGLEFILE_CONFIG = SinglefileConfigs(**DEFAULT_GLOBAL_CONFIG)
|
||||
|
||||
|
||||
|
||||
|
|
@ -79,7 +74,7 @@ class SinglefileBinary(BaseBinary):
|
|||
# },
|
||||
# 'npm': {
|
||||
# 'abspath': lambda: bin_abspath('single-file', PATH=npm.PATH) or bin_abspath('single-file-node.js', PATH=npm.PATH),
|
||||
# 'subdeps': lambda: f'single-file-cli@>={min_version} <{max_version}',
|
||||
# 'packages': lambda: f'single-file-cli@>={min_version} <{max_version}',
|
||||
# },
|
||||
}
|
||||
|
||||
|
|
@ -99,20 +94,16 @@ SINGLEFILE_BINARY = SinglefileBinary()
|
|||
SINGLEFILE_EXTRACTOR = SinglefileExtractor()
|
||||
|
||||
class SinglefilePlugin(BasePlugin):
|
||||
name: str = 'builtin_plugins.singlefile'
|
||||
app_label: str ='singlefile'
|
||||
verbose_name: str = 'SingleFile'
|
||||
|
||||
configs: List[InstanceOf[BaseConfigSet]] = SINGLEFILE_CONFIGS
|
||||
binaries: List[InstanceOf[BaseBinary]] = [SINGLEFILE_BINARY]
|
||||
extractors: List[InstanceOf[BaseExtractor]] = [SINGLEFILE_EXTRACTOR]
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
SINGLEFILE_CONFIG,
|
||||
SINGLEFILE_BINARY,
|
||||
SINGLEFILE_EXTRACTOR,
|
||||
]
|
||||
|
||||
|
||||
|
||||
PLUGIN = SinglefilePlugin()
|
||||
DJANGO_APP = PLUGIN.AppConfig
|
||||
# CONFIGS = PLUGIN.configs
|
||||
# BINARIES = PLUGIN.binaries
|
||||
# EXTRACTORS = PLUGIN.extractors
|
||||
# REPLAYERS = PLUGIN.replayers
|
||||
# CHECKS = PLUGIN.checks
|
||||
|
|
|
|||
|
|
@ -1,17 +1,13 @@
|
|||
import sys
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from subprocess import run, PIPE, CompletedProcess
|
||||
from typing import List, Dict
|
||||
from subprocess import run, PIPE
|
||||
from pydantic import InstanceOf, Field
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
from pydantic_pkgr import BinProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict
|
||||
from plugantic.base_plugin import BasePlugin, BaseConfigSet, BaseBinary, BaseBinProvider
|
||||
from plugantic.base_configset import ConfigSectionName
|
||||
|
||||
from pkg.settings import env, apt, brew
|
||||
from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
|
||||
from plugantic.base_plugin import BasePlugin
|
||||
from plugantic.base_configset import BaseConfigSet, ConfigSectionName
|
||||
from plugantic.base_binary import BaseBinary, env, apt, brew
|
||||
from plugantic.base_hook import BaseHook
|
||||
|
||||
from builtin_plugins.pip.apps import pip
|
||||
|
||||
|
|
@ -67,12 +63,14 @@ FFMPEG_BINARY = FfmpegBinary()
|
|||
|
||||
|
||||
class YtdlpPlugin(BasePlugin):
|
||||
name: str = 'builtin_plugins.ytdlp'
|
||||
app_label: str = 'ytdlp'
|
||||
verbose_name: str = 'YTDLP'
|
||||
|
||||
configs: List[InstanceOf[BaseConfigSet]] = [YTDLP_CONFIG]
|
||||
binaries: List[InstanceOf[BaseBinary]] = [YTDLP_BINARY, FFMPEG_BINARY]
|
||||
hooks: List[InstanceOf[BaseHook]] = [
|
||||
YTDLP_CONFIG,
|
||||
YTDLP_BINARY,
|
||||
FFMPEG_BINARY,
|
||||
]
|
||||
|
||||
|
||||
PLUGIN = YtdlpPlugin()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
__package__ = 'archivebox.core'
|
||||
|
||||
|
||||
from typing import Optional, List, Dict, Iterable
|
||||
from typing import Optional, Dict, Iterable
|
||||
from django_stubs_ext.db.models import TypedModelMeta
|
||||
|
||||
import json
|
||||
|
|
@ -9,7 +9,6 @@ import json
|
|||
from pathlib import Path
|
||||
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.utils.functional import cached_property
|
||||
from django.utils.text import slugify
|
||||
from django.core.cache import cache
|
||||
|
|
@ -107,7 +106,7 @@ class Tag(ABIDModel):
|
|||
|
||||
@property
|
||||
def api_docs_url(self) -> str:
|
||||
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
|
||||
return '/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
|
||||
|
||||
class SnapshotTag(models.Model):
|
||||
id = models.AutoField(primary_key=True)
|
||||
|
|
@ -215,7 +214,7 @@ class Snapshot(ABIDModel):
|
|||
|
||||
@property
|
||||
def api_docs_url(self) -> str:
|
||||
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
|
||||
return '/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
|
||||
|
||||
def get_absolute_url(self):
|
||||
return f'/{self.archive_path}'
|
||||
|
|
@ -315,7 +314,7 @@ class Snapshot(ABIDModel):
|
|||
def latest_title(self) -> Optional[str]:
|
||||
if self.title:
|
||||
return self.title # whoopdedoo that was easy
|
||||
|
||||
|
||||
# check if ArchiveResult set has already been prefetched, if so use it instead of fetching it from db again
|
||||
if hasattr(self, '_prefetched_objects_cache') and 'archiveresult_set' in self._prefetched_objects_cache:
|
||||
try:
|
||||
|
|
@ -329,7 +328,7 @@ class Snapshot(ABIDModel):
|
|||
) or [None])[-1]
|
||||
except IndexError:
|
||||
pass
|
||||
|
||||
|
||||
|
||||
try:
|
||||
# take longest successful title from ArchiveResult db history
|
||||
|
|
@ -395,7 +394,7 @@ class Snapshot(ABIDModel):
|
|||
class ArchiveResultManager(models.Manager):
|
||||
def indexable(self, sorted: bool = True):
|
||||
"""Return only ArchiveResults containing text suitable for full-text search (sorted in order of typical result quality)"""
|
||||
|
||||
|
||||
INDEXABLE_METHODS = [ r[0] for r in ARCHIVE_METHODS_INDEXING_PRECEDENCE ]
|
||||
qs = self.get_queryset().filter(extractor__in=INDEXABLE_METHODS, status='succeeded')
|
||||
|
||||
|
|
@ -466,7 +465,7 @@ class ArchiveResult(ABIDModel):
|
|||
class Meta(TypedModelMeta):
|
||||
verbose_name = 'Archive Result'
|
||||
verbose_name_plural = 'Archive Results Log'
|
||||
|
||||
|
||||
|
||||
def __str__(self):
|
||||
# return f'[{self.abid}] 📅 {self.start_ts.strftime("%Y-%m-%d %H:%M")} 📄 {self.extractor} {self.snapshot.url}'
|
||||
|
|
@ -480,11 +479,11 @@ class ArchiveResult(ABIDModel):
|
|||
def api_url(self) -> str:
|
||||
# /api/v1/core/archiveresult/{uulid}
|
||||
return reverse_lazy('api-1:get_archiveresult', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
|
||||
|
||||
|
||||
@property
|
||||
def api_docs_url(self) -> str:
|
||||
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
|
||||
|
||||
return '/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
|
||||
|
||||
def get_absolute_url(self):
|
||||
return f'/{self.snapshot.archive_path}/{self.output_path()}'
|
||||
|
||||
|
|
|
|||
|
|
@ -40,27 +40,18 @@ INSTALLED_PLUGINS = {
|
|||
**find_plugins_in_dir(USERDATA_PLUGINS_DIR, prefix='user_plugins'),
|
||||
}
|
||||
|
||||
### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
|
||||
### Plugins Globals (filled by builtin_plugins.npm.apps.NpmPlugin.register() after Django startup)
|
||||
PLUGINS = AttrDict({})
|
||||
HOOKS = AttrDict({})
|
||||
|
||||
CONFIGS = AttrDict({})
|
||||
BINPROVIDERS = AttrDict({})
|
||||
BINARIES = AttrDict({})
|
||||
EXTRACTORS = AttrDict({})
|
||||
REPLAYERS = AttrDict({})
|
||||
CHECKS = AttrDict({})
|
||||
ADMINDATAVIEWS = AttrDict({})
|
||||
# CONFIGS = AttrDict({})
|
||||
# BINPROVIDERS = AttrDict({})
|
||||
# BINARIES = AttrDict({})
|
||||
# EXTRACTORS = AttrDict({})
|
||||
# REPLAYERS = AttrDict({})
|
||||
# CHECKS = AttrDict({})
|
||||
# ADMINDATAVIEWS = AttrDict({})
|
||||
|
||||
PLUGIN_KEYS = AttrDict({
|
||||
'CONFIGS': CONFIGS,
|
||||
'BINPROVIDERS': BINPROVIDERS,
|
||||
'BINARIES': BINARIES,
|
||||
'EXTRACTORS': EXTRACTORS,
|
||||
'REPLAYERS': REPLAYERS,
|
||||
'CHECKS': CHECKS,
|
||||
'ADMINDATAVIEWS': ADMINDATAVIEWS,
|
||||
})
|
||||
|
||||
################################################################################
|
||||
### Django Core Settings
|
||||
|
|
@ -95,12 +86,11 @@ INSTALLED_APPS = [
|
|||
'signal_webhooks', # handles REST API outbound webhooks https://github.com/MrThearMan/django-signal-webhooks
|
||||
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
|
||||
|
||||
# our own apps
|
||||
# Our ArchiveBox-provided apps
|
||||
'abid_utils', # handles ABID ID creation, handling, and models
|
||||
'plugantic', # ArchiveBox plugin API definition + finding/registering/calling interface
|
||||
'core', # core django model with Snapshot, ArchiveResult, etc.
|
||||
'api', # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
|
||||
'pkg', # ArchiveBox runtime package management interface for subdependencies
|
||||
|
||||
# ArchiveBox plugins
|
||||
*INSTALLED_PLUGINS.keys(), # all plugin django-apps found in archivebox/builtin_plugins and data/user_plugins
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
from django.contrib import admin
|
||||
|
||||
# Register your models here.
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
__package__ = 'archivebox.pkg'
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class PkgsConfig(AppConfig):
|
||||
name = 'pkg'
|
||||
verbose_name = 'Package Management'
|
||||
|
||||
default_auto_field = 'django.db.models.BigAutoField'
|
||||
|
||||
def ready(self):
|
||||
from .settings import LOADED_DEPENDENCIES
|
||||
|
||||
# print(LOADED_DEPENDENCIES)
|
||||
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from django.db import models
|
||||
|
||||
# Create your models here.
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
__package__ = 'archivebox.pkg'
|
||||
|
||||
import os
|
||||
import sys
|
||||
import shutil
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
|
||||
import django
|
||||
from django.conf import settings
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
|
||||
from pydantic_pkgr import Binary, BinProvider, BrewProvider, PipProvider, NpmProvider, AptProvider, EnvProvider, SemVer
|
||||
from pydantic_pkgr.binprovider import bin_abspath
|
||||
|
||||
from ..config import NODE_BIN_PATH, bin_path
|
||||
|
||||
apt = AptProvider()
|
||||
brew = BrewProvider()
|
||||
env = EnvProvider(PATH=os.environ.get('PATH', '/bin'))
|
||||
|
||||
# Defined in their own plugins:
|
||||
#pip = PipProvider(PATH=str(Path(sys.executable).parent))
|
||||
#npm = NpmProvider(PATH=NODE_BIN_PATH)
|
||||
|
||||
LOADED_DEPENDENCIES = {}
|
||||
|
||||
for bin_name, binary_spec in settings.BINARIES.items():
|
||||
try:
|
||||
settings.BINARIES[bin_name] = binary_spec.load()
|
||||
except Exception as e:
|
||||
# print(f"- ❌ Binary {bin_name} failed to load with error: {e}")
|
||||
continue
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from django.test import TestCase
|
||||
|
||||
# Create your tests here.
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
from django.shortcuts import render
|
||||
|
||||
# Create your views here.
|
||||
|
|
@ -1,8 +1,5 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import json
|
||||
import importlib
|
||||
|
||||
from django.apps import AppConfig
|
||||
|
||||
class PluganticConfig(AppConfig):
|
||||
|
|
@ -10,6 +7,6 @@ class PluganticConfig(AppConfig):
|
|||
name = 'plugantic'
|
||||
|
||||
def ready(self) -> None:
|
||||
from django.conf import settings
|
||||
|
||||
print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')
|
||||
pass
|
||||
# from django.conf import settings
|
||||
# print(f'[🧩] Detected {len(settings.INSTALLED_PLUGINS)} settings.INSTALLED_PLUGINS to load...')
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
from typing import List, Type, Any, Dict
|
||||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
from pydantic_core import core_schema
|
||||
from pydantic import GetCoreSchemaHandler, BaseModel
|
||||
from typing import Dict
|
||||
|
||||
from django.utils.functional import classproperty
|
||||
from django.core.checks import Warning, Tags, register
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
class BaseAdminDataView(BaseModel):
|
||||
name: str = 'NPM Installed Packages'
|
||||
class BaseAdminDataView(BaseHook):
|
||||
hook_type: HookType = "ADMINDATAVIEW"
|
||||
|
||||
verbose_name: str = 'NPM Installed Packages'
|
||||
route: str = '/npm/installed/'
|
||||
view: str = 'builtin_plugins.npm.admin.installed_list_view'
|
||||
items: Dict[str, str] = {
|
||||
|
|
@ -16,19 +17,22 @@ class BaseAdminDataView(BaseModel):
|
|||
'view': 'builtin_plugins.npm.admin.installed_detail_view',
|
||||
}
|
||||
|
||||
def as_route(self) -> Dict[str, str | Dict[str, str]]:
|
||||
return {
|
||||
'route': self.route,
|
||||
'view': self.view,
|
||||
'name': self.name,
|
||||
'items': self.items,
|
||||
}
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
"""Regsiter AdminDataViews.as_route() in settings.ADMIN_DATA_VIEWS.URLS at runtime"""
|
||||
self._plugin = parent_plugin # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
|
||||
# self._plugin = parent_plugin # circular ref to parent only here for easier debugging! never depend on circular backref to parent in real code!
|
||||
|
||||
route = self.as_route()
|
||||
self.register_route_in_admin_data_view_urls(settings)
|
||||
|
||||
settings.ADMINDATAVIEWS = getattr(settings, "ADMINDATAVIEWS", None) or AttrDict({})
|
||||
settings.ADMINDATAVIEWS[self.id] = self
|
||||
|
||||
super().register(settings, parent_plugin)
|
||||
|
||||
def register_route_in_admin_data_view_urls(self, settings):
|
||||
route = {
|
||||
"route": self.route,
|
||||
"view": self.view,
|
||||
"name": self.verbose_name,
|
||||
"items": self.items,
|
||||
}
|
||||
if route not in settings.ADMIN_DATA_VIEWS.URLS:
|
||||
settings.ADMIN_DATA_VIEWS.URLS += [route] # append our route (update in place)
|
||||
|
||||
settings.ADMIN_DATA_VIEWS.URLS += [route] # append our route (update in place)
|
||||
|
|
|
|||
|
|
@ -1,25 +1,18 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import sys
|
||||
import inspect
|
||||
import importlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
from typing import Any, Optional, Dict, List
|
||||
from typing_extensions import Self
|
||||
from subprocess import run, PIPE
|
||||
import os
|
||||
from typing import Dict, List
|
||||
|
||||
from pydantic import Field, InstanceOf
|
||||
from pydantic_pkgr import Binary, SemVer, BinName, BinProvider, EnvProvider, AptProvider, BrewProvider, PipProvider, BinProviderName, ProviderLookupDict
|
||||
from pydantic_pkgr.binprovider import HostBinPath
|
||||
from pydantic_pkgr import Binary, BinProvider, BinProviderName, ProviderLookupDict, AptProvider, BrewProvider, EnvProvider
|
||||
|
||||
import django
|
||||
from django.core.cache import cache
|
||||
from django.db.backends.sqlite3.base import Database as sqlite3
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
|
||||
class BaseBinProvider(BinProvider):
|
||||
class BaseBinProvider(BaseHook, BinProvider):
|
||||
hook_type: HookType = 'BINPROVIDER'
|
||||
|
||||
# def on_get_abspath(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
|
||||
# Class = super()
|
||||
# get_abspath_func = lambda: Class.on_get_abspath(bin_name, **context)
|
||||
|
|
@ -33,68 +26,30 @@ class BaseBinProvider(BinProvider):
|
|||
# return get_version_func()
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
# self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.BINPROVIDERS[self.name] = self
|
||||
settings.BINPROVIDERS = getattr(settings, "BINPROVIDERS", None) or AttrDict({})
|
||||
settings.BINPROVIDERS[self.id] = self
|
||||
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
|
||||
class BaseBinary(Binary):
|
||||
class BaseBinary(BaseHook, Binary):
|
||||
hook_type: HookType = "BINARY"
|
||||
|
||||
binproviders_supported: List[InstanceOf[BinProvider]] = Field(default_factory=list, alias='binproviders')
|
||||
provider_overrides: Dict[BinProviderName, ProviderLookupDict] = Field(default_factory=dict, alias='overrides')
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
# self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.BINARIES[self.name] = self
|
||||
settings.BINARIES = getattr(settings, "BINARIES", None) or AttrDict({})
|
||||
settings.BINARIES[self.id] = self
|
||||
|
||||
# def get_ytdlp_version() -> str:
|
||||
# import yt_dlp
|
||||
# return yt_dlp.version.__version__
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
|
||||
|
||||
|
||||
# class YtdlpBinary(Binary):
|
||||
# name: BinName = 'yt-dlp'
|
||||
# providers_supported: List[BinProvider] = [
|
||||
# EnvProvider(),
|
||||
# PipProvider(),
|
||||
# BrewProvider(),
|
||||
# AptProvider(),
|
||||
# ]
|
||||
# provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
|
||||
# 'pip': {
|
||||
# 'version': get_ytdlp_version,
|
||||
# },
|
||||
# 'brew': {
|
||||
# 'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
# },
|
||||
# 'apt': {
|
||||
# 'subdeps': lambda: 'yt-dlp ffmpeg',
|
||||
# }
|
||||
# }
|
||||
|
||||
# class WgetBinary(Binary):
|
||||
# name: BinName = 'wget'
|
||||
# providers_supported: List[BinProvider] = [EnvProvider(), AptProvider(), BrewProvider()]
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# PYTHON_BINARY = PythonBinary()
|
||||
# SQLITE_BINARY = SqliteBinary()
|
||||
# DJANGO_BINARY = DjangoBinary()
|
||||
# WGET_BINARY = WgetBinary()
|
||||
# YTDLP_BINARY = YtdlpPBinary()
|
||||
|
||||
# print('-------------------------------------DEFINING BINARIES---------------------------------')
|
||||
# print(PYTHON_BINARY)
|
||||
# print(SQLITE_BINARY)
|
||||
# print(DJANGO_BINARY)
|
||||
# print(WGET_BINARY)
|
||||
# print(YTDLP_BINARY)
|
||||
apt = AptProvider()
|
||||
brew = BrewProvider()
|
||||
env = EnvProvider(PATH=os.environ.get("PATH", "/bin"))
|
||||
|
|
|
|||
|
|
@ -1,28 +1,16 @@
|
|||
from typing import List, Type, Any
|
||||
__package__ = "archivebox.plugantic"
|
||||
|
||||
from pydantic_core import core_schema
|
||||
from pydantic import GetCoreSchemaHandler, BaseModel
|
||||
from typing import List
|
||||
|
||||
from django.utils.functional import classproperty
|
||||
from django.core.checks import Warning, Tags, register
|
||||
|
||||
class BaseCheck:
|
||||
label: str = ''
|
||||
tag: str = Tags.database
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
class BaseCheck(BaseHook):
|
||||
hook_type: HookType = "CHECK"
|
||||
|
||||
@classmethod
|
||||
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
|
||||
return core_schema.typed_dict_schema(
|
||||
{
|
||||
'name': core_schema.typed_dict_field(core_schema.str_schema()),
|
||||
'tag': core_schema.typed_dict_field(core_schema.str_schema()),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@classproperty
|
||||
def name(cls) -> str:
|
||||
return cls.label or cls.__name__
|
||||
tag: str = Tags.database
|
||||
|
||||
@staticmethod
|
||||
def check(settings, logger) -> List[Warning]:
|
||||
|
|
@ -38,18 +26,26 @@ class BaseCheck:
|
|||
return errors
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
# Regsiter in ArchiveBox plugins runtime settings
|
||||
self._plugin = parent_plugin
|
||||
settings.CHECKS[self.name] = self
|
||||
# self._plugin = parent_plugin # backref to parent is for debugging only, never rely on this!
|
||||
|
||||
self.register_with_django_check_system() # (SIDE EFFECT)
|
||||
|
||||
# install hook into settings.CHECKS
|
||||
settings.CHECKS = getattr(settings, "CHECKS", None) or AttrDict({})
|
||||
settings.CHECKS[self.id] = self
|
||||
|
||||
# record installed hook in settings.HOOKS
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
def register_with_django_check_system(self):
|
||||
|
||||
# Register using Django check framework
|
||||
def run_check(app_configs, **kwargs) -> List[Warning]:
|
||||
from django.conf import settings
|
||||
import logging
|
||||
settings = settings
|
||||
logger = logging.getLogger('checks')
|
||||
return self.check(settings, logger)
|
||||
|
||||
run_check.__name__ = self.label or self.__class__.__name__
|
||||
return self.check(settings, logging.getLogger("checks"))
|
||||
|
||||
run_check.__name__ = self.id
|
||||
run_check.tags = [self.tag]
|
||||
register(self.tag)(run_check)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ __package__ = 'archivebox.plugantic'
|
|||
|
||||
|
||||
from typing import List, Literal
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
|
||||
ConfigSectionName = Literal[
|
||||
'GENERAL_CONFIG',
|
||||
|
|
@ -21,23 +22,16 @@ ConfigSectionNames: List[ConfigSectionName] = [
|
|||
|
||||
|
||||
class BaseConfigSet(BaseHook):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
|
||||
hook_type: HookType = 'CONFIG'
|
||||
|
||||
section: ConfigSectionName = 'GENERAL_CONFIG'
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
"""Installs the ConfigSet into Django settings.CONFIGS (and settings.HOOKS)."""
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
# self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
# install hook into settings.CONFIGS
|
||||
settings.CONFIGS[self.name] = self
|
||||
settings.CONFIGS = getattr(settings, "CONFIGS", None) or AttrDict({})
|
||||
settings.CONFIGS[self.id] = self
|
||||
|
||||
# record installed hook in settings.HOOKS
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,28 +3,13 @@ __package__ = 'archivebox.plugantic'
|
|||
from typing import Optional, List, Literal, Annotated, Dict, Any
|
||||
from typing_extensions import Self
|
||||
|
||||
from abc import ABC
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import BaseModel, model_validator, field_serializer, AfterValidator, Field
|
||||
from pydantic import model_validator, AfterValidator
|
||||
from pydantic_pkgr import BinName
|
||||
|
||||
# from .binaries import (
|
||||
# Binary,
|
||||
# YtdlpBinary,
|
||||
# WgetBinary,
|
||||
# )
|
||||
|
||||
|
||||
# stubs
|
||||
class Snapshot:
|
||||
pass
|
||||
|
||||
class ArchiveResult:
|
||||
pass
|
||||
|
||||
def get_wget_output_path(*args, **kwargs) -> Path:
|
||||
return Path('.').resolve()
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
|
||||
|
||||
|
|
@ -38,7 +23,9 @@ HandlerFuncStr = Annotated[str, AfterValidator(lambda s: s.startswith('self.'))]
|
|||
CmdArgsList = Annotated[List[str], AfterValidator(no_empty_args)]
|
||||
|
||||
|
||||
class BaseExtractor(ABC, BaseModel):
|
||||
class BaseExtractor(BaseHook):
|
||||
hook_type: HookType = 'EXTRACTOR'
|
||||
|
||||
name: ExtractorName
|
||||
binary: BinName
|
||||
|
||||
|
|
@ -56,17 +43,20 @@ class BaseExtractor(ABC, BaseModel):
|
|||
if self.args is None:
|
||||
self.args = [*self.default_args, *self.extra_args]
|
||||
return self
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.EXTRACTORS[self.name] = self
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
# self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
settings.EXTRACTORS = getattr(settings, "EXTRACTORS", None) or AttrDict({})
|
||||
settings.EXTRACTORS[self.id] = self
|
||||
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
|
||||
|
||||
def get_output_path(self, snapshot) -> Path:
|
||||
return Path(self.name)
|
||||
return Path(self.id.lower())
|
||||
|
||||
def should_extract(self, snapshot) -> bool:
|
||||
output_dir = self.get_output_path(snapshot)
|
||||
|
|
@ -106,7 +96,7 @@ class BaseExtractor(ABC, BaseModel):
|
|||
# binary: Binary = YtdlpBinary()
|
||||
|
||||
# def get_output_path(self, snapshot) -> Path:
|
||||
# return Path(self.name)
|
||||
# return 'media/'
|
||||
|
||||
|
||||
# class WgetExtractor(Extractor):
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
import json
|
||||
from typing import Optional, List, Literal, ClassVar
|
||||
from pathlib import Path
|
||||
from pydantic import BaseModel, Field, ConfigDict, computed_field
|
||||
from typing import List, Literal
|
||||
from pydantic import BaseModel, ConfigDict, Field, computed_field
|
||||
|
||||
|
||||
HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
|
||||
|
|
@ -50,31 +49,39 @@ class BaseHook(BaseModel):
|
|||
|
||||
"""
|
||||
model_config = ConfigDict(
|
||||
extra='allow',
|
||||
extra="allow",
|
||||
arbitrary_types_allowed=True,
|
||||
from_attributes=True,
|
||||
populate_by_name=True,
|
||||
validate_defaults=True,
|
||||
validate_assignment=True,
|
||||
revalidate_instances="always",
|
||||
)
|
||||
|
||||
# verbose_name: str = Field()
|
||||
|
||||
hook_type: HookType = 'CONFIG'
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return f'{self.__module__}.{__class__.__name__}'
|
||||
def id(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def hook_module(self) -> str:
|
||||
return f'{self.__module__}.{self.__class__.__name__}'
|
||||
|
||||
hook_type: HookType = Field()
|
||||
|
||||
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
"""Load a record of an installed hook into global Django settings.HOOKS at runtime."""
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Hook {self.name} has invalid JSON schema.'
|
||||
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
# assert json.dumps(self.model_json_schema(), indent=4), f"Hook {self.hook_module} has invalid JSON schema."
|
||||
|
||||
# record installed hook in settings.HOOKS
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.HOOKS[self.name] = self
|
||||
settings.HOOKS[self.id] = self
|
||||
|
||||
print('REGISTERED HOOK:', self.name)
|
||||
# print("REGISTERED HOOK:", self.hook_module)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,8 @@ import inspect
|
|||
from pathlib import Path
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.core.checks import register
|
||||
|
||||
from typing import List, ClassVar, Type, Dict
|
||||
from typing import List, Type, Dict
|
||||
from typing_extensions import Self
|
||||
|
||||
from pydantic import (
|
||||
|
|
@ -20,142 +19,99 @@ from pydantic import (
|
|||
validate_call,
|
||||
)
|
||||
|
||||
from .base_configset import BaseConfigSet
|
||||
from .base_binary import BaseBinProvider, BaseBinary
|
||||
from .base_extractor import BaseExtractor
|
||||
from .base_replayer import BaseReplayer
|
||||
from .base_check import BaseCheck
|
||||
from .base_admindataview import BaseAdminDataView
|
||||
from .base_hook import BaseHook, HookType
|
||||
|
||||
from ..config import ANSI, AttrDict
|
||||
from ..config import AttrDict
|
||||
|
||||
|
||||
class BasePlugin(BaseModel):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
|
||||
|
||||
# Required by AppConfig:
|
||||
name: str = Field() # e.g. 'builtin_plugins.singlefile' (DottedImportPath)
|
||||
app_label: str = Field() # e.g. 'singlefile' (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
|
||||
verbose_name: str = Field() # e.g. 'SingleFile' (human-readable *short* label, for use in column names, form labels, etc.)
|
||||
verbose_name: str = Field() # e.g. 'SingleFile' (human-readable *short* label, for use in column names, form labels, etc.)
|
||||
|
||||
# All the hooks the plugin will install:
|
||||
configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
|
||||
binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||
binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||
extractors: List[InstanceOf[BaseExtractor]] = Field(default=[])
|
||||
replayers: List[InstanceOf[BaseReplayer]] = Field(default=[])
|
||||
checks: List[InstanceOf[BaseCheck]] = Field(default=[])
|
||||
admindataviews: List[InstanceOf[BaseAdminDataView]] = Field(default=[])
|
||||
hooks: List[InstanceOf[BaseHook]] = Field(default=[])
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def id(self) -> str:
|
||||
return self.__class__.__name__
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def plugin_module(self) -> str: # DottedImportPath
|
||||
""" "
|
||||
Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
|
||||
e.g. 'archivebox.builtin_plugins.npm.apps.NpmPlugin' -> 'builtin_plugins.npm'
|
||||
"""
|
||||
return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit('.apps.', 1)[0]
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def plugin_dir(self) -> Path:
|
||||
return Path(inspect.getfile(self.__class__)).parent.resolve()
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate(self) -> Self:
|
||||
"""Validate the plugin's build-time configuration here before it's registered in Django at runtime."""
|
||||
|
||||
assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
|
||||
assert self.app_label and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
|
||||
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f"Plugin {self.plugin_module} has invalid JSON schema."
|
||||
return self
|
||||
|
||||
@property
|
||||
def AppConfig(plugin_self) -> Type[AppConfig]:
|
||||
"""Generate a Django AppConfig class for this plugin."""
|
||||
|
||||
|
||||
class PluginAppConfig(AppConfig):
|
||||
"""Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
|
||||
name = plugin_self.name
|
||||
name = plugin_self.plugin_module
|
||||
app_label = plugin_self.app_label
|
||||
verbose_name = plugin_self.verbose_name
|
||||
|
||||
default_auto_field = 'django.db.models.AutoField'
|
||||
|
||||
|
||||
def ready(self):
|
||||
from django.conf import settings
|
||||
|
||||
# plugin_self.validate()
|
||||
plugin_self.register(settings)
|
||||
|
||||
return PluginAppConfig
|
||||
|
||||
@computed_field
|
||||
|
||||
@property
|
||||
def BINPROVIDERS(self) -> Dict[str, BaseBinProvider]:
|
||||
return AttrDict({binprovider.name: binprovider for binprovider in self.binproviders})
|
||||
|
||||
@computed_field
|
||||
def HOOKS_BY_ID(self) -> Dict[str, InstanceOf[BaseHook]]:
|
||||
return AttrDict({hook.id: hook for hook in self.hooks})
|
||||
|
||||
@property
|
||||
def BINARIES(self) -> Dict[str, BaseBinary]:
|
||||
return AttrDict({binary.python_name: binary for binary in self.binaries})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CONFIGS(self) -> Dict[str, BaseConfigSet]:
|
||||
return AttrDict({config.name: config for config in self.configs})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def EXTRACTORS(self) -> Dict[str, BaseExtractor]:
|
||||
return AttrDict({extractor.name: extractor for extractor in self.extractors})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def REPLAYERS(self) -> Dict[str, BaseReplayer]:
|
||||
return AttrDict({replayer.name: replayer for replayer in self.replayers})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def CHECKS(self) -> Dict[str, BaseCheck]:
|
||||
return AttrDict({check.name: check for check in self.checks})
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
|
||||
return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
|
||||
def HOOKS_BY_TYPE(self) -> Dict[HookType, Dict[str, InstanceOf[BaseHook]]]:
|
||||
hooks = AttrDict({})
|
||||
for hook in self.hooks:
|
||||
hooks[hook.hook_type] = hooks.get(hook.hook_type) or AttrDict({})
|
||||
hooks[hook.hook_type][hook.id] = hook
|
||||
return hooks
|
||||
|
||||
|
||||
def register(self, settings=None):
|
||||
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
|
||||
|
||||
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
|
||||
assert all(hasattr(settings, key) for key in ['PLUGINS', 'CONFIGS', 'BINARIES', 'EXTRACTORS', 'REPLAYERS', 'ADMINDATAVIEWS']), 'Tried to register plugin in settings but couldnt find required global dicts in settings.'
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.plugin_module} has invalid JSON schema.'
|
||||
|
||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
||||
|
||||
assert self.app_label not in settings.PLUGINS, f'Tried to register plugin {self.name} but it conflicts with existing plugin of the same name ({self.app_label}).'
|
||||
assert self.id not in settings.PLUGINS, f'Tried to register plugin {self.plugin_module} but it conflicts with existing plugin of the same name ({self.app_label}).'
|
||||
|
||||
### Mutate django.conf.settings... values in-place to include plugin-provided overrides
|
||||
settings.PLUGINS[self.app_label] = self
|
||||
settings.PLUGINS[self.id] = self
|
||||
|
||||
for config in self.CONFIGS.values():
|
||||
config.register(settings, parent_plugin=self)
|
||||
|
||||
for binprovider in self.BINPROVIDERS.values():
|
||||
binprovider.register(settings, parent_plugin=self)
|
||||
|
||||
for binary in self.BINARIES.values():
|
||||
binary.register(settings, parent_plugin=self)
|
||||
|
||||
for extractor in self.EXTRACTORS.values():
|
||||
extractor.register(settings, parent_plugin=self)
|
||||
for hook in self.hooks:
|
||||
hook.register(settings, parent_plugin=self)
|
||||
|
||||
for replayer in self.REPLAYERS.values():
|
||||
replayer.register(settings, parent_plugin=self)
|
||||
|
||||
for check in self.CHECKS.values():
|
||||
check.register(settings, parent_plugin=self)
|
||||
|
||||
for admindataview in self.ADMINDATAVIEWS.values():
|
||||
admindataview.register(settings, parent_plugin=self)
|
||||
|
||||
# TODO: add parsers? custom templates? persona fixtures?
|
||||
|
||||
plugin_prefix, plugin_shortname = self.name.split('.', 1)
|
||||
|
||||
print(
|
||||
f' > {ANSI.black}{plugin_prefix.upper().replace("_PLUGINS", "").ljust(15)} ' +
|
||||
f'{ANSI.lightyellow}{plugin_shortname.ljust(12)} ' +
|
||||
f'{ANSI.black}CONFIGSx{len(self.configs)} BINARIESx{len(self.binaries)} EXTRACTORSx{len(self.extractors)} REPLAYERSx{len(self.replayers)} CHECKSx{len(self.CHECKS)} ADMINDATAVIEWSx{len(self.ADMINDATAVIEWS)}{ANSI.reset}'
|
||||
)
|
||||
print('√ REGISTERED PLUGIN:', self.plugin_module)
|
||||
|
||||
# @validate_call
|
||||
# def install_binaries(self) -> Self:
|
||||
|
|
@ -169,7 +125,7 @@ class BasePlugin(BaseModel):
|
|||
@validate_call
|
||||
def load_binaries(self, cache=True) -> Self:
|
||||
new_binaries = []
|
||||
for idx, binary in enumerate(self.binaries):
|
||||
for idx, binary in enumerate(self.HOOKS_BY_TYPE['BINARY'].values()):
|
||||
new_binaries.append(binary.load(cache=cache) or binary)
|
||||
return self.model_copy(update={
|
||||
'binaries': new_binaries,
|
||||
|
|
@ -184,20 +140,6 @@ class BasePlugin(BaseModel):
|
|||
# 'binaries': new_binaries,
|
||||
# })
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def module_dir(self) -> Path:
|
||||
return Path(inspect.getfile(self.__class__)).parent.resolve()
|
||||
|
||||
@computed_field
|
||||
@property
|
||||
def module_path(self) -> str: # DottedImportPath
|
||||
""""
|
||||
Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
|
||||
e.g. 'archivebox.builtin_plugins.npm'
|
||||
"""
|
||||
return self.name.strip('archivebox.')
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
__package__ = 'archivebox.plugantic'
|
||||
|
||||
|
||||
from pydantic import BaseModel
|
||||
from .base_hook import BaseHook, HookType
|
||||
from ..config_stubs import AttrDict
|
||||
|
||||
|
||||
|
||||
class BaseReplayer(BaseModel):
|
||||
class BaseReplayer(BaseHook):
|
||||
"""Describes how to render an ArchiveResult in several contexts"""
|
||||
name: str = 'GenericReplayer'
|
||||
|
||||
hook_type: HookType = 'REPLAYER'
|
||||
|
||||
url_pattern: str = '*'
|
||||
|
||||
row_template: str = 'plugins/generic_replayer/templates/row.html'
|
||||
|
|
@ -21,13 +23,12 @@ class BaseReplayer(BaseModel):
|
|||
# thumbnail_view: LazyImportStr = 'plugins.generic_replayer.views.get_icon'
|
||||
|
||||
def register(self, settings, parent_plugin=None):
|
||||
if settings is None:
|
||||
from django.conf import settings as django_settings
|
||||
settings = django_settings
|
||||
# self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
|
||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||
settings.REPLAYERS[self.name] = self
|
||||
settings.REPLAYERS = getattr(settings, 'REPLAYERS', None) or AttrDict({})
|
||||
settings.REPLAYERS[self.id] = self
|
||||
|
||||
super().register(settings, parent_plugin=parent_plugin)
|
||||
|
||||
# class MediaReplayer(BaseReplayer):
|
||||
# name: str = 'MediaReplayer'
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
__package__ = 'archivebox.pkg.management.commands'
|
||||
__package__ = 'archivebox.plugantic.management.commands'
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.conf import settings
|
||||
|
|
@ -7,8 +7,7 @@ from pydantic_pkgr import Binary, BinProvider, BrewProvider, EnvProvider, SemVer
|
|||
from pydantic_pkgr.binprovider import bin_abspath
|
||||
|
||||
from ....config import NODE_BIN_PATH, bin_path
|
||||
|
||||
from pkg.settings import env
|
||||
from ...base_binary import env
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
|
|
@ -235,7 +235,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
|
|||
'binaries': plugin.binaries,
|
||||
'extractors': plugin.extractors,
|
||||
'replayers': plugin.replayers,
|
||||
'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', *settings.PLUGIN_KEYS.keys()))),
|
||||
'schema': obj_to_yaml(plugin.model_dump(include=('name', 'verbose_name', 'app_label', 'hooks'))),
|
||||
},
|
||||
"help_texts": {
|
||||
# TODO
|
||||
|
|
|
|||
|
|
@ -120,6 +120,9 @@ target-version = "py310"
|
|||
src = ["archivebox"]
|
||||
exclude = ["*.pyi", "typings/", "migrations/", "vendor/"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
ignore = ["E731"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = [ "tests" ]
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue