change plugins to have both a .register that runs at import and .ready that runs later

This commit is contained in:
Nick Sweeting 2024-09-10 00:00:41 -07:00
parent f1cca5bbba
commit 4df90fbb40
No known key found for this signature in database
6 changed files with 76 additions and 12 deletions

View file

@ -3,6 +3,7 @@ __package__ = 'archivebox.builtin_plugins.npm'
from typing import List, Optional
from pydantic import InstanceOf, Field
from django.conf import settings
from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
from plugantic.base_plugin import BasePlugin
@ -65,4 +66,5 @@ class NpmPlugin(BasePlugin):
PLUGIN = NpmPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig

View file

@ -9,6 +9,7 @@ import django
from django.db.backends.sqlite3.base import Database as sqlite3 # type: ignore[import-type]
from django.core.checks import Error, Tags
from django.conf import settings
from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
from plugantic.base_plugin import BasePlugin
@ -139,4 +140,5 @@ class PipPlugin(BasePlugin):
]
PLUGIN = PipPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig

View file

@ -1,6 +1,8 @@
from pathlib import Path
from typing import List, Dict, Optional
from django.conf import settings
# Depends on other PyPI/vendor packages:
from pydantic import InstanceOf, Field
from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
@ -101,9 +103,11 @@ class SinglefilePlugin(BasePlugin):
SINGLEFILE_CONFIG,
SINGLEFILE_BINARY,
SINGLEFILE_EXTRACTOR,
SINGLEFILE_QUEUE,
]
PLUGIN = SinglefilePlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig

View file

@ -2,6 +2,7 @@ from typing import List, Dict
from subprocess import run, PIPE
from pydantic import InstanceOf, Field
from django.conf import settings
from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
from plugantic.base_plugin import BasePlugin
@ -74,4 +75,5 @@ class YtdlpPlugin(BasePlugin):
PLUGIN = YtdlpPlugin()
PLUGIN.register(settings)
DJANGO_APP = PLUGIN.AppConfig

View file

@ -1,14 +1,15 @@
__package__ = 'archivebox.plugantic'
import json
import inspect
from huey.api import TaskWrapper
from pathlib import Path
from typing import List, Literal
from pydantic import BaseModel, ConfigDict, Field, computed_field
HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
class BaseHook(BaseModel):
"""
@ -56,10 +57,14 @@ class BaseHook(BaseModel):
validate_defaults=True,
validate_assignment=False,
revalidate_instances="subclass-instances",
ignored_types=(TaskWrapper, ),
)
# verbose_name: str = Field()
is_registered: bool = False
is_ready: bool = False
@computed_field
@property
@ -69,12 +74,21 @@ class BaseHook(BaseModel):
@computed_field
@property
def hook_module(self) -> str:
"""e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
return f'{self.__module__}.{self.__class__.__name__}'
@property
def plugin_module(self) -> str:
"""e.g. builtin_plugins.singlefile"""
return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
@computed_field
@property
def plugin_dir(self) -> Path:
return Path(inspect.getfile(self.__class__)).parent.resolve()
hook_type: HookType = Field()
def register(self, settings, parent_plugin=None):
"""Load a record of an installed hook into global Django settings.HOOKS at runtime."""
self._plugin = parent_plugin # for debugging only, never rely on this!
@ -84,4 +98,19 @@ class BaseHook(BaseModel):
# record installed hook in settings.HOOKS
settings.HOOKS[self.id] = self
if settings.HOOKS[self.id].is_registered:
raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
settings.HOOKS[self.id].is_registered = True
# print("REGISTERED HOOK:", self.hook_module)
def ready(self, settings):
"""Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS."
if settings.HOOKS[self.id].is_ready:
raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!")
settings.HOOKS[self.id].is_ready = True

View file

@ -34,6 +34,9 @@ class BasePlugin(BaseModel):
# All the hooks the plugin will install:
hooks: List[InstanceOf[BaseHook]] = Field(default=[])
is_registered: bool = False
is_ready: bool = False
@computed_field
@property
def id(self) -> str:
@ -81,7 +84,7 @@ class BasePlugin(BaseModel):
def ready(self):
from django.conf import settings
plugin_self.register(settings)
plugin_self.ready(settings)
return PluginAppConfig
@ -97,9 +100,8 @@ class BasePlugin(BaseModel):
hooks[hook.hook_type][hook.id] = hook
return hooks
def register(self, settings=None):
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called)."""
if settings is None:
from django.conf import settings as django_settings
@ -112,11 +114,34 @@ class BasePlugin(BaseModel):
### Mutate django.conf.settings... values in-place to include plugin-provided overrides
settings.PLUGINS[self.id] = self
if settings.PLUGINS[self.id].is_registered:
raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!")
for hook in self.hooks:
hook.register(settings, parent_plugin=self)
settings.PLUGINS[self.id].is_registered = True
# print('√ REGISTERED PLUGIN:', self.plugin_module)
def ready(self, settings=None):
"""Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
if settings is None:
from django.conf import settings as django_settings
settings = django_settings
assert (
self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered
), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS."
if settings.PLUGINS[self.id].is_ready:
raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!")
for hook in self.hooks:
hook.ready(settings)
settings.PLUGINS[self.id].is_ready = True
# @validate_call
# def install_binaries(self) -> Self:
# new_binaries = []