Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-21 19:53:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

fix abx handling of obj, module, and class based plugins, fix archivebox version cmd

Browse source
This commit is contained in:
Nick Sweeting 2024-11-18 18:30:28 -08:00
parent 0acd388c02
commit e50f8cb3b6
No known key found for this signature in database
3 changed files with 44 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
archivebox/pkgs/abx-spec-archivebox/abx_spec_archivebox/reads.py
View file

@ -1,33 +0,0 @@
__package__ = 'abx.archivebox'
from benedict import benedict
def get_scope_config(defaults: benedict | None = None, persona=None, seed=None, crawl=None, snapshot=None, archiveresult=None, extra_config=None):
"""Get all the relevant config for the given scope, in correct precedence order"""
from django.conf import settings
default_config: benedict = defaults or settings.CONFIG
snapshot = snapshot or (archiveresult and archiveresult.snapshot)
crawl = crawl or (snapshot and snapshot.crawl)
seed = seed or (crawl and crawl.seed)
persona = persona or (crawl and crawl.persona)
persona_config = persona.config if persona else {}
seed_config = seed.config if seed else {}
crawl_config = crawl.config if crawl else {}
snapshot_config = snapshot.config if snapshot else {}
archiveresult_config = archiveresult.config if archiveresult else {}
extra_config = extra_config or {}
return benedict({
**default_config, # defaults / config file / environment variables
**persona_config, # lowest precedence
**seed_config,
**crawl_config,
**snapshot_config,
**archiveresult_config,
**extra_config, # highest precedence
})

41
archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py
View file

@ -62,32 +62,35 @@ class ConfigPluginSpec:
@staticmethod
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_SCOPE_CONFIG(extra=None, archiveresult=None, snapshot=None, crawl=None, user=None, collection=..., environment=..., machine=..., default=...) -> dict[ConfigKeyStr, Any]:
def get_SCOPE_CONFIG(extra=None, archiveresult=None, snapshot=None, crawl=None, user=None, request=None, collection=..., environment=..., machine=..., default=...) -> dict[ConfigKeyStr, Any]:
"""Get the config as it applies to you right now, based on the current context"""
return benedict({
**pm.hook.get_default_config(default=default),
# **pm.hook.get_machine_config(machine),
**pm.hook.get_machine_config(machine=machine),
**pm.hook.get_environment_config(environment=environment),
**pm.hook.get_collection_config(collection=collection),
**pm.hook.get_user_config(user=user),
**pm.hook.get_crawl_config(crawl=crawl),
**pm.hook.get_snapshot_config(snapshot=snapshot),
**pm.hook.get_archiveresult_config(archiveresult=archiveresult),
# **pm.hook.get_request_config(request=request),
**pm.hook.get_request_config(request=request),
**(extra or {}),
})
@staticmethod
# @abx.hookspec(firstresult=True)
# @abx.hookimpl
# def get_request_config(request) -> dict:
# session = getattr(request, 'session', None)
# return getattr(session, 'config', None) or {}
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_request_config(request=None) -> dict:
if not request:
return {}
return request.session.get('config', None) or {}
@staticmethod
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_archiveresult_config(archiveresult) -> dict[ConfigKeyStr, Any]:
def get_archiveresult_config(archiveresult=None) -> dict[ConfigKeyStr, Any]:
if not archiveresult:
return {}
return getattr(archiveresult, 'config', None) or {}
@staticmethod
@ -99,7 +102,9 @@ class ConfigPluginSpec:
@staticmethod
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_crawl_config(crawl) -> dict[ConfigKeyStr, Any]:
def get_crawl_config(crawl=None) -> dict[ConfigKeyStr, Any]:
if not crawl:
return {}
return getattr(crawl, 'config', None) or {}
@staticmethod
@ -133,14 +138,14 @@ class ConfigPluginSpec:
}) if environment == ... else environment
@staticmethod
# @abx.hookspec(firstresult=True)
# @abx.hookimpl
# def get_machine_config(machine=...) -> dict:
# # ... = ellipsis, means automatically get the machine config from the currently executing machine
# # {} = empty dict, override to ignore the machine config
# if machine == ...:
# machine = Machine.objects.get_current()
# return getattr(machine, 'config', None) or {}
@abx.hookspec(firstresult=True)
@abx.hookimpl
def get_machine_config(machine=...) -> dict:
# ... = ellipsis, means automatically get the machine config from the currently executing machine
# {} = empty dict, override to ignore the machine config
# if machine == ...:
# machine = Machine.objects.get_current()
return getattr(machine, 'config', None) or {}
@staticmethod
@abx.hookspec(firstresult=True)

28
archivebox/pkgs/abx/abx.py
View file

@ -284,7 +284,7 @@ def get_plugin(plugin: PluginId | ModuleType | Type) -> PluginInfo:
# raise ValueError(f'Invalid plugin, must be a module, class, or plugin ID (package name): {plugin}')
assert module
assert module and hasattr(module, '__package__')
plugin_file = Path(inspect.getfile(module))
plugin_package = module.__package__ or module.__name__
@ -356,10 +356,22 @@ def get_all_hook_specs() -> Dict[str, Dict[str, Any]]:
for hook_name in get_all_hook_names():
for plugin_module in pm.get_plugins():
if hasattr(plugin_module, hook_name):
hookspecopts = pm.parse_hookspec_opts(plugin_module, hook_name)
if inspect.ismodule(plugin_module):
plugin = plugin_module
plugin_module = plugin_module
elif inspect.isclass(plugin_module):
plugin = plugin_module
plugin_module = inspect.getmodule(plugin)
else:
plugin = type(plugin_module)
plugin_module = inspect.getmodule(plugin)
assert plugin and plugin_module and hasattr(plugin_module, '__package__')
if hasattr(plugin, hook_name):
hookspecopts = pm.parse_hookspec_opts(plugin, hook_name)
if hookspecopts:
method = getattr(plugin_module, hook_name)
method = getattr(plugin, hook_name)
signature = inspect.signature(method)
return_type = signature.return_annotation if signature.return_annotation != inspect._empty else None
@ -381,9 +393,10 @@ def get_all_hook_specs() -> Dict[str, Dict[str, Any]]:
'signature': call_signature,
'hookspec_opts': hookspecopts,
'hookspec_signature': signature,
'hookspec_plugin': plugin_module.__package__,
'hookspec_plugin': method.__package__,
}
return hook_specs
return benedict(hook_specs)
@ -466,7 +479,8 @@ def get_plugin_hooks(plugin: PluginId | ModuleType | Type | None) -> Dict[AttrNa
elif inspect.ismodule(plugin) or inspect.isclass(plugin):
plugin_module = plugin
else:
raise ValueError(f'Invalid plugin, cannot get hooks: {plugin}')
plugin_module = type(plugin)
# raise ValueError(f'Invalid plugin, cannot get hooks: {plugin}')
for attr_name in dir(plugin_module):
if attr_name.startswith('_'):