mirror of
https://github.com/ArchiveBox/ArchiveBox
synced 2024-11-10 06:34:16 +00:00
add BaseHook concept to underlie all Plugin hooks
This commit is contained in:
parent
ed5357cec9
commit
44669fab73
12 changed files with 212 additions and 79 deletions
|
@ -148,11 +148,12 @@ def abid_part_from_prefix(prefix: str) -> str:
|
||||||
return prefix + '_'
|
return prefix + '_'
|
||||||
|
|
||||||
@enforce_types
|
@enforce_types
|
||||||
def abid_part_from_uri(uri: str, salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
def abid_part_from_uri(uri: Any, salt: str=DEFAULT_ABID_URI_SALT) -> str:
|
||||||
"""
|
"""
|
||||||
'E4A5CCD9' # takes first 8 characters of sha256(url)
|
'E4A5CCD9' # takes first 8 characters of sha256(url)
|
||||||
"""
|
"""
|
||||||
uri = str(uri)
|
uri = str(uri).strip()
|
||||||
|
assert uri not in ('None', '')
|
||||||
return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
|
return uri_hash(uri, salt=salt)[:ABID_URI_LEN]
|
||||||
|
|
||||||
@enforce_types
|
@enforce_types
|
||||||
|
@ -201,7 +202,7 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
|
||||||
|
|
||||||
|
|
||||||
@enforce_types
|
@enforce_types
|
||||||
def abid_hashes_from_values(prefix: str, ts: datetime, uri: str, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
|
def abid_hashes_from_values(prefix: str, ts: datetime, uri: Any, subtype: str | int, rand: Union[str, UUID, None, int], salt: str=DEFAULT_ABID_URI_SALT) -> Dict[str, str]:
|
||||||
return {
|
return {
|
||||||
'prefix': abid_part_from_prefix(prefix),
|
'prefix': abid_part_from_prefix(prefix),
|
||||||
'ts': abid_part_from_ts(ts),
|
'ts': abid_part_from_ts(ts),
|
||||||
|
|
|
@ -9,7 +9,7 @@ from django.utils.html import format_html
|
||||||
from django.utils.safestring import mark_safe
|
from django.utils.safestring import mark_safe
|
||||||
from django.shortcuts import redirect
|
from django.shortcuts import redirect
|
||||||
|
|
||||||
from abid_utils.abid import ABID, abid_part_from_ts, abid_part_from_uri, abid_part_from_rand, abid_part_from_subtype
|
from .abid import ABID
|
||||||
|
|
||||||
from api.auth import get_or_create_api_token
|
from api.auth import get_or_create_api_token
|
||||||
|
|
||||||
|
@ -94,13 +94,18 @@ def get_abid_info(self, obj, request=None):
|
||||||
|
|
||||||
|
|
||||||
class ABIDModelAdmin(admin.ModelAdmin):
|
class ABIDModelAdmin(admin.ModelAdmin):
|
||||||
list_display = ('created_at', 'created_by', 'abid', '__str__')
|
list_display = ('created_at', 'created_by', 'abid')
|
||||||
sort_fields = ('created_at', 'created_by', 'abid', '__str__')
|
sort_fields = ('created_at', 'created_by', 'abid')
|
||||||
readonly_fields = ('created_at', 'modified_at', '__str__', 'abid_info')
|
readonly_fields = ('created_at', 'modified_at', 'abid_info')
|
||||||
|
# fields = [*readonly_fields]
|
||||||
|
|
||||||
@admin.display(description='API Identifiers')
|
def _get_obj_does_not_exist_redirect(self, request, opts, object_id):
|
||||||
def abid_info(self, obj):
|
try:
|
||||||
return get_abid_info(self, obj, request=self.request)
|
object_pk = self.model.id_from_abid(object_id)
|
||||||
|
return redirect(self.request.path.replace(object_id, object_pk), permanent=False)
|
||||||
|
except (self.model.DoesNotExist, ValidationError):
|
||||||
|
pass
|
||||||
|
return super()._get_obj_does_not_exist_redirect(request, opts, object_id) # type: ignore
|
||||||
|
|
||||||
def queryset(self, request):
|
def queryset(self, request):
|
||||||
self.request = request
|
self.request = request
|
||||||
|
@ -108,15 +113,6 @@ class ABIDModelAdmin(admin.ModelAdmin):
|
||||||
|
|
||||||
def change_view(self, request, object_id, form_url="", extra_context=None):
|
def change_view(self, request, object_id, form_url="", extra_context=None):
|
||||||
self.request = request
|
self.request = request
|
||||||
|
|
||||||
if object_id:
|
|
||||||
try:
|
|
||||||
object_uuid = str(self.model.objects.only('pk').get(abid=self.model.abid_prefix + object_id.split('_', 1)[-1]).pk)
|
|
||||||
if object_id != object_uuid:
|
|
||||||
return redirect(self.request.path.replace(object_id, object_uuid), permanent=False)
|
|
||||||
except (self.model.DoesNotExist, ValidationError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
return super().change_view(request, object_id, form_url, extra_context)
|
return super().change_view(request, object_id, form_url, extra_context)
|
||||||
|
|
||||||
def get_form(self, request, obj=None, **kwargs):
|
def get_form(self, request, obj=None, **kwargs):
|
||||||
|
@ -126,9 +122,24 @@ class ABIDModelAdmin(admin.ModelAdmin):
|
||||||
form.base_fields['created_by'].initial = request.user
|
form.base_fields['created_by'].initial = request.user
|
||||||
return form
|
return form
|
||||||
|
|
||||||
|
def get_formset(self, request, formset=None, obj=None, **kwargs):
|
||||||
|
formset = super().get_formset(request, formset, obj, **kwargs)
|
||||||
|
formset.form.base_fields['created_at'].disabled = True
|
||||||
|
return formset
|
||||||
|
|
||||||
def save_model(self, request, obj, form, change):
|
def save_model(self, request, obj, form, change):
|
||||||
old_abid = obj.abid
|
self.request = request
|
||||||
|
|
||||||
|
old_abid = getattr(obj, '_previous_abid', None) or obj.abid
|
||||||
|
|
||||||
super().save_model(request, obj, form, change)
|
super().save_model(request, obj, form, change)
|
||||||
|
obj.refresh_from_db()
|
||||||
|
|
||||||
new_abid = obj.abid
|
new_abid = obj.abid
|
||||||
if new_abid != old_abid:
|
if new_abid != old_abid:
|
||||||
messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any references to the old ABID will need to be updated)")
|
messages.warning(request, f"The object's ABID has been updated! {old_abid} -> {new_abid} (any external references to the old ABID will need to be updated manually)")
|
||||||
|
# import ipdb; ipdb.set_trace()
|
||||||
|
|
||||||
|
@admin.display(description='API Identifiers')
|
||||||
|
def abid_info(self, obj):
|
||||||
|
return get_abid_info(self, obj, request=self.request)
|
||||||
|
|
|
@ -11,7 +11,8 @@ from datetime import datetime, timedelta
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from charidfield import CharIDField # type: ignore[import-untyped]
|
from charidfield import CharIDField # type: ignore[import-untyped]
|
||||||
|
|
||||||
from django.core.exceptions import ValidationError
|
from django.contrib import admin
|
||||||
|
from django.core.exceptions import ValidationError, NON_FIELD_ERRORS
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.db.utils import OperationalError
|
from django.db.utils import OperationalError
|
||||||
|
@ -71,24 +72,6 @@ class AutoDateTimeField(models.DateTimeField):
|
||||||
class ABIDError(Exception):
|
class ABIDError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
class ABIDFieldsCannotBeChanged(ValidationError, ABIDError):
|
|
||||||
"""
|
|
||||||
Properties used as unique identifiers (to generate ABID) cannot be edited after an object is created.
|
|
||||||
Create a new object instead with your desired changes (and it will be issued a new ABID).
|
|
||||||
"""
|
|
||||||
def __init__(self, ABID_FRESH_DIFFS, obj):
|
|
||||||
self.ABID_FRESH_DIFFS = ABID_FRESH_DIFFS
|
|
||||||
self.obj = obj
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
keys_changed = ', '.join(diff['abid_src'] for diff in self.ABID_FRESH_DIFFS.values())
|
|
||||||
return (
|
|
||||||
f"This {self.obj.__class__.__name__}(abid={str(self.obj.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
|
|
||||||
f'\nThe following changes cannot be made because they would alter the ABID:' +
|
|
||||||
'\n ' + "\n ".join(f' - {diff["summary"]}' for diff in self.ABID_FRESH_DIFFS.values()) +
|
|
||||||
f"\nYou must reduce your changes to not affect these fields, or create a new {self.obj.__class__.__name__} object instead."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ABIDModel(models.Model):
|
class ABIDModel(models.Model):
|
||||||
"""
|
"""
|
||||||
|
@ -112,6 +95,10 @@ class ABIDModel(models.Model):
|
||||||
class Meta(TypedModelMeta):
|
class Meta(TypedModelMeta):
|
||||||
abstract = True
|
abstract = True
|
||||||
|
|
||||||
|
@admin.display(description='Summary')
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f'[{self.abid or (self.abid_prefix + "NEW")}] {self.__class__.__name__} {eval(self.abid_uri_src)}'
|
||||||
|
|
||||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||||
"""Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
|
"""Overriden __init__ method ensures we have a stable creation timestamp that fields can use within initialization code pre-saving to DB."""
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
|
@ -121,30 +108,60 @@ class ABIDModel(models.Model):
|
||||||
# (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
|
# (ordinarily fields cant depend on other fields until the obj is saved to db and recalled)
|
||||||
self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
|
self._init_timestamp = ts_from_abid(abid_part_from_ts(timezone.now()))
|
||||||
|
|
||||||
def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
|
def clean(self, abid_drift_allowed: bool | None=None) -> None:
|
||||||
"""Overriden save method ensures new ABID is generated while a new object is first saving."""
|
|
||||||
|
|
||||||
if self._state.adding:
|
if self._state.adding:
|
||||||
# only runs once when a new object is first saved to the DB
|
# only runs once when a new object is first saved to the DB
|
||||||
# sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
|
# sets self.id, self.pk, self.created_by, self.created_at, self.modified_at
|
||||||
|
self._previous_abid = None
|
||||||
self.abid = str(self.issue_new_abid())
|
self.abid = str(self.issue_new_abid())
|
||||||
|
|
||||||
else:
|
else:
|
||||||
# otherwise if updating, make sure none of the field changes would invalidate existing ABID
|
# otherwise if updating, make sure none of the field changes would invalidate existing ABID
|
||||||
if self.ABID_FRESH_DIFFS:
|
abid_diffs = self.ABID_FRESH_DIFFS
|
||||||
ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
|
if abid_diffs:
|
||||||
|
|
||||||
change_error = ABIDFieldsCannotBeChanged(self.ABID_FRESH_DIFFS, obj=self)
|
keys_changed = ', '.join(diff['abid_src'] for diff in abid_diffs.values())
|
||||||
if ovewrite_abid:
|
full_summary = (
|
||||||
print(f'#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={abid_drift_allowed}), this will break any references to its previous ABID!')
|
f"This {self.__class__.__name__}(abid={str(self.ABID)}) was assigned a fixed, unique ID (ABID) based on its contents when it was created. " +
|
||||||
|
f"\nYou must reduce your changes to not affect these fields [{keys_changed}], or create a new {self.__class__.__name__} object instead."
|
||||||
|
)
|
||||||
|
|
||||||
|
change_error = ValidationError({
|
||||||
|
NON_FIELD_ERRORS: ValidationError(full_summary),
|
||||||
|
**{
|
||||||
|
# url: ValidationError('Cannot update self.url= https://example.com/old -> https://example.com/new ...')
|
||||||
|
diff['abid_src'].replace('self.', '') if diff['old_val'] != diff['new_val'] else NON_FIELD_ERRORS
|
||||||
|
: ValidationError(
|
||||||
|
'Cannot update %(abid_src)s= "%(old_val)s" -> "%(new_val)s" (would alter %(model)s.ABID.%(key)s=%(old_hash)s to %(new_hash)s)',
|
||||||
|
code='ABIDConflict',
|
||||||
|
params=diff,
|
||||||
|
)
|
||||||
|
for diff in abid_diffs.values()
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
should_ovewrite_abid = self.abid_drift_allowed if (abid_drift_allowed is None) else abid_drift_allowed
|
||||||
|
if should_ovewrite_abid:
|
||||||
|
print(f'\n#### DANGER: Changing ABID of existing record ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed}), this will break any references to its previous ABID!')
|
||||||
print(change_error)
|
print(change_error)
|
||||||
|
self._previous_abid = self.abid
|
||||||
self.abid = str(self.issue_new_abid(force_new=True))
|
self.abid = str(self.issue_new_abid(force_new=True))
|
||||||
print(f'#### DANGER: OVERWROTE OLD ABID. NEW ABID=', self.abid)
|
print(f'#### DANGER: OVERWROTE OLD ABID. NEW ABID=', self.abid)
|
||||||
else:
|
else:
|
||||||
raise change_error
|
print(f'\n#### WARNING: ABID of existing record is outdated and has not been updated ({self.__class__.__name__}.abid_drift_allowed={self.abid_drift_allowed})')
|
||||||
|
print(change_error)
|
||||||
|
|
||||||
|
def save(self, *args: Any, abid_drift_allowed: bool | None=None, **kwargs: Any) -> None:
|
||||||
|
"""Overriden save method ensures new ABID is generated while a new object is first saving."""
|
||||||
|
|
||||||
|
self.clean(abid_drift_allowed=abid_drift_allowed)
|
||||||
|
|
||||||
return super().save(*args, **kwargs)
|
return super().save(*args, **kwargs)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def id_from_abid(cls, abid: str) -> str:
|
||||||
|
return str(cls.objects.only('pk').get(abid=cls.abid_prefix + str(abid).split('_', 1)[-1]).pk)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def ABID_SOURCES(self) -> Dict[str, str]:
|
def ABID_SOURCES(self) -> Dict[str, str]:
|
||||||
""""Get the dict of fresh ABID component values based on the live object's properties."""
|
""""Get the dict of fresh ABID component values based on the live object's properties."""
|
||||||
|
@ -196,10 +213,10 @@ class ABIDModel(models.Model):
|
||||||
fresh_hashes = self.ABID_FRESH_HASHES
|
fresh_hashes = self.ABID_FRESH_HASHES
|
||||||
return {
|
return {
|
||||||
key: {
|
key: {
|
||||||
|
'key': key,
|
||||||
'model': self.__class__.__name__,
|
'model': self.__class__.__name__,
|
||||||
'pk': self.pk,
|
'pk': self.pk,
|
||||||
'abid_src': abid_sources[key],
|
'abid_src': abid_sources[key],
|
||||||
'abid_section': key,
|
|
||||||
'old_val': existing_values.get(key, None),
|
'old_val': existing_values.get(key, None),
|
||||||
'old_hash': getattr(existing_abid, key),
|
'old_hash': getattr(existing_abid, key),
|
||||||
'new_val': fresh_values[key],
|
'new_val': fresh_values[key],
|
||||||
|
@ -215,7 +232,6 @@ class ABIDModel(models.Model):
|
||||||
Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
|
Issue a new ABID based on the current object's properties, can only be called once on new objects (before they are saved to DB).
|
||||||
"""
|
"""
|
||||||
if not force_new:
|
if not force_new:
|
||||||
assert self.abid is None, f'Can only issue new ABID for new objects that dont already have one {self.abid}'
|
|
||||||
assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
|
assert self._state.adding, 'Can only issue new ABID when model._state.adding is True'
|
||||||
assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
|
assert eval(self.abid_uri_src), f'Can only issue new ABID if self.abid_uri_src is defined ({self.abid_uri_src}={eval(self.abid_uri_src)})'
|
||||||
|
|
||||||
|
@ -286,7 +302,7 @@ class ABIDModel(models.Model):
|
||||||
Compute the REST API URL to access this object.
|
Compute the REST API URL to access this object.
|
||||||
e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
|
e.g. /api/v1/core/snapshot/snp_01BJQMF54D093DXEAWZ6JYRP
|
||||||
"""
|
"""
|
||||||
return reverse_lazy('api-1:get_any', args=[self.abid])
|
return reverse_lazy('api-1:get_any', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def api_docs_url(self) -> str:
|
def api_docs_url(self) -> str:
|
||||||
|
@ -296,7 +312,12 @@ class ABIDModel(models.Model):
|
||||||
"""
|
"""
|
||||||
return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
|
return f'/api/v1/docs#/{self._meta.app_label.title()}%20Models/api_v1_{self._meta.app_label}_get_{self._meta.db_table}'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def admin_change_url(self) -> str:
|
||||||
|
return f"/admin/{self._meta.app_label}/{self._meta.model_name}/{self.pk}/change/"
|
||||||
|
|
||||||
|
def get_absolute_url(self):
|
||||||
|
return self.api_docs_url
|
||||||
|
|
||||||
####################################################
|
####################################################
|
||||||
|
|
||||||
|
|
|
@ -28,9 +28,10 @@ class APIToken(ABIDModel):
|
||||||
# ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
|
# ABID: apt_<created_ts>_<token_hash>_<user_id_hash>_<uuid_rand>
|
||||||
abid_prefix = 'apt_'
|
abid_prefix = 'apt_'
|
||||||
abid_ts_src = 'self.created_at'
|
abid_ts_src = 'self.created_at'
|
||||||
abid_uri_src = 'self.token'
|
abid_uri_src = 'self.created_by_id'
|
||||||
abid_subtype_src = 'self.created_by_id'
|
abid_subtype_src = '"01"'
|
||||||
abid_rand_src = 'self.id'
|
abid_rand_src = 'self.id'
|
||||||
|
abid_drift_allowed = True
|
||||||
|
|
||||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
@ -99,6 +100,7 @@ class OutboundWebhook(ABIDModel, WebhookBase):
|
||||||
abid_uri_src = 'self.endpoint'
|
abid_uri_src = 'self.endpoint'
|
||||||
abid_subtype_src = 'self.ref'
|
abid_subtype_src = 'self.ref'
|
||||||
abid_rand_src = 'self.id'
|
abid_rand_src = 'self.id'
|
||||||
|
abid_drift_allowed = True
|
||||||
|
|
||||||
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
|
||||||
abid = ABIDField(prefix=abid_prefix)
|
abid = ABIDField(prefix=abid_prefix)
|
||||||
|
@ -121,3 +123,6 @@ class OutboundWebhook(ABIDModel, WebhookBase):
|
||||||
class Meta(WebhookBase.Meta):
|
class Meta(WebhookBase.Meta):
|
||||||
verbose_name = 'API Outbound Webhook'
|
verbose_name = 'API Outbound Webhook'
|
||||||
|
|
||||||
|
|
||||||
|
def __str__(self) -> str:
|
||||||
|
return f'[{self.abid}] {self.ref} -> {self.endpoint}'
|
||||||
|
|
|
@ -103,7 +103,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
|
||||||
'PUBLIC_SNAPSHOTS': {'type': bool, 'default': True},
|
'PUBLIC_SNAPSHOTS': {'type': bool, 'default': True},
|
||||||
'PUBLIC_ADD_VIEW': {'type': bool, 'default': False},
|
'PUBLIC_ADD_VIEW': {'type': bool, 'default': False},
|
||||||
'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'},
|
'FOOTER_INFO': {'type': str, 'default': 'Content is hosted for personal archiving purposes only. Contact server owner for any takedown requests.'},
|
||||||
'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 100},
|
'SNAPSHOTS_PER_PAGE': {'type': int, 'default': 40},
|
||||||
'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None},
|
'CUSTOM_TEMPLATES_DIR': {'type': str, 'default': None},
|
||||||
'TIME_ZONE': {'type': str, 'default': 'UTC'},
|
'TIME_ZONE': {'type': str, 'default': 'UTC'},
|
||||||
'TIMEZONE': {'type': str, 'default': 'UTC'},
|
'TIMEZONE': {'type': str, 'default': 'UTC'},
|
||||||
|
|
|
@ -254,7 +254,7 @@ class ArchiveResultInline(admin.TabularInline):
|
||||||
try:
|
try:
|
||||||
return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
|
return self.parent_model.objects.get(pk=resolved.kwargs['object_id'])
|
||||||
except (self.parent_model.DoesNotExist, ValidationError):
|
except (self.parent_model.DoesNotExist, ValidationError):
|
||||||
return self.parent_model.objects.get(abid=self.parent_model.abid_prefix + resolved.kwargs['object_id'].split('_', 1)[-1])
|
return self.parent_model.objects.get(pk=self.parent_model.id_from_abid(resolved.kwargs['object_id']))
|
||||||
|
|
||||||
@admin.display(
|
@admin.display(
|
||||||
description='Completed',
|
description='Completed',
|
||||||
|
@ -685,6 +685,7 @@ class ArchiveResultAdmin(ABIDModelAdmin):
|
||||||
list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
|
list_per_page = CONFIG.SNAPSHOTS_PER_PAGE
|
||||||
|
|
||||||
paginator = AccelleratedPaginator
|
paginator = AccelleratedPaginator
|
||||||
|
save_on_top = True
|
||||||
|
|
||||||
def change_view(self, request, object_id, form_url="", extra_context=None):
|
def change_view(self, request, object_id, form_url="", extra_context=None):
|
||||||
self.request = request
|
self.request = request
|
||||||
|
|
|
@ -103,7 +103,7 @@ class Tag(ABIDModel):
|
||||||
@property
|
@property
|
||||||
def api_url(self) -> str:
|
def api_url(self) -> str:
|
||||||
# /api/v1/core/snapshot/{uulid}
|
# /api/v1/core/snapshot/{uulid}
|
||||||
return reverse_lazy('api-1:get_tag', args=[self.abid])
|
return reverse_lazy('api-1:get_tag', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def api_docs_url(self) -> str:
|
def api_docs_url(self) -> str:
|
||||||
|
@ -211,12 +211,15 @@ class Snapshot(ABIDModel):
|
||||||
@property
|
@property
|
||||||
def api_url(self) -> str:
|
def api_url(self) -> str:
|
||||||
# /api/v1/core/snapshot/{uulid}
|
# /api/v1/core/snapshot/{uulid}
|
||||||
return reverse_lazy('api-1:get_snapshot', args=[self.abid])
|
return reverse_lazy('api-1:get_snapshot', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def api_docs_url(self) -> str:
|
def api_docs_url(self) -> str:
|
||||||
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
|
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
|
||||||
|
|
||||||
|
def get_absolute_url(self):
|
||||||
|
return f'/{self.archive_path}'
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def title_stripped(self) -> str:
|
def title_stripped(self) -> str:
|
||||||
return (self.title or '').replace("\n", " ").replace("\r", "")
|
return (self.title or '').replace("\n", " ").replace("\r", "")
|
||||||
|
@ -476,12 +479,15 @@ class ArchiveResult(ABIDModel):
|
||||||
@property
|
@property
|
||||||
def api_url(self) -> str:
|
def api_url(self) -> str:
|
||||||
# /api/v1/core/archiveresult/{uulid}
|
# /api/v1/core/archiveresult/{uulid}
|
||||||
return reverse_lazy('api-1:get_archiveresult', args=[self.abid])
|
return reverse_lazy('api-1:get_archiveresult', args=[self.abid]) # + f'?api_key={get_or_create_api_token(request.user)}'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def api_docs_url(self) -> str:
|
def api_docs_url(self) -> str:
|
||||||
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
|
return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
|
||||||
|
|
||||||
|
def get_absolute_url(self):
|
||||||
|
return f'/{self.snapshot.archive_path}/{self.output_path()}'
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def extractor_module(self):
|
def extractor_module(self):
|
||||||
return EXTRACTORS[self.extractor]
|
return EXTRACTORS[self.extractor]
|
||||||
|
|
|
@ -40,6 +40,7 @@ INSTALLED_PLUGINS = {
|
||||||
|
|
||||||
### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
|
### Plugins Globals (filled by plugantic.apps.load_plugins() after Django startup)
|
||||||
PLUGINS = AttrDict({})
|
PLUGINS = AttrDict({})
|
||||||
|
HOOKS = AttrDict({})
|
||||||
|
|
||||||
CONFIGS = AttrDict({})
|
CONFIGS = AttrDict({})
|
||||||
BINPROVIDERS = AttrDict({})
|
BINPROVIDERS = AttrDict({})
|
||||||
|
|
|
@ -1,14 +1,14 @@
|
||||||
from typing import List, Type, Any
|
from typing import List, Type, Any
|
||||||
|
|
||||||
from pydantic_core import core_schema
|
from pydantic_core import core_schema
|
||||||
from pydantic import GetCoreSchemaHandler
|
from pydantic import GetCoreSchemaHandler, BaseModel
|
||||||
|
|
||||||
from django.utils.functional import classproperty
|
from django.utils.functional import classproperty
|
||||||
from django.core.checks import Warning, Tags, register
|
from django.core.checks import Warning, Tags, register
|
||||||
|
|
||||||
class BaseCheck:
|
class BaseCheck:
|
||||||
label: str = ''
|
label: str = ''
|
||||||
tag = Tags.database
|
tag: str = Tags.database
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
|
def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema:
|
||||||
|
|
|
@ -5,6 +5,7 @@ from typing import Optional, List, Literal
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from pydantic import BaseModel, Field, ConfigDict, computed_field
|
from pydantic import BaseModel, Field, ConfigDict, computed_field
|
||||||
|
|
||||||
|
from .base_hook import BaseHook, HookType
|
||||||
|
|
||||||
ConfigSectionName = Literal[
|
ConfigSectionName = Literal[
|
||||||
'GENERAL_CONFIG',
|
'GENERAL_CONFIG',
|
||||||
|
@ -20,24 +21,26 @@ ConfigSectionNames: List[ConfigSectionName] = [
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
class BaseConfigSet(BaseModel):
|
class BaseConfigSet(BaseHook):
|
||||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
|
model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow', populate_by_name=True)
|
||||||
|
hook_type: HookType = 'CONFIG'
|
||||||
|
|
||||||
section: ConfigSectionName = 'GENERAL_CONFIG'
|
section: ConfigSectionName = 'GENERAL_CONFIG'
|
||||||
|
|
||||||
@computed_field
|
|
||||||
@property
|
|
||||||
def name(self) -> str:
|
|
||||||
return self.__class__.__name__
|
|
||||||
|
|
||||||
def register(self, settings, parent_plugin=None):
|
def register(self, settings, parent_plugin=None):
|
||||||
|
"""Installs the ConfigSet into Django settings.CONFIGS (and settings.HOOKS)."""
|
||||||
if settings is None:
|
if settings is None:
|
||||||
from django.conf import settings as django_settings
|
from django.conf import settings as django_settings
|
||||||
settings = django_settings
|
settings = django_settings
|
||||||
|
|
||||||
self._plugin = parent_plugin # for debugging only, never rely on this!
|
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||||
|
|
||||||
|
# install hook into settings.CONFIGS
|
||||||
settings.CONFIGS[self.name] = self
|
settings.CONFIGS[self.name] = self
|
||||||
|
|
||||||
|
# record installed hook in settings.HOOKS
|
||||||
|
super().register(settings, parent_plugin=parent_plugin)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# class WgetToggleConfig(ConfigSet):
|
# class WgetToggleConfig(ConfigSet):
|
||||||
|
|
71
archivebox/plugantic/base_hook.py
Normal file
71
archivebox/plugantic/base_hook.py
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
__package__ = 'archivebox.plugantic'
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Optional, List, Literal, ClassVar
|
||||||
|
from pathlib import Path
|
||||||
|
from pydantic import BaseModel, Field, ConfigDict, computed_field
|
||||||
|
|
||||||
|
|
||||||
|
HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
|
||||||
|
hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class BaseHook(BaseModel):
|
||||||
|
"""
|
||||||
|
A Plugin consists of a list of Hooks, applied to django.conf.settings when AppConfig.read() -> Plugin.register() is called.
|
||||||
|
Plugin.register() then calls each Hook.register() on the provided settings.
|
||||||
|
each Hook.regsiter() function (ideally pure) takes a django.conf.settings as input and returns a new one back.
|
||||||
|
or
|
||||||
|
it modifies django.conf.settings in-place to add changes corresponding to its HookType.
|
||||||
|
e.g. for a HookType.CONFIG, the Hook.register() function places the hook in settings.CONFIG (and settings.HOOKS)
|
||||||
|
An example of an impure Hook would be a CHECK that modifies settings but also calls django.core.checks.register(check).
|
||||||
|
|
||||||
|
|
||||||
|
setup_django() -> imports all settings.INSTALLED_APPS...
|
||||||
|
# django imports AppConfig, models, migrations, admins, etc. for all installed apps
|
||||||
|
# django then calls AppConfig.ready() on each installed app...
|
||||||
|
|
||||||
|
builtin_plugins.npm.NpmPlugin().AppConfig.ready() # called by django
|
||||||
|
builtin_plugins.npm.NpmPlugin().register(settings) ->
|
||||||
|
builtin_plugins.npm.NpmConfigSet().register(settings)
|
||||||
|
plugantic.base_configset.BaseConfigSet().register(settings)
|
||||||
|
plugantic.base_hook.BaseHook().register(settings, parent_plugin=builtin_plugins.npm.NpmPlugin())
|
||||||
|
|
||||||
|
...
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
model_config = ConfigDict(
|
||||||
|
extra='allow',
|
||||||
|
arbitrary_types_allowed=True,
|
||||||
|
from_attributes=True,
|
||||||
|
populate_by_name=True,
|
||||||
|
validate_defaults=True,
|
||||||
|
validate_assignment=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
hook_type: HookType = 'CONFIG'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name(self) -> str:
|
||||||
|
return f'{self.__module__}.{__class__.__name__}'
|
||||||
|
|
||||||
|
def register(self, settings, parent_plugin=None):
|
||||||
|
"""Load a record of an installed hook into global Django settings.HOOKS at runtime."""
|
||||||
|
|
||||||
|
if settings is None:
|
||||||
|
from django.conf import settings as django_settings
|
||||||
|
settings = django_settings
|
||||||
|
|
||||||
|
assert json.dumps(self.model_json_schema(), indent=4), f'Hook {self.name} has invalid JSON schema.'
|
||||||
|
|
||||||
|
self._plugin = parent_plugin # for debugging only, never rely on this!
|
||||||
|
|
||||||
|
# record installed hook in settings.HOOKS
|
||||||
|
settings.HOOKS[self.name] = self
|
||||||
|
|
||||||
|
hook_prefix, plugin_shortname = self.name.split('.', 1)
|
||||||
|
|
||||||
|
print('REGISTERED HOOK:', self.name)
|
|
@ -1,6 +1,8 @@
|
||||||
__package__ = 'archivebox.plugantic'
|
__package__ = 'archivebox.plugantic'
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import inspect
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from django.apps import AppConfig
|
from django.apps import AppConfig
|
||||||
from django.core.checks import register
|
from django.core.checks import register
|
||||||
|
@ -32,12 +34,11 @@ class BasePlugin(BaseModel):
|
||||||
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
|
model_config = ConfigDict(arbitrary_types_allowed=True, extra='ignore', populate_by_name=True)
|
||||||
|
|
||||||
# Required by AppConfig:
|
# Required by AppConfig:
|
||||||
name: str = Field() # e.g. 'builtin_plugins.singlefile'
|
name: str = Field() # e.g. 'builtin_plugins.singlefile' (DottedImportPath)
|
||||||
app_label: str = Field() # e.g. 'singlefile'
|
app_label: str = Field() # e.g. 'singlefile' (one-word machine-readable representation, to use as url-safe id/db-table prefix_/attr name)
|
||||||
verbose_name: str = Field() # e.g. 'SingleFile'
|
verbose_name: str = Field() # e.g. 'SingleFile' (human-readable *short* label, for use in column names, form labels, etc.)
|
||||||
default_auto_field: ClassVar[str] = 'django.db.models.AutoField'
|
|
||||||
|
|
||||||
# Required by Plugantic:
|
# All the hooks the plugin will install:
|
||||||
configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
|
configs: List[InstanceOf[BaseConfigSet]] = Field(default=[])
|
||||||
binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
binproviders: List[InstanceOf[BaseBinProvider]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||||
binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
binaries: List[InstanceOf[BaseBinary]] = Field(default=[]) # e.g. [Binary(name='yt-dlp')]
|
||||||
|
@ -53,20 +54,23 @@ class BasePlugin(BaseModel):
|
||||||
assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
|
assert self.name and self.app_label and self.verbose_name, f'{self.__class__.__name__} is missing .name or .app_label or .verbose_name'
|
||||||
|
|
||||||
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
assert json.dumps(self.model_json_schema(), indent=4), f'Plugin {self.name} has invalid JSON schema.'
|
||||||
|
return self
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def AppConfig(plugin_self) -> Type[AppConfig]:
|
def AppConfig(plugin_self) -> Type[AppConfig]:
|
||||||
"""Generate a Django AppConfig class for this plugin."""
|
"""Generate a Django AppConfig class for this plugin."""
|
||||||
|
|
||||||
class PluginAppConfig(AppConfig):
|
class PluginAppConfig(AppConfig):
|
||||||
|
"""Django AppConfig for plugin, allows it to be loaded as a Django app listed in settings.INSTALLED_APPS."""
|
||||||
name = plugin_self.name
|
name = plugin_self.name
|
||||||
app_label = plugin_self.app_label
|
app_label = plugin_self.app_label
|
||||||
verbose_name = plugin_self.verbose_name
|
verbose_name = plugin_self.verbose_name
|
||||||
|
default_auto_field = 'django.db.models.AutoField'
|
||||||
|
|
||||||
def ready(self):
|
def ready(self):
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
|
|
||||||
plugin_self.validate()
|
# plugin_self.validate()
|
||||||
plugin_self.register(settings)
|
plugin_self.register(settings)
|
||||||
|
|
||||||
return PluginAppConfig
|
return PluginAppConfig
|
||||||
|
@ -106,11 +110,6 @@ class BasePlugin(BaseModel):
|
||||||
def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
|
def ADMINDATAVIEWS(self) -> Dict[str, BaseCheck]:
|
||||||
return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
|
return AttrDict({admindataview.name: admindataview for admindataview in self.admindataviews})
|
||||||
|
|
||||||
@computed_field
|
|
||||||
@property
|
|
||||||
def PLUGIN_KEYS(self) -> List[str]:
|
|
||||||
return
|
|
||||||
|
|
||||||
def register(self, settings=None):
|
def register(self, settings=None):
|
||||||
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
|
"""Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
|
||||||
|
|
||||||
|
@ -185,6 +184,20 @@ class BasePlugin(BaseModel):
|
||||||
# 'binaries': new_binaries,
|
# 'binaries': new_binaries,
|
||||||
# })
|
# })
|
||||||
|
|
||||||
|
@computed_field
|
||||||
|
@property
|
||||||
|
def module_dir(self) -> Path:
|
||||||
|
return Path(inspect.getfile(self.__class__)).parent.resolve()
|
||||||
|
|
||||||
|
@computed_field
|
||||||
|
@property
|
||||||
|
def module_path(self) -> str: # DottedImportPath
|
||||||
|
""""
|
||||||
|
Dotted import path of the plugin's module (after its loaded via settings.INSTALLED_APPS).
|
||||||
|
e.g. 'archivebox.builtin_plugins.npm'
|
||||||
|
"""
|
||||||
|
return self.name.strip('archivebox.')
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue