ABID, Admin UI, and REST API improvements to prepare for v0.8 release (#1488)

archivebox/abid_utils/abid.py

@@ -37,6 +37,8 @@ class ABID(NamedTuple):
     subtype: str           # e.g. 01
     rand: str              # e.g. ZYEBQE
     
+    # salt: str = DEFAULT_ABID_URI_SALT
+
     def __getattr__(self, attr: str) -> Any:
         return getattr(self.ulid, attr)
 
@@ -73,6 +75,10 @@ class ABID(NamedTuple):
             rand=suffix[20:26].upper(),
         )
     
+    @property
+    def uri_salt(self) -> str:
+        return DEFAULT_ABID_URI_SALT
+
     @property
     def suffix(self):
         return ''.join((self.ts, self.uri, self.subtype, self.rand))
@@ -180,7 +186,7 @@ def abid_part_from_rand(rand: Union[str, UUID, None, int]) -> str:
     return str(rand)[-ABID_RAND_LEN:].upper()
 
 
-def abid_from_values(prefix, ts, uri, subtype, rand) -> ABID:
+def abid_from_values(prefix, ts, uri, subtype, rand, salt=DEFAULT_ABID_URI_SALT) -> ABID:
     """
     Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
     """
@@ -188,7 +194,7 @@ def abid_from_values(prefix, ts, uri, subtype, rand) -> ABID:
     abid = ABID(
         prefix=abid_part_from_prefix(prefix),
         ts=abid_part_from_ts(ts),
-        uri=abid_part_from_uri(uri),
+        uri=abid_part_from_uri(uri, salt=salt),
         subtype=abid_part_from_subtype(subtype),
         rand=abid_part_from_rand(rand),
     )

archivebox/abid_utils/models.py

@@ -26,6 +26,7 @@ from .abid import (
     ABID_RAND_LEN,
     ABID_SUFFIX_LEN,
     DEFAULT_ABID_PREFIX,
+    DEFAULT_ABID_URI_SALT,
     abid_part_from_prefix,
     abid_from_values
 )
@@ -69,8 +70,8 @@ class ABIDModel(models.Model):
     abid_subtype_src = 'None'               # e.g. 'self.extractor'
     abid_rand_src = 'None'                  # e.g. 'self.uuid' or 'self.id'
 
-    id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
-    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    # id = models.UUIDField(primary_key=True, default=uuid4, editable=True)
+    # uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
     abid = ABIDField(prefix=abid_prefix)
 
     created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk)
@@ -132,6 +133,7 @@ class ABIDModel(models.Model):
             uri=uri,
             subtype=subtype,
             rand=rand,
+            salt=DEFAULT_ABID_URI_SALT,
         )
         assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
         return abid

archivebox/api/v1_api.py

@@ -63,7 +63,7 @@ api = NinjaAPIWithIOCapture(
     version='1.0.0',
     csrf=False,
     auth=API_AUTH_METHODS,
-    urls_namespace="api",
+    urls_namespace="api-1",
     docs=Swagger(settings={"persistAuthorization": True}),
     # docs_decorator=login_required,
     # renderer=ORJSONRenderer(),

archivebox/api/v1_core.py

@@ -1,14 +1,17 @@
 __package__ = 'archivebox.api'
 
+import math
 from uuid import UUID
-from typing import List, Optional
+from typing import List, Optional, Union, Any
 from datetime import datetime
 
 from django.db.models import Q
 from django.shortcuts import get_object_or_404
+from django.core.exceptions import ValidationError
+from django.contrib.auth import get_user_model
 
 from ninja import Router, Schema, FilterSchema, Field, Query
-from ninja.pagination import paginate
+from ninja.pagination import paginate, PaginationBase
 
 from core.models import Snapshot, ArchiveResult, Tag
 from abid_utils.abid import ABID
@@ -17,23 +20,61 @@ router = Router(tags=['Core Models'])
 
 
 
+class CustomPagination(PaginationBase):
+    class Input(Schema):
+        limit: int = 200
+        offset: int = 0
+        page: int = 0
+
+
+    class Output(Schema):
+        total_items: int
+        total_pages: int
+        page: int
+        limit: int
+        offset: int
+        num_items: int
+        items: List[Any]
+
+    def paginate_queryset(self, queryset, pagination: Input, **params):
+        limit = min(pagination.limit, 500)
+        offset = pagination.offset or (pagination.page * limit)
+        total = queryset.count()
+        total_pages = math.ceil(total / limit)
+        current_page = math.ceil(offset / (limit + 1))
+        items = queryset[offset : offset + limit]
+        return {
+            'total_items': total,
+            'total_pages': total_pages,
+            'page': current_page,
+            'limit': limit,
+            'offset': offset,
+            'num_items': len(items),
+            'items': items,
+        }
+
 
 ### ArchiveResult #########################################################################
 
 class ArchiveResultSchema(Schema):
+    TYPE: str = 'core.models.ArchiveResult'
+
+    id: UUID
+    old_id: int
     abid: str
-    uuid: UUID
-    pk: str
+
     modified: datetime
     created: datetime
     created_by_id: str
+    created_by_username: str
 
     snapshot_abid: str
+    snapshot_timestamp: str
     snapshot_url: str
     snapshot_tags: str
 
     extractor: str
-    cmd_version: str
+    cmd_version: Optional[str]
     cmd: List[str]
     pwd: str
     status: str
@@ -43,6 +84,11 @@ class ArchiveResultSchema(Schema):
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
     
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
     @staticmethod
     def resolve_pk(obj):
         return str(obj.pk)
@@ -59,6 +105,10 @@ class ArchiveResultSchema(Schema):
     def resolve_created(obj):
         return obj.start_ts
 
+    @staticmethod
+    def resolve_snapshot_timestamp(obj):
+        return obj.snapshot.timestamp
+    
     @staticmethod
     def resolve_snapshot_url(obj):
         return obj.snapshot.url
@@ -73,11 +123,10 @@ class ArchiveResultSchema(Schema):
 
 
 class ArchiveResultFilterSchema(FilterSchema):
-    uuid: Optional[UUID] = Field(None, q='uuid')
-    # abid: Optional[str] = Field(None, q='abid')
+    id: Optional[str] = Field(None, q=['id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
 
-    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains'])
-    snapshot_uuid: Optional[UUID] = Field(None, q='snapshot_uuid__icontains')
+    search: Optional[str] = Field(None, q=['snapshot__url__icontains', 'snapshot__title__icontains', 'snapshot__tags__name__icontains', 'extractor', 'output__icontains', 'id__startswith', 'abid__icontains', 'old_id__startswith', 'snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
+    snapshot_id: Optional[str] = Field(None, q=['snapshot__id__startswith', 'snapshot__abid__icontains', 'snapshot__timestamp__startswith'])
     snapshot_url: Optional[str] = Field(None, q='snapshot__url__icontains')
     snapshot_tag: Optional[str] = Field(None, q='snapshot__tags__name__icontains')
     
@@ -93,19 +142,19 @@ class ArchiveResultFilterSchema(FilterSchema):
     created__lt: Optional[datetime] = Field(None, q='updated__lt')
 
 
-@router.get("/archiveresults", response=List[ArchiveResultSchema])
-@paginate
-def list_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
+@router.get("/archiveresults", response=List[ArchiveResultSchema], url_name="get_archiveresult")
+@paginate(CustomPagination)
+def get_archiveresults(request, filters: ArchiveResultFilterSchema = Query(...)):
     """List all ArchiveResult entries matching these filters."""
     qs = ArchiveResult.objects.all()
-    results = filters.filter(qs)
+    results = filters.filter(qs).distinct()
     return results
 
 
-@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema)
+@router.get("/archiveresult/{archiveresult_id}", response=ArchiveResultSchema, url_name="get_archiveresult")
 def get_archiveresult(request, archiveresult_id: str):
-    """Get a specific ArchiveResult by abid, uuid, or pk."""
-    return ArchiveResult.objects.get(Q(pk__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(uuid__icontains=archiveresult_id))
+    """Get a specific ArchiveResult by pk, abid, or old_id."""
+    return ArchiveResult.objects.get(Q(id__icontains=archiveresult_id) | Q(abid__icontains=archiveresult_id) | Q(old_id__icontains=archiveresult_id))
 
 
 # @router.post("/archiveresult", response=ArchiveResultSchema)
@@ -137,12 +186,16 @@ def get_archiveresult(request, archiveresult_id: str):
 
 
 class SnapshotSchema(Schema):
+    TYPE: str = 'core.models.Snapshot'
+
+    id: UUID
+    old_id: UUID
     abid: str
-    uuid: UUID
-    pk: str
+
     modified: datetime
     created: datetime
     created_by_id: str
+    created_by_username: str
 
     url: str
     tags: str
@@ -161,6 +214,11 @@ class SnapshotSchema(Schema):
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
     
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+
     @staticmethod
     def resolve_pk(obj):
         return str(obj.pk)
@@ -189,10 +247,14 @@ class SnapshotSchema(Schema):
 
 
 class SnapshotFilterSchema(FilterSchema):
+    id: Optional[str] = Field(None, q=['id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
+
+    old_id: Optional[str] = Field(None, q='old_id__icontains')
     abid: Optional[str] = Field(None, q='abid__icontains')
-    uuid: Optional[str] = Field(None, q='uuid__icontains')
-    pk: Optional[str] = Field(None, q='pk__icontains')
-    created_by_id: str = Field(None, q='created_by_id__icontains')
+
+    created_by_id: str = Field(None, q='created_by_id')
+    created_by_username: str = Field(None, q='created_by__username__icontains')
+
     created__gte: datetime = Field(None, q='created__gte')
     created__lt: datetime = Field(None, q='created__lt')
     created: datetime = Field(None, q='created')
@@ -200,7 +262,7 @@ class SnapshotFilterSchema(FilterSchema):
     modified__gte: datetime = Field(None, q='modified__gte')
     modified__lt: datetime = Field(None, q='modified__lt')
 
-    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'abid__icontains', 'uuid__icontains'])
+    search: Optional[str] = Field(None, q=['url__icontains', 'title__icontains', 'tags__name__icontains', 'id__icontains', 'abid__icontains', 'old_id__icontains', 'timestamp__startswith'])
     url: Optional[str] = Field(None, q='url')
     tag: Optional[str] = Field(None, q='tags__name')
     title: Optional[str] = Field(None, q='title__icontains')
@@ -211,35 +273,33 @@ class SnapshotFilterSchema(FilterSchema):
 
 
 
-@router.get("/snapshots", response=List[SnapshotSchema])
-@paginate
-def list_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=True):
+@router.get("/snapshots", response=List[SnapshotSchema], url_name="get_snapshots")
+@paginate(CustomPagination)
+def get_snapshots(request, filters: SnapshotFilterSchema = Query(...), with_archiveresults: bool=False):
     """List all Snapshot entries matching these filters."""
     request.with_archiveresults = with_archiveresults
 
     qs = Snapshot.objects.all()
-    results = filters.filter(qs)
+    results = filters.filter(qs).distinct()
     return results
 
-@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema)
+@router.get("/snapshot/{snapshot_id}", response=SnapshotSchema, url_name="get_snapshot")
 def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
     """Get a specific Snapshot by abid, uuid, or pk."""
     request.with_archiveresults = with_archiveresults
     snapshot = None
     try:
-        snapshot = Snapshot.objects.get(Q(uuid__startswith=snapshot_id) | Q(abid__startswith=snapshot_id)| Q(pk__startswith=snapshot_id))
+        snapshot = Snapshot.objects.get(Q(abid__startswith=snapshot_id) | Q(id__startswith=snapshot_id) | Q(old_id__startswith=snapshot_id) | Q(timestamp__startswith=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
     try:
-        snapshot = snapshot or Snapshot.objects.get()
+        snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
     except Snapshot.DoesNotExist:
         pass
 
-    try:
-        snapshot = snapshot or Snapshot.objects.get(Q(uuid__icontains=snapshot_id) | Q(abid__icontains=snapshot_id))
-    except Snapshot.DoesNotExist:
-        pass
+    if not snapshot:
+        raise Snapshot.DoesNotExist
 
     return snapshot
 
@@ -271,21 +331,94 @@ def get_snapshot(request, snapshot_id: str, with_archiveresults: bool=True):
 
 
 class TagSchema(Schema):
-    abid: Optional[UUID] = Field(None, q='abid')
-    uuid: Optional[UUID] = Field(None, q='uuid')
-    pk: Optional[UUID] = Field(None, q='pk')
+    TYPE: str = 'core.models.Tag'
+
+    id: UUID
+    old_id: str
+    abid: str
+
     modified: datetime
     created: datetime
     created_by_id: str
+    created_by_username: str
 
     name: str
     slug: str
+    num_snapshots: int
+    snapshots: List[SnapshotSchema]
 
+    @staticmethod
+    def resolve_old_id(obj):
+        return str(obj.old_id)
 
     @staticmethod
     def resolve_created_by_id(obj):
         return str(obj.created_by_id)
     
-@router.get("/tags", response=List[TagSchema])
-def list_tags(request):
-    return Tag.objects.all()
+    @staticmethod
+    def resolve_created_by_username(obj):
+        User = get_user_model()
+        return User.objects.get(id=obj.created_by_id).username
+    
+    @staticmethod
+    def resolve_num_snapshots(obj, context):
+        return obj.snapshot_set.all().distinct().count()
+
+    @staticmethod
+    def resolve_snapshots(obj, context):
+        if context['request'].with_snapshots:
+            return obj.snapshot_set.all().distinct()
+        return Snapshot.objects.none()
+
+@router.get("/tags", response=List[TagSchema], url_name="get_tags")
+@paginate(CustomPagination)
+def get_tags(request):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+    return Tag.objects.all().distinct()
+
+@router.get("/tag/{tag_id}", response=TagSchema, url_name="get_tag")
+def get_tag(request, tag_id: str, with_snapshots: bool=True):
+    request.with_snapshots = with_snapshots
+    request.with_archiveresults = False
+    tag = None
+    try:
+        tag = tag or Tag.objects.get(old_id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError, ValueError):
+        pass
+
+    try:
+        tag = Tag.objects.get(abid__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+
+    try:
+        tag = tag or Tag.objects.get(id__icontains=tag_id)
+    except (Tag.DoesNotExist, ValidationError):
+        pass
+    return tag
+
+
+
+@router.get("/any/{abid}", response=Union[SnapshotSchema, ArchiveResultSchema, TagSchema], url_name="get_any")
+def get_any(request, abid: str):
+    request.with_snapshots = False
+    request.with_archiveresults = False
+
+    response = None
+    try:
+        response = response or get_snapshot(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_archiveresult(request, abid)
+    except Exception:
+        pass
+
+    try:
+        response = response or get_tag(request, abid)
+    except Exception:
+        pass
+
+    return response

archivebox/config.py

@@ -1036,6 +1036,11 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'enabled': True,
             'is_valid': config['SOURCES_DIR'].exists(),
         },
+        'PERSONAS_DIR': {
+            'path': config['PERSONAS_DIR'].resolve(),
+            'enabled': True,
+            'is_valid': config['PERSONAS_DIR'].exists(),
+        },
         'LOGS_DIR': {
             'path': config['LOGS_DIR'].resolve(),
             'enabled': True,
@@ -1051,11 +1056,6 @@ def get_data_locations(config: ConfigDict) -> ConfigValue:
             'enabled': bool(config['CUSTOM_TEMPLATES_DIR']),
             'is_valid': config['CUSTOM_TEMPLATES_DIR'] and Path(config['CUSTOM_TEMPLATES_DIR']).exists(),
         },
-        'PERSONAS_DIR': {
-            'path': config['PERSONAS_DIR'].resolve(),
-            'enabled': True,
-            'is_valid': config['PERSONAS_DIR'].exists(),
-        },
         # managed by bin/docker_entrypoint.sh and python-crontab:
         # 'CRONTABS_DIR': {
         #     'path': config['CRONTABS_DIR'].resolve(),

archivebox/core/admin.py

@@ -1,17 +1,19 @@
 __package__ = 'archivebox.core'
 
+import json
 from io import StringIO
 from pathlib import Path
 from contextlib import redirect_stdout
 from datetime import datetime, timezone
 
 from django.contrib import admin
-from django.db.models import Count
-from django.urls import path
+from django.db.models import Count, Q
+from django.urls import path, reverse
 from django.utils.html import format_html
 from django.utils.safestring import mark_safe
 from django.shortcuts import render, redirect
 from django.contrib.auth import get_user_model
+from django.core.exceptions import ValidationError
 from django import forms
 
 
@@ -20,7 +22,7 @@ from signal_webhooks.admin import WebhookAdmin, get_webhook_model
 
 from ..util import htmldecode, urldecode, ansi_to_html
 
-from core.models import Snapshot, ArchiveResult, Tag
+from core.models import Snapshot, ArchiveResult, Tag, SnapshotTag
 from core.forms import AddLinkForm
 
 from core.mixins import SearchResultsAdminMixin
@@ -124,31 +126,55 @@ archivebox_admin.get_urls = get_urls(archivebox_admin.get_urls).__get__(archiveb
 
 
 class ArchiveResultInline(admin.TabularInline):
+    name = 'Archive Results Log'
     model = ArchiveResult
+    # fk_name = 'snapshot'
+    extra = 1
+    readonly_fields = ('result_id', 'start_ts', 'end_ts', 'extractor', 'command', 'cmd_version')
+    fields = ('id', *readonly_fields, 'status', 'output')
+    show_change_link = True
+    # # classes = ['collapse']
+    # # list_display_links = ['abid']
+
+    def result_id(self, obj):
+        return format_html('<a href="{}"><small><code>[{}]</code></small></a>', reverse('admin:core_archiveresult_change', args=(obj.id,)), obj.abid)
+    
+    def command(self, obj):
+        return format_html('<small><code>{}</code></small>', " ".join(obj.cmd or []))
+
 
 class TagInline(admin.TabularInline):
-    model = Snapshot.tags.through
+    model = Tag.snapshot_set.through
+    # fk_name = 'snapshot'
+    fields = ('id', 'tag')
+    extra = 1
+    # min_num = 1
+    max_num = 1000
+    autocomplete_fields = (
+        'tag',
+    )
 
 from django.contrib.admin.helpers import ActionForm
-from django.contrib.admin.widgets import AutocompleteSelectMultiple
+from django.contrib.admin.widgets import FilteredSelectMultiple
 
-class AutocompleteTags:
-    model = Tag
-    search_fields = ['name']
-    name = 'tags'
-    remote_field = TagInline
+# class AutocompleteTags:
+#     model = Tag
+#     search_fields = ['name']
+#     name = 'name'
+#     # source_field = 'name'
+#     remote_field = Tag._meta.get_field('name')
 
-class AutocompleteTagsAdminStub:
-    name = 'admin'
+# class AutocompleteTagsAdminStub:
+#     name = 'admin'
 
 
 class SnapshotActionForm(ActionForm):
     tags = forms.ModelMultipleChoiceField(
         queryset=Tag.objects.all(),
         required=False,
-        widget=AutocompleteSelectMultiple(
-            AutocompleteTags(),
-            AutocompleteTagsAdminStub(),
+        widget=FilteredSelectMultiple(
+            'core_tag__name',
+            False,
         ),
     )
 
@@ -168,52 +194,92 @@ def get_abid_info(self, obj):
     return format_html(
         # URL Hash: <code style="font-size: 10px; user-select: all">{}</code><br/>
         '''
-        &nbsp; &nbsp; DB ID:&nbsp; &nbsp; &nbsp; <code style="font-size: 16px; user-select: all; border-radius: 8px; background-color: #fdd; padding: 1px 4px; border: 1px solid #aaa; margin-bottom: 14px;"><b>{}</b></code><br/>
-        &nbsp; &nbsp; &nbsp; &nbsp;.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/>
-        &nbsp; &nbsp; &nbsp; &nbsp;.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code> &nbsp; &nbsp;<br/>
-        <br/>
+        <a href="{}" style="font-size: 16px; font-family: monospace; user-select: all; border-radius: 8px; background-color: #ddf; padding: 3px 5px; border: 1px solid #aaa; margin-bottom: 8px; display: inline-block; vertical-align: top;">{}</a> &nbsp; &nbsp; <a href="{}" style="color: limegreen; font-size: 0.9em; vertical-align: 1px; font-family: monospace;">📖 API DOCS</a>
+        <br/><hr/>
         <div style="opacity: 0.8">
-        &nbsp; &nbsp; ABID: &nbsp; &nbsp; &nbsp; <code style="font-size: 16px; user-select: all; border-radius: 8px; background-color: #fdd; padding: 1px 4px; border: 1px solid #aaa; margin-bottom: 14px;"><b>{}</b></code><br/>
-        &nbsp; &nbsp; &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
-        &nbsp; &nbsp; &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/>
-        &nbsp; &nbsp; &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})  &nbsp; &nbsp; 
-        &nbsp; &nbsp; &nbsp; &nbsp; RAND: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})<br/><hr/>
-        &nbsp; &nbsp; &nbsp; &nbsp; <small style="opacity: 0.8">as ULID: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
-        &nbsp; &nbsp; &nbsp; &nbsp; <small style="opacity: 0.8">as UUID: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/><br/>
+        &nbsp; &nbsp; TS: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all"><b>{}</b></code> &nbsp; &nbsp; &nbsp;&nbsp; ({})<br/>
+        &nbsp; &nbsp; URI: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> &nbsp;&nbsp; &nbsp; &nbsp; &nbsp;&nbsp; (<span style="display:inline-block; vertical-align: -4px; user-select: all; width: 230px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</span>)<br/>
+        &nbsp; &nbsp; SUBTYPE: &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({})  &nbsp; &nbsp; 
+        &nbsp; RAND: &nbsp; <code style="font-size: 10px; user-select: all"><b>{}</b></code> ({}) &nbsp; &nbsp;
+        &nbsp; SALT: &nbsp; <code style="font-size: 10px; user-select: all"><b style="display:inline-block; user-select: all; width: 50px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">{}</b></code>
+        <br/><hr/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.abid.uuid: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; <code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.8">.id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
+        &nbsp; &nbsp; <small style="opacity: 0.5">.old_id: &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;<code style="font-size: 10px; user-select: all">{}</code></small><br/>
         </div>
         ''',
-        obj.pk,
-        obj.id,
-        obj.uuid,
-        obj.abid,
+        obj.api_url, obj.api_url, obj.api_docs_url,
         obj.ABID.ts, obj.abid_values['ts'].isoformat() if isinstance(obj.abid_values['ts'], datetime) else obj.abid_values['ts'],
         obj.ABID.uri, str(obj.abid_values['uri']),
         obj.ABID.subtype, str(obj.abid_values['subtype']),
         obj.ABID.rand, str(obj.abid_values['rand'])[-7:],
-        obj.ABID.ulid,
-        obj.ABID.uuid,
+        obj.ABID.uri_salt,
+        str(obj.abid),
+        str(obj.ABID.uuid),
+        obj.id,
+        getattr(obj, 'old_id', ''),
     )
 
 
 @admin.register(Snapshot, site=archivebox_admin)
 class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
+    class Meta:
+        model = Snapshot
+
     list_display = ('added', 'title_str', 'files', 'size', 'url_str')
+    # list_editable = ('title',)
     sort_fields = ('title_str', 'url_str', 'added', 'files')
-    readonly_fields = ('admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'identifiers')
-    search_fields = ('id', 'url', 'abid', 'uuid', 'timestamp', 'title', 'tags__name')
-    fields = ('url', 'timestamp', 'created_by', 'tags', 'title', *readonly_fields)
-    list_filter = ('added', 'updated', 'tags', 'archiveresult__status', 'created_by')
+    readonly_fields = ('tags', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'added', 'updated', 'created', 'modified', 'API', 'link_dir')
+    search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
+    list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags')
+    fields = ('url', 'created_by', 'title', *readonly_fields)
     ordering = ['-added']
     actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
     autocomplete_fields = ['tags']
-    inlines = [ArchiveResultInline]
+    inlines = [TagInline, ArchiveResultInline]
     list_per_page = SNAPSHOTS_PER_PAGE
 
     action_form = SnapshotActionForm
 
+    save_on_top = True
+
     def changelist_view(self, request, extra_context=None):
         extra_context = extra_context or {}
+        try:
             return super().changelist_view(request, extra_context | GLOBAL_CONTEXT)
+        except Exception as e:
+            self.message_user(request, f'Error occurred while loading the page: {str(e)} {request.GET} {request.POST}')
+            return super().changelist_view(request, GLOBAL_CONTEXT)
+
+    def change_view(self, request, object_id, form_url="", extra_context=None):
+        snapshot = None
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+        
+        try:
+            snapshot = snapshot or Snapshot.objects.get(abid=Snapshot.abid_prefix + object_id.split('_', 1)[-1])
+        except (Snapshot.DoesNotExist, ValidationError):
+            pass
+
+
+        try:
+            snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
+        except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
+            pass
+
+        if snapshot:
+            object_id = str(snapshot.id)
+
+        return super().change_view(
+            request,
+            object_id,
+            form_url,
+            extra_context=extra_context,
+        )
 
     def get_urls(self):
         urls = super().get_urls()
@@ -224,7 +290,7 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
     def get_queryset(self, request):
         self.request = request
-        return super().get_queryset(request).prefetch_related('tags').annotate(archiveresult_count=Count('archiveresult'))
+        return super().get_queryset(request).prefetch_related('tags', 'archiveresult_set').annotate(archiveresult_count=Count('archiveresult'))
 
     def tag_list(self, obj):
         return ', '.join(obj.tags.values_list('name', flat=True))
@@ -285,8 +351,11 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
             obj.extension or '-',
         )
 
-    def identifiers(self, obj):
+    def API(self, obj):
+        try:
             return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
 
     @admin.display(
         description='Title',
@@ -446,20 +515,34 @@ class SnapshotAdmin(SearchResultsAdminMixin, admin.ModelAdmin):
 
 
 
+# @admin.register(SnapshotTag, site=archivebox_admin)
+# class SnapshotTagAdmin(admin.ModelAdmin):
+#     list_display = ('id', 'snapshot', 'tag')
+#     sort_fields = ('id', 'snapshot', 'tag')
+#     search_fields = ('id', 'snapshot_id', 'tag_id')
+#     fields = ('snapshot', 'id')
+#     actions = ['delete_selected']
+#     ordering = ['-id']
+
+#     def API(self, obj):
+#         return get_abid_info(self, obj)
 
 
 @admin.register(Tag, site=archivebox_admin)
 class TagAdmin(admin.ModelAdmin):
-    list_display = ('slug', 'name', 'num_snapshots', 'snapshots', 'abid')
-    sort_fields = ('id', 'name', 'slug', 'abid')
-    readonly_fields = ('created', 'modified', 'identifiers', 'num_snapshots', 'snapshots')
-    search_fields = ('id', 'abid', 'uuid', 'name', 'slug')
-    fields = ('name', 'slug', 'created_by', *readonly_fields, )
+    list_display = ('abid', 'name', 'created', 'created_by', 'num_snapshots', 'snapshots')
+    sort_fields = ('name', 'slug', 'abid', 'created_by', 'created')
+    readonly_fields = ('slug', 'abid', 'created', 'modified', 'API', 'num_snapshots', 'snapshots')
+    search_fields = ('abid', 'name', 'slug')
+    fields = ('name', 'created_by', *readonly_fields)
     actions = ['delete_selected']
-    ordering = ['-id']
+    ordering = ['-created']
 
-    def identifiers(self, obj):
+    def API(self, obj):
+        try:
             return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
 
     def num_snapshots(self, tag):
         return format_html(
@@ -472,11 +555,10 @@ class TagAdmin(admin.ModelAdmin):
         total_count = tag.snapshot_set.count()
         return mark_safe('<br/>'.join(
             format_html(
-                '{} <code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a> {}</code>',
-                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                '<code><a href="/admin/core/snapshot/{}/change"><b>[{}]</b></a></code> {}',
                 snap.pk,
-                snap.abid,
-                snap.url,
+                snap.updated.strftime('%Y-%m-%d %H:%M') if snap.updated else 'pending...',
+                snap.url[:64],
             )
             for snap in tag.snapshot_set.order_by('-updated')[:10]
         ) + (f'<br/><a href="/admin/core/snapshot/?tags__id__exact={tag.id}">and {total_count-10} more...<a>' if tag.snapshot_set.count() > 10 else ''))
@@ -486,9 +568,9 @@ class TagAdmin(admin.ModelAdmin):
 class ArchiveResultAdmin(admin.ModelAdmin):
     list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
     sort_fields = ('start_ts', 'extractor', 'status')
-    readonly_fields = ('snapshot_info', 'tags_str', 'created_by', 'created', 'modified', 'identifiers')
-    search_fields = ('id', 'uuid', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
-    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd',  'start_ts', 'end_ts', 'cmd_version', *readonly_fields)
+    readonly_fields = ('snapshot_info', 'tags_str', 'created', 'modified', 'API')
+    search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
+    fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'cmd',  'start_ts', 'end_ts', 'created_by', 'cmd_version', *readonly_fields)
     autocomplete_fields = ['snapshot']
 
     list_filter = ('status', 'extractor', 'start_ts', 'cmd_version')
@@ -507,8 +589,11 @@ class ArchiveResultAdmin(admin.ModelAdmin):
             result.snapshot.url[:128],
         )
 
-    def identifiers(self, obj):
+    def API(self, obj):
+        try:
             return get_abid_info(self, obj)
+        except Exception as e:
+            return str(e)
 
     @admin.display(
         description='Snapshot Tags'

archivebox/core/migrations/0024_auto_20240513_1143.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations
 from datetime import datetime
-from abid_utils.abid import abid_from_values
+from abid_utils.abid import abid_from_values, DEFAULT_ABID_URI_SALT
 
 
 def calculate_abid(self):
@@ -41,6 +41,7 @@ def calculate_abid(self):
         uri=uri,
         subtype=subtype,
         rand=rand,
+        salt=DEFAULT_ABID_URI_SALT,
     )
     assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
     return abid
@@ -64,7 +65,8 @@ def generate_snapshot_abids(apps, schema_editor):
         snapshot.abid_rand_src = 'self.uuid'
 
         snapshot.abid = calculate_abid(snapshot)
-        snapshot.save(update_fields=["abid"])
+        snapshot.uuid = snapshot.abid.uuid
+        snapshot.save(update_fields=["abid", "uuid"])
 
 def generate_archiveresult_abids(apps, schema_editor):
     print('   Generating ArchiveResult.abid values... (may take an hour or longer for large collections...)')

archivebox/core/migrations/0027_update_snapshot_ids.py

@@ -0,0 +1,106 @@
+# Generated by Django 5.0.6 on 2024-08-18 02:48
+
+from django.db import migrations
+
+from django.db import migrations
+from datetime import datetime
+from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
+
+
+def calculate_abid(self):
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+    prefix = self.abid_prefix
+    ts = eval(self.abid_ts_src)
+    uri = eval(self.abid_uri_src)
+    subtype = eval(self.abid_subtype_src)
+    rand = eval(self.abid_rand_src)
+
+    if (not prefix) or prefix == 'obj_':
+        suggested_abid = self.__class__.__name__[:3].lower()
+        raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+    if not ts:
+        ts = datetime.utcfromtimestamp(0)
+        print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+    if not uri:
+        uri = str(self)
+        print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+    if not subtype:
+        subtype = self.__class__.__name__
+        print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+    if not rand:
+        rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+        print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+    abid = abid_from_values(
+        prefix=prefix,
+        ts=ts,
+        uri=uri,
+        subtype=subtype,
+        rand=rand,
+        salt=DEFAULT_ABID_URI_SALT,
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+    return abid
+
+def update_snapshot_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    num_total = Snapshot.objects.all().count()
+    print(f'   Updating {num_total} Snapshot.id, Snapshot.uuid values in place...')
+    for idx, snapshot in enumerate(Snapshot.objects.all().only('abid').iterator()):
+        assert snapshot.abid
+        snapshot.abid_prefix = 'snp_'
+        snapshot.abid_ts_src = 'self.added'
+        snapshot.abid_uri_src = 'self.url'
+        snapshot.abid_subtype_src = '"01"'
+        snapshot.abid_rand_src = 'self.uuid'
+
+        snapshot.abid = calculate_abid(snapshot)
+        snapshot.uuid = snapshot.abid.uuid
+        snapshot.save(update_fields=["abid", "uuid"])
+        assert str(ABID.parse(snapshot.abid).uuid) == str(snapshot.uuid)
+        if idx % 1000 == 0:
+            print(f'Migrated {idx}/{num_total} Snapshot objects...')
+
+def update_archiveresult_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid', 'snapshot_id').iterator()):
+        assert result.abid
+        result.abid_prefix = 'res_'
+        result.snapshot = Snapshot.objects.get(pk=result.snapshot_id)
+        result.snapshot_added = result.snapshot.added
+        result.snapshot_url = result.snapshot.url
+        result.abid_ts_src = 'self.snapshot_added'
+        result.abid_uri_src = 'self.snapshot_url'
+        result.abid_subtype_src = 'self.extractor'
+        result.abid_rand_src = 'self.id'
+
+        result.abid = calculate_abid(result)
+        result.uuid = result.abid.uuid
+        result.uuid = ABID.parse(result.abid).uuid
+        result.save(update_fields=["abid", "uuid"])
+        assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
+        if idx % 5000 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0026_archiveresult_created_archiveresult_created_by_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(update_snapshot_ids, reverse_code=migrations.RunPython.noop),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+    ]
+
+

archivebox/core/migrations/0028_alter_archiveresult_uuid.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 04:28
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0027_update_snapshot_ids'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4),
+        ),
+    ]

archivebox/core/migrations/0029_alter_archiveresult_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 04:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0028_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.BigIntegerField(primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0030_alter_archiveresult_uuid.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0029_alter_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(unique=True),
+        ),
+    ]

archivebox/core/migrations/0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:09
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0030_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.IntegerField(default=uuid.uuid4, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, null=True, unique=True),
+        ),
+    ]

archivebox/core/migrations/0032_alter_archiveresult_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:20
+
+import core.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0031_alter_archiveresult_id_alter_archiveresult_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

archivebox/core/migrations/0033_rename_id_archiveresult_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0032_alter_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py

@@ -0,0 +1,41 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:37
+
+import core.models
+import uuid
+from django.db import migrations, models
+
+from abid_utils.abid import ABID
+
+
+def update_archiveresult_ids(apps, schema_editor):
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.id, ArchiveResult.uuid values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('abid').iterator()):
+        assert result.abid
+        result.uuid = ABID.parse(result.abid).uuid
+        result.save(update_fields=["uuid"])
+        assert str(ABID.parse(result.abid).uuid) == str(result.uuid)
+        if idx % 2500 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0033_rename_id_archiveresult_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='ID'),
+        ),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0035_remove_archiveresult_uuid_archiveresult_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:49
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0034_alter_archiveresult_old_id_alter_archiveresult_uuid'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='uuid',
+            new_name='id',
+        ),
+    ]

archivebox/core/migrations/0036_alter_archiveresult_id_alter_archiveresult_old_id.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.0.6 on 2024-08-18 05:59
+
+import core.models
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0035_remove_archiveresult_uuid_archiveresult_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, verbose_name='Old ID'),
+        ),
+    ]

archivebox/core/migrations/0037_rename_id_snapshot_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0036_alter_archiveresult_id_alter_archiveresult_old_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0038_rename_uuid_snapshot_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0037_rename_id_snapshot_old_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshot',
+            old_name='uuid',
+            new_name='id',
+        ),
+    ]

archivebox/core/migrations/0039_rename_snapshot_archiveresult_snapshot_old.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0038_rename_uuid_snapshot_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='archiveresult',
+            old_name='snapshot',
+            new_name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0040_archiveresult_snapshot.py

@@ -0,0 +1,34 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:46
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+def update_archiveresult_snapshot_ids(apps, schema_editor):
+    ArchiveResult = apps.get_model("core", "ArchiveResult")
+    Snapshot = apps.get_model("core", "Snapshot")
+    num_total = ArchiveResult.objects.all().count()
+    print(f'   Updating {num_total} ArchiveResult.snapshot_id values in place... (may take an hour or longer for large collections...)')
+    for idx, result in enumerate(ArchiveResult.objects.all().only('snapshot_old_id').iterator(chunk_size=5000)):
+        assert result.snapshot_old_id
+        snapshot = Snapshot.objects.only('id').get(old_id=result.snapshot_old_id)
+        result.snapshot_id = snapshot.id
+        result.save(update_fields=["snapshot_id"])
+        assert str(result.snapshot_id) == str(snapshot.id)
+        if idx % 5000 == 0:
+            print(f'Migrated {idx}/{num_total} ArchiveResult objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0039_rename_snapshot_archiveresult_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults', to='core.snapshot', to_field='id'),
+        ),
+        migrations.RunPython(update_archiveresult_snapshot_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0041_alter_archiveresult_snapshot_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:50
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0040_archiveresult_snapshot'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot_old',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='archiveresults_old', to='core.snapshot'),
+        ),
+    ]

archivebox/core/migrations/0042_remove_archiveresult_snapshot_old.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0041_alter_archiveresult_snapshot_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='archiveresult',
+            name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 5.0.6 on 2024-08-18 06:52
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0042_remove_archiveresult_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+    ]

archivebox/core/migrations/0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more.py

@@ -0,0 +1,40 @@
+# Generated by Django 5.0.6 on 2024-08-19 23:01
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0043_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
+    ]
+
+    operations = [
+        migrations.SeparateDatabaseAndState(
+            database_operations=[
+                # No-op, SnapshotTag model already exists in DB
+            ],
+            state_operations=[
+                migrations.CreateModel(
+                    name='SnapshotTag',
+                    fields=[
+                        ('id', models.AutoField(primary_key=True, serialize=False)),
+                        ('snapshot', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.snapshot')),
+                        ('tag', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='core.tag')),
+                    ],
+                    options={
+                        'db_table': 'core_snapshot_tags',
+                        'unique_together': {('snapshot', 'tag')},
+                    },
+                ),
+                migrations.AlterField(
+                    model_name='snapshot',
+                    name='tags',
+                    field=models.ManyToManyField(blank=True, related_name='snapshot_set', through='core.SnapshotTag', to='core.tag'),
+                ),
+            ],
+        ),
+    ]

archivebox/core/migrations/0045_alter_snapshot_old_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 01:54
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0044_alter_archiveresult_snapshot_alter_tag_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 5.0.6 on 2024-08-20 01:55
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0045_alter_snapshot_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='snapshot',
+            name='old_id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0047_alter_snapshottag_unique_together_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:16
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0046_alter_archiveresult_snapshot_alter_snapshot_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
+        ),
+    ]

archivebox/core/migrations/0048_alter_archiveresult_snapshot_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:17
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0047_alter_snapshottag_unique_together_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='archiveresult',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
+        ),
+    ]

archivebox/core/migrations/0049_rename_snapshot_snapshottag_snapshot_old_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0048_alter_archiveresult_snapshot_and_more'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshottag',
+            old_name='snapshot',
+            new_name='snapshot_old',
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot_old', 'tag')},
+        ),
+    ]

archivebox/core/migrations/0050_alter_snapshottag_snapshot_old.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:30
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0049_rename_snapshot_snapshottag_snapshot_old_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot_old',
+            field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot', to_field='old_id'),
+        ),
+    ]

archivebox/core/migrations/0051_snapshottag_snapshot_alter_snapshottag_snapshot_old.py

@@ -0,0 +1,40 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:31
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def update_snapshottag_ids(apps, schema_editor):
+    Snapshot = apps.get_model("core", "Snapshot")
+    SnapshotTag = apps.get_model("core", "SnapshotTag")
+    num_total = SnapshotTag.objects.all().count()
+    print(f'   Updating {num_total} SnapshotTag.snapshot_id values in place... (may take an hour or longer for large collections...)')
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('snapshot_old_id').iterator()):
+        assert snapshottag.snapshot_old_id
+        snapshot = Snapshot.objects.get(old_id=snapshottag.snapshot_old_id)
+        snapshottag.snapshot_id = snapshot.id
+        snapshottag.save(update_fields=["snapshot_id"])
+        assert str(snapshottag.snapshot_id) == str(snapshot.id)
+        if idx % 100 == 0:
+            print(f'Migrated {idx}/{num_total} SnapshotTag objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0050_alter_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(blank=True, db_column='snapshot_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot_old',
+            field=models.ForeignKey(db_column='snapshot_old_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottag_old_set', to='core.snapshot', to_field='old_id'),
+        ),
+        migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0052_alter_snapshottag_unique_together_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:37
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0051_snapshottag_snapshot_alter_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together=set(),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='snapshot',
+            field=models.ForeignKey(db_column='snapshot_id', on_delete=django.db.models.deletion.CASCADE, to='core.snapshot'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot', 'tag')},
+        ),
+    ]

archivebox/core/migrations/0053_remove_snapshottag_snapshot_old.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:38
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0052_alter_snapshottag_unique_together_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='snapshottag',
+            name='snapshot_old',
+        ),
+    ]

archivebox/core/migrations/0054_alter_snapshot_timestamp.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 02:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0053_remove_snapshottag_snapshot_old'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshot',
+            name='timestamp',
+            field=models.CharField(db_index=True, editable=False, max_length=32, unique=True),
+        ),
+    ]

archivebox/core/migrations/0055_alter_tag_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0054_alter_snapshot_timestamp'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='slug',
+            field=models.SlugField(editable=False, max_length=100, unique=True),
+        ),
+    ]

archivebox/core/migrations/0056_remove_tag_uuid.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0055_alter_tag_slug'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='tag',
+            name='uuid',
+        ),
+    ]

archivebox/core/migrations/0057_rename_id_tag_old_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:29
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0056_remove_tag_uuid'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='tag',
+            old_name='id',
+            new_name='old_id',
+        ),
+    ]

archivebox/core/migrations/0058_alter_tag_old_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:30
+
+import core.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0057_rename_id_tag_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, primary_key=True, serialize=False, verbose_name='Old ID'),
+        ),
+    ]

archivebox/core/migrations/0059_tag_id.py

@@ -0,0 +1,81 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:33
+
+from django.db import migrations, models
+from abid_utils.models import ABID, abid_from_values
+
+
+def calculate_abid(self):
+    """
+    Return a freshly derived ABID (assembled from attrs defined in ABIDModel.abid_*_src).
+    """
+    prefix = self.abid_prefix
+    ts = eval(self.abid_ts_src)
+    uri = eval(self.abid_uri_src)
+    subtype = eval(self.abid_subtype_src)
+    rand = eval(self.abid_rand_src)
+
+    if (not prefix) or prefix == 'obj_':
+        suggested_abid = self.__class__.__name__[:3].lower()
+        raise Exception(f'{self.__class__.__name__}.abid_prefix must be defined to calculate ABIDs (suggested: {suggested_abid})')
+
+    if not ts:
+        ts = datetime.utcfromtimestamp(0)
+        print(f'[!] WARNING: Generating ABID with ts=0000000000 placeholder because {self.__class__.__name__}.abid_ts_src={self.abid_ts_src} is unset!', ts.isoformat())
+
+    if not uri:
+        uri = str(self)
+        print(f'[!] WARNING: Generating ABID with uri=str(self) placeholder because {self.__class__.__name__}.abid_uri_src={self.abid_uri_src} is unset!', uri)
+
+    if not subtype:
+        subtype = self.__class__.__name__
+        print(f'[!] WARNING: Generating ABID with subtype={subtype} placeholder because {self.__class__.__name__}.abid_subtype_src={self.abid_subtype_src} is unset!', subtype)
+
+    if not rand:
+        rand = getattr(self, 'uuid', None) or getattr(self, 'id', None) or getattr(self, 'pk')
+        print(f'[!] WARNING: Generating ABID with rand=self.id placeholder because {self.__class__.__name__}.abid_rand_src={self.abid_rand_src} is unset!', rand)
+
+    abid = abid_from_values(
+        prefix=prefix,
+        ts=ts,
+        uri=uri,
+        subtype=subtype,
+        rand=rand,
+    )
+    assert abid.ulid and abid.uuid and abid.typeid, f'Failed to calculate {prefix}_ABID for {self.__class__.__name__}'
+    return abid
+
+
+def update_archiveresult_ids(apps, schema_editor):
+    Tag = apps.get_model("core", "Tag")
+    num_total = Tag.objects.all().count()
+    print(f'   Updating {num_total} Tag.id, ArchiveResult.uuid values in place...')
+    for idx, tag in enumerate(Tag.objects.all().iterator()):
+        assert tag.name
+        tag.abid_prefix = 'tag_'
+        tag.abid_ts_src = 'self.created'
+        tag.abid_uri_src = 'self.slug'
+        tag.abid_subtype_src = '"03"'
+        tag.abid_rand_src = 'self.old_id'
+        tag.abid = calculate_abid(tag)
+        tag.id = tag.abid.uuid
+        tag.save(update_fields=["abid", "id"])
+        assert str(ABID.parse(tag.abid).uuid) == str(tag.id)
+        if idx % 10 == 0:
+            print(f'Migrated {idx}/{num_total} Tag objects...')
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0058_alter_tag_old_id'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(blank=True, null=True),
+        ),
+        migrations.RunPython(update_archiveresult_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0060_alter_tag_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:42
+
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0059_tag_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+        ),
+    ]

archivebox/core/migrations/0061_rename_tag_snapshottag_old_tag_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0060_alter_tag_id'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='snapshottag',
+            old_name='tag',
+            new_name='old_tag',
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot', 'old_tag')},
+        ),
+    ]

archivebox/core/migrations/0062_alter_snapshottag_old_tag.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:44
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0061_rename_tag_snapshottag_old_tag_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='old_tag',
+            field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
+        ),
+    ]

archivebox/core/migrations/0063_snapshottag_tag_alter_snapshottag_old_tag.py

@@ -0,0 +1,40 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:45
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+def update_snapshottag_ids(apps, schema_editor):
+    Tag = apps.get_model("core", "Tag")
+    SnapshotTag = apps.get_model("core", "SnapshotTag")
+    num_total = SnapshotTag.objects.all().count()
+    print(f'   Updating {num_total} SnapshotTag.tag_id values in place... (may take an hour or longer for large collections...)')
+    for idx, snapshottag in enumerate(SnapshotTag.objects.all().only('old_tag_id').iterator()):
+        assert snapshottag.old_tag_id
+        tag = Tag.objects.get(old_id=snapshottag.old_tag_id)
+        snapshottag.tag_id = tag.id
+        snapshottag.save(update_fields=["tag_id"])
+        assert str(snapshottag.tag_id) == str(tag.id)
+        if idx % 100 == 0:
+            print(f'Migrated {idx}/{num_total} SnapshotTag objects...')
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0062_alter_snapshottag_old_tag'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(blank=True, db_column='tag_id', null=True, on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='old_tag',
+            field=models.ForeignKey(db_column='old_tag_id', on_delete=django.db.models.deletion.CASCADE, related_name='snapshottags_old', to='core.tag'),
+        ),
+        migrations.RunPython(update_snapshottag_ids, reverse_code=migrations.RunPython.noop),
+    ]

archivebox/core/migrations/0064_alter_snapshottag_unique_together_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:50
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0063_snapshottag_tag_alter_snapshottag_old_tag'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together=set(),
+        ),
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='snapshottag',
+            unique_together={('snapshot', 'tag')},
+        ),
+    ]

archivebox/core/migrations/0065_remove_snapshottag_old_tag.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:51
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0064_alter_snapshottag_unique_together_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='snapshottag',
+            name='old_tag',
+        ),
+    ]

archivebox/core/migrations/0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id.py

@@ -0,0 +1,31 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:52
+
+import core.models
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0065_remove_snapshottag_old_tag'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag', to_field='id'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='old_id',
+            field=models.BigIntegerField(default=core.models.rand_int_id, serialize=False, unique=True, verbose_name='Old ID'),
+        ),
+    ]

archivebox/core/migrations/0067_alter_snapshottag_tag.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.6 on 2024-08-20 03:53
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0066_alter_snapshottag_tag_alter_tag_id_alter_tag_old_id'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='snapshottag',
+            name='tag',
+            field=models.ForeignKey(db_column='tag_id', on_delete=django.db.models.deletion.CASCADE, to='core.tag'),
+        ),
+    ]

archivebox/core/migrations/0068_alter_archiveresult_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.6 on 2024-08-20 07:26
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0067_alter_snapshottag_tag'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='archiveresult',
+            options={'verbose_name': 'Archive Result', 'verbose_name_plural': 'Archive Results Log'},
+        ),
+    ]

archivebox/core/models.py

@@ -5,6 +5,7 @@ from typing import Optional, List, Dict
 from django_stubs_ext.db.models import TypedModelMeta
 
 import json
+import random
 
 import uuid
 from uuid import uuid4
@@ -14,9 +15,8 @@ from django.db import models
 from django.utils.functional import cached_property
 from django.utils.text import slugify
 from django.core.cache import cache
-from django.urls import reverse
+from django.urls import reverse, reverse_lazy
 from django.db.models import Case, When, Value, IntegerField
-from django.contrib.auth.models import User   # noqa
 
 from abid_utils.models import ABIDModel, ABIDField
 
@@ -35,6 +35,8 @@ STATUS_CHOICES = [
     ("skipped", "skipped")
 ]
 
+def rand_int_id():
+    return random.getrandbits(32)
 
 
 # class BaseModel(models.Model):
@@ -48,24 +50,26 @@ STATUS_CHOICES = [
 #         abstract = True
 
 
+
+
 class Tag(ABIDModel):
     """
     Based on django-taggit model + ABID base.
     """
     abid_prefix = 'tag_'
     abid_ts_src = 'self.created'          # TODO: add created/modified time
-    abid_uri_src = 'self.name'
+    abid_uri_src = 'self.slug'
     abid_subtype_src = '"03"'
-    abid_rand_src = 'self.id'
+    abid_rand_src = 'self.old_id'
 
-    # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
-    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID')  # legacy PK
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
     abid = ABIDField(prefix=abid_prefix)
 
 
     name = models.CharField(unique=True, blank=False, max_length=100)
-    slug = models.SlugField(unique=True, blank=True, max_length=100)
+    slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
     # slug is autoset on save from name, never set it manually
 
 
@@ -76,6 +80,10 @@ class Tag(ABIDModel):
     def __str__(self):
         return self.name
 
+    # @property
+    # def old_id(self):
+    #     return self.id
+
     def slugify(self, tag, i=None):
         slug = slugify(tag)
         if i is not None:
@@ -104,37 +112,66 @@ class Tag(ABIDModel):
         else:
             return super().save(*args, **kwargs)
         
+    @property
+    def api_url(self) -> str:
+        # /api/v1/core/snapshot/{uulid}
+        return reverse_lazy('api-1:get_tag', args=[self.abid])
+
+    @property
+    def api_docs_url(self) -> str:
+        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_tag'
+
+class SnapshotTag(models.Model):
+    id = models.AutoField(primary_key=True)
+
+    snapshot = models.ForeignKey('Snapshot', db_column='snapshot_id', on_delete=models.CASCADE, to_field='id')
+    tag = models.ForeignKey(Tag, db_column='tag_id', on_delete=models.CASCADE, to_field='id')
+
+    class Meta:
+        db_table = 'core_snapshot_tags'
+        unique_together = [('snapshot', 'tag')]
 
 class Snapshot(ABIDModel):
     abid_prefix = 'snp_'
     abid_ts_src = 'self.added'
     abid_uri_src = 'self.url'
     abid_subtype_src = '"01"'
-    abid_rand_src = 'self.id'
+    abid_rand_src = 'self.old_id'
 
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)  # legacy pk
-    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    old_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)  # legacy pk
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True)
     abid = ABIDField(prefix=abid_prefix)
 
     url = models.URLField(unique=True, db_index=True)
-    timestamp = models.CharField(max_length=32, unique=True, db_index=True)
+    timestamp = models.CharField(max_length=32, unique=True, db_index=True, editable=False)
 
     title = models.CharField(max_length=512, null=True, blank=True, db_index=True)
     
+    tags = models.ManyToManyField(Tag, blank=True, through=SnapshotTag, related_name='snapshot_set', through_fields=('snapshot', 'tag'))
+
     added = models.DateTimeField(auto_now_add=True, db_index=True)
     updated = models.DateTimeField(auto_now=True, blank=True, null=True, db_index=True)
-    tags = models.ManyToManyField(Tag, blank=True)
 
     keys = ('url', 'timestamp', 'title', 'tags', 'updated')
 
+    @property
+    def uuid(self):
+        return self.id
 
     def __repr__(self) -> str:
-        title = self.title or '-'
-        return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
+        title = (self.title_stripped or '-')[:64]
+        return f'[{self.timestamp}] {self.url[:64]} ({title})'
 
     def __str__(self) -> str:
-        title = self.title or '-'
-        return f'[{self.timestamp}] {self.url[:64]} ({title[:64]})'
+        title = (self.title_stripped or '-')[:64]
+        return f'[{self.timestamp}] {self.url[:64]} ({title})'
+
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        try:
+            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'Snapshot.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
+        except AssertionError as e:
+            print(e)
 
     @classmethod
     def from_json(cls, info: dict):
@@ -168,6 +205,19 @@ class Snapshot(ABIDModel):
     def icons(self) -> str:
         return snapshot_icons(self)
     
+    @property
+    def api_url(self) -> str:
+        # /api/v1/core/snapshot/{uulid}
+        return reverse_lazy('api-1:get_snapshot', args=[self.abid])
+    
+    @property
+    def api_docs_url(self) -> str:
+        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_snapshot'
+    
+    @cached_property
+    def title_stripped(self) -> str:
+        return (self.title or '').replace("\n", " ").replace("\r", "")
+
     @cached_property
     def extension(self) -> str:
         from ..util import extension
@@ -317,21 +367,21 @@ class ArchiveResultManager(models.Manager):
             qs = qs.annotate(indexing_precedence=Case(*precedence, default=Value(1000),output_field=IntegerField())).order_by('indexing_precedence')
         return qs
 
-
 class ArchiveResult(ABIDModel):
     abid_prefix = 'res_'
     abid_ts_src = 'self.snapshot.added'
     abid_uri_src = 'self.snapshot.url'
     abid_subtype_src = 'self.extractor'
-    abid_rand_src = 'self.uuid'
+    abid_rand_src = 'self.id'
     EXTRACTOR_CHOICES = EXTRACTOR_CHOICES
 
-    id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')   # legacy pk TODO: move to UUIDField
-    # id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    uuid = models.UUIDField(blank=True, null=True, editable=True, unique=True)
+    old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
     abid = ABIDField(prefix=abid_prefix)
 
-    snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE)
+    snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')
+
     extractor = models.CharField(choices=EXTRACTOR_CHOICES, max_length=32)
     cmd = models.JSONField()
     pwd = models.CharField(max_length=256)
@@ -344,15 +394,36 @@ class ArchiveResult(ABIDModel):
     objects = ArchiveResultManager()
 
     class Meta(TypedModelMeta):
-        verbose_name = 'Result'
+        verbose_name = 'Archive Result'
+        verbose_name_plural = 'Archive Results Log'
+        
 
     def __str__(self):
         return self.extractor
 
+    def save(self, *args, **kwargs):
+        super().save(*args, **kwargs)
+        try:
+            assert str(self.id) == str(self.ABID.uuid) == str(self.uuid), f'ArchiveResult.id ({self.id}) does not match .ABID.uuid ({self.ABID.uuid})'
+        except AssertionError as e:
+            print(e)
+
+    @property
+    def uuid(self):
+        return self.id
+
     @cached_property
     def snapshot_dir(self):
         return Path(self.snapshot.link_dir)
 
+    @property
+    def api_url(self) -> str:
+        # /api/v1/core/archiveresult/{uulid}
+        return reverse_lazy('api-1:get_archiveresult', args=[self.abid])
+    
+    @property
+    def api_docs_url(self) -> str:
+        return f'/api/v1/docs#/Core%20Models/api_v1_core_get_archiveresult'
 
     @property
     def extractor_module(self):

archivebox/core/settings.py

@@ -120,6 +120,8 @@ MIDDLEWARE = [
 ### Authentication Settings
 ################################################################################
 
+# AUTH_USER_MODEL = 'auth.User'   # cannot be easily changed unfortunately
+
 AUTHENTICATION_BACKENDS = [
     'django.contrib.auth.backends.RemoteUserBackend',
     'django.contrib.auth.backends.ModelBackend',
@@ -463,6 +465,7 @@ SIGNAL_WEBHOOKS = {
     },
 }
 
+DATA_UPLOAD_MAX_NUMBER_FIELDS = None
 
 ADMIN_DATA_VIEWS = {
     "NAME": "Environment",

archivebox/core/urls.py

@@ -38,7 +38,7 @@ urlpatterns = [
     path('accounts/', include('django.contrib.auth.urls')),
     path('admin/', archivebox_admin.urls),
     
-    path("api/",      include('api.urls')),
+    path("api/",      include('api.urls'), name='api'),
 
     path('health/', HealthCheckView.as_view(), name='healthcheck'),
     path('error/', lambda *_: 1/0),

archivebox/core/views.py

@@ -181,6 +181,7 @@ class SnapshotView(View):
         except (IndexError, ValueError):
             slug, archivefile = path.split('/', 1)[0], 'index.html'
 
+
         # slug is a timestamp
         if slug.replace('.','').isdigit():
 
@@ -227,7 +228,7 @@ class SnapshotView(View):
                         snap.timestamp,
                         snap.timestamp,
                         snap.url,
-                        snap.title or '',
+                        snap.title_stripped[:64] or '',
                     )
                     for snap in Snapshot.objects.filter(timestamp__startswith=slug).only('url', 'timestamp', 'title', 'added').order_by('-added')
                 )
@@ -278,12 +279,35 @@ class SnapshotView(View):
                     content_type="text/html",
                     status=404,
                 )
+            
+        # # slud is an ID
+        # ulid = slug.split('_', 1)[-1]
+        # try:
+        #     try:
+        #         snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid) | Q(old_id=ulid))
+        #     except Snapshot.DoesNotExist:
+        #         pass
+
+        #     try:
+        #         snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug) | Q(old_id__startswith=slug))
+        #     except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned):
+        #         pass
+
+        #     try:
+        #         snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
+        #     except Snapshot.DoesNotExist:
+        #         pass
+        #     return redirect(f'/archive/{snapshot.timestamp}/index.html')
+        # except Snapshot.DoesNotExist:
+        #     pass
+
         # slug is a URL
         try:
             try:
-                # try exact match on full url first
+                # try exact match on full url / ABID first
                 snapshot = Snapshot.objects.get(
                     Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
+                    | Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
                 )
             except Snapshot.DoesNotExist:
                 # fall back to match on exact base_url
@@ -317,15 +341,17 @@ class SnapshotView(View):
         except Snapshot.MultipleObjectsReturned:
             snapshot_hrefs = mark_safe('<br/>').join(
                 format_html(
-                    '{} <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
+                    '{} <code style="font-size: 0.8em">{}</code> <a href="/archive/{}/index.html"><b><code>{}</code></b></a> {} <b>{}</b>',
                     snap.added.strftime('%Y-%m-%d %H:%M:%S'),
+                    snap.abid,
                     snap.timestamp,
                     snap.timestamp,
                     snap.url,
-                    snap.title or '',
+                    snap.title_stripped[:64] or '',
                 )
                 for snap in Snapshot.objects.filter(
                     Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
+                    | Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
                 ).only('url', 'timestamp', 'title', 'added').order_by('-added')
             )
             return HttpResponse(

archivebox/index/schema.py

@@ -266,7 +266,7 @@ class Link:
     @cached_property
     def snapshot(self):
         from core.models import Snapshot
-        return Snapshot.objects.only('uuid').get(url=self.url)
+        return Snapshot.objects.only('id').get(url=self.url)
 
     @cached_property
     def snapshot_id(self):
@@ -274,7 +274,7 @@ class Link:
 
     @cached_property
     def snapshot_uuid(self):
-        return str(self.snapshot.uuid)
+        return str(self.snapshot.id)
 
     @cached_property
     def snapshot_abid(self):

archivebox/manage.py

@@ -7,7 +7,7 @@ if __name__ == '__main__':
     # versions of ./manage.py commands whenever possible. When that's not possible
     # (e.g. makemigrations), you can comment out this check temporarily
 
-    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv):
+    if not ('makemigrations' in sys.argv or 'migrate' in sys.argv or 'startapp' in sys.argv or 'squashmigrations' in sys.argv):
         print("[X] Don't run ./manage.py directly (unless you are a developer running makemigrations):")
         print()
         print('    Hint: Use these archivebox CLI commands instead of the ./manage.py equivalents:')

archivebox/templates/admin/base.html

@@ -45,6 +45,13 @@
             {% endif %}
         {% endblock %}
         
+        <script
+            src="https://code.jquery.com/jquery-3.7.1.slim.min.js"
+            integrity="sha256-kmHvs0B+OpCW5GVHUNjv9rOmY0IvSIRcf7zGUDTDQM8="
+            crossorigin="anonymous"></script>
+        <link href="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/css/select2.min.css" rel="stylesheet" />
+        <script src="https://cdn.jsdelivr.net/npm/select2@4.1.0-rc.0/dist/js/select2.min.js"></script>
+
         <link rel="stylesheet" type="text/css" href="{% static "admin.css" %}">
         
         <script>
@@ -264,6 +271,11 @@
                         .appendTo(buttons)
                 })
                 console.log('Converted', buttons.children().length, 'admin actions from dropdown to buttons')
+                jQuery('select[multiple]').select2();
+            }
+            function fixInlineAddRow() {
+                $('#id_snapshottag-MAX_NUM_FORMS').val('1000')
+                $('.add-row').show()
             }
 
             function setupSnapshotGridListToggle() {
@@ -281,7 +293,7 @@
                 // if we arrive at the index with a url like ??id__startswith=...
                 // we were hotlinked here with the intention of making it easy for the user to perform some
                 // actions on the given snapshot. therefore we should preselect the snapshot to save them a click
-                if (window.location.search.startsWith('?id__startswith=') || window.location.search.startsWith('?id__exact=')) {
+                if (window.location.search.startsWith('?')) {
                     const result_checkboxes = [...document.querySelectorAll('#result_list .action-checkbox input[type=checkbox]')]
                     if (result_checkboxes.length === 1) {
                         result_checkboxes[0].click()
@@ -290,6 +302,7 @@
             }
             $(document).ready(function() {
                 fix_actions()
+                fixInlineAddRow()
                 setupSnapshotGridListToggle()
                 setTimeOffset()
                 selectSnapshotIfHotlinked()

archivebox/templates/core/snapshot.html

@@ -351,7 +351,7 @@
                             <a href="warc/" title="Any WARC archives for the page">WARC</a> | 
                             <a href="media/" title="Audio, Video, and Subtitle files.">Media</a> | 
                             <a href="git/" title="Any git repos at the url">Git</a> | 
-                            <a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> | 
+                            <a href="/admin/core/snapshot/?q={{snapshot_id}}" title="Go to the Snapshot admin to update, overwrite, or delete this Snapshot">Actions</a> | 
                             <a href="/admin/core/snapshot/{{snapshot_id}}/change/" title="Edit this snapshot in the Admin UI">Admin</a> | 
                             <a href="." title="Webserver-provided index of files directory.">See all files...</a><br/>
                         </div>

archivebox/templates/core/snapshot_live.html

@@ -349,7 +349,7 @@
                             </a>
                         </div>
                         <div class="badge badge-{{status_color}}" style="float: left">
-                            <a href="/admin/core/snapshot/?id__startswith={{snapshot_id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
+                            <a href="/admin/core/snapshot/?q={{snapshot_id}}" title="Click to see options to pull, re-snapshot, or delete this Snapshot">
                                 {{status|upper}}
                             </a>
                         </div>

archivebox/templates/static/admin.css

@@ -116,7 +116,6 @@ body.model-snapshot.change-list #content .object-tools {
     margin-right: 0px;
     width: auto;
     max-height: 40px;
-    overflow: hidden;
     display: block;
 }
 @media (max-width: 1000px) {
@@ -166,14 +165,14 @@ body.model-snapshot.change-list #content .object-tools {
     margin-right: 25px;
 }
 
-#content #changelist .actions .select2-selection {
+#content #changelist .actions > label {
     max-height: 25px;
 }
-#content #changelist .actions .select2-container--admin-autocomplete.select2-container {
+#content #changelist .actions > label {
     width: auto !important;
     min-width: 90px;
 }
-#content #changelist .actions .select2-selection__rendered .select2-selection__choice {
+#content #changelist .actions > label > select {
     margin-top: 3px;
 }