Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-21 11:43:07 +00:00
Code Issues Projects Releases Packages Wiki Activity

rename abid_utils to base_models

Browse source
This commit is contained in:
Nick Sweeting 2024-11-18 19:40:05 -08:00
parent 0db6437c4e
commit 569081a9eb
No known key found for this signature in database
38 changed files with 80 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
archivebox/__init__.py
View file

@ -85,8 +85,8 @@ ARCHIVEBOX_BUILTIN_PLUGINS = {
'workers': PACKAGE_DIR / 'workers',
'core': PACKAGE_DIR / 'core',
'crawls': PACKAGE_DIR / 'crawls',
# 'machine': PACKAGE_DIR / 'machine'
# 'search': PACKAGE_DIR / 'search',
# 'core': PACKAGE_DIR / 'core',
}
# Load all user-defined ArchiveBox plugins

1
archivebox/abid_utils/__init__.py
View file

@ -1 +0,0 @@
__package__ = 'abid_utils'

7
archivebox/abid_utils/apps.py
View file

@ -1,7 +0,0 @@
from django.apps import AppConfig
class AbidUtilsConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'abid_utils'

3
archivebox/abid_utils/tests.py
View file

@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.

2
archivebox/api/admin.py
View file

@ -3,7 +3,7 @@ __package__ = 'archivebox.api'
from signal_webhooks.admin import WebhookAdmin
from signal_webhooks.utils import get_webhook_model
from abid_utils.admin import ABIDModelAdmin
from archivebox.base_models.admin import ABIDModelAdmin
from api.models import APIToken

5
archivebox/api/migrations/0003_rename_user_apitoken_created_by_apitoken_abid_and_more.py
View file

@ -1,6 +1,5 @@
# Generated by Django 5.0.6 on 2024-06-03 01:52
import abid_utils.models
import charidfield.fields
import django.db.models.deletion
import signal_webhooks.fields
@ -9,6 +8,8 @@ import uuid
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -63,7 +64,7 @@ class Migration(migrations.Migration):
('id', models.UUIDField(blank=True, null=True, unique=True)),
('uuid', models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False)),
('abid', charidfield.fields.CharIDField(blank=True, db_index=True, default=None, help_text='ABID-format identifier for this entity (e.g. snp_01BJQMF54D093DXEAWZ6JYRPAQ)', max_length=30, null=True, prefix='whk_', unique=True)),
('created_by', models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
('created_by', models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'verbose_name': 'API Outbound Webhook',

5
archivebox/api/migrations/0007_alter_apitoken_created_by.py
View file

@ -1,10 +1,11 @@
# Generated by Django 5.1 on 2024-08-20 22:52
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -17,6 +18,6 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='apitoken',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
]

7
archivebox/api/migrations/0008_alter_apitoken_created_alter_apitoken_created_by_and_more.py
View file

@ -1,10 +1,11 @@
# Generated by Django 5.1 on 2024-09-04 23:32
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -17,7 +18,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='apitoken',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='apitoken',
@ -32,7 +33,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='outboundwebhook',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='outboundwebhook',

5
archivebox/api/migrations/0009_rename_created_apitoken_created_at_and_more.py
View file

@ -1,8 +1,9 @@
# Generated by Django 5.1 on 2024-09-05 00:26
import abid_utils.models
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -29,7 +30,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='outboundwebhook',
name='created_at',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='outboundwebhook',

2
archivebox/api/models.py
View file

@ -11,7 +11,7 @@ from signal_webhooks.models import WebhookBase
from django_stubs_ext.db.models import TypedModelMeta
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField
from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField

1
archivebox/base_models/__init__.py Normal file
View file

@ -0,0 +1 @@
__package__ = 'archivebox.base_models'

2
archivebox/abid_utils/abid.py → archivebox/base_models/abid.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.abid_utils'
__package__ = 'archivebox.base_models'
from typing import NamedTuple, Any, Union, Dict

2
archivebox/abid_utils/admin.py → archivebox/base_models/admin.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.abid_utils'
__package__ = 'archivebox.base_models'
from typing import Any

7
archivebox/base_models/apps.py Normal file
View file

@ -0,0 +1,7 @@
# from django.apps import AppConfig
# class AbidUtilsConfig(AppConfig):
# default_auto_field = 'django.db.models.BigAutoField'
# name = 'base_models'

0
archivebox/abid_utils/migrations/__init__.py → archivebox/base_models/migrations/__init__.py
View file

0
archivebox/abid_utils/models.py → archivebox/base_models/models.py
View file

2
archivebox/cli/archivebox_add.py
View file

@ -53,7 +53,7 @@ def add(urls: str | list[str],
from crawls.models import Seed, Crawl
from workers.orchestrator import Orchestrator
from abid_utils.models import get_or_create_system_user_pk
from archivebox.base_models.models import get_or_create_system_user_pk

2
archivebox/core/admin_archiveresults.py
View file

@ -17,8 +17,8 @@ import abx
from archivebox.config import DATA_DIR
from archivebox.config.common import SERVER_CONFIG
from archivebox.misc.paginators import AccelleratedPaginator
from archivebox.base_models.admin import ABIDModelAdmin
from abid_utils.admin import ABIDModelAdmin
from core.models import ArchiveResult, Snapshot

2
archivebox/core/admin_snapshots.py
View file

@ -23,7 +23,7 @@ from archivebox.index.html import snapshot_icons
from archivebox.extractors import archive_links
from archivebox.main import remove
from archivebox.abid_utils.admin import ABIDModelAdmin
from archivebox.base_models.admin import ABIDModelAdmin
from archivebox.workers.tasks import bg_archive_links, bg_add
from core.models import Tag

2
archivebox/core/admin_tags.py
View file

@ -5,8 +5,8 @@ from django.utils.html import format_html, mark_safe
import abx
from abid_utils.admin import ABIDModelAdmin
from archivebox.misc.paginators import AccelleratedPaginator
from archivebox.base_models.admin import ABIDModelAdmin
from core.models import Tag

3
archivebox/core/migrations/0024_auto_20240513_1143.py
View file

@ -2,7 +2,8 @@
from django.db import migrations
from datetime import datetime
from abid_utils.abid import abid_from_values, DEFAULT_ABID_URI_SALT
from archivebox.base_models.abid import abid_from_values, DEFAULT_ABID_URI_SALT
def calculate_abid(self):

15
archivebox/core/migrations/0026_archiveresult_created_archiveresult_created_by_and_more.py
View file

@ -1,11 +1,12 @@
# Generated by Django 5.0.6 on 2024-05-13 13:01
import abid_utils.models
import django.db.models.deletion
import django.utils.timezone
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
def updated_created_by_ids(apps, schema_editor):
"""Get or create a system user with is_superuser=True to be the default owner for new DB rows"""
@ -45,7 +46,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='archiveresult',
name='created_by',
field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(null=True, default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='archiveresult',
@ -61,7 +62,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='snapshot',
name='created_by',
field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(null=True, default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='snapshot',
@ -77,7 +78,7 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='tag',
name='created_by',
field=models.ForeignKey(null=True, default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(null=True, default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='tag',
@ -101,16 +102,16 @@ class Migration(migrations.Migration):
migrations.AddField(
model_name='snapshot',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='archiveresult',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
migrations.AddField(
model_name='tag',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
),
]

2
archivebox/core/migrations/0027_update_snapshot_ids.py
View file

@ -3,7 +3,7 @@
from django.db import migrations
from datetime import datetime
from abid_utils.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
from archivebox.base_models.abid import ABID, abid_from_values, DEFAULT_ABID_URI_SALT
def calculate_abid(self):

3
archivebox/core/migrations/0034_alter_archiveresult_old_id_alter_archiveresult_uuid.py
View file

@ -1,11 +1,10 @@
# Generated by Django 5.0.6 on 2024-08-18 05:37
import core.models
import uuid
import random
from django.db import migrations, models
from abid_utils.abid import ABID
from archivebox.base_models.abid import ABID
def rand_int_id():

4
archivebox/core/migrations/0059_tag_id.py
View file

@ -2,8 +2,8 @@
from datetime import datetime
from django.db import migrations, models
from abid_utils.abid import abid_from_values
from abid_utils.models import ABID
from archivebox.base_models.abid import abid_from_values
from archivebox.base_models.models import ABID
def calculate_abid(self):
"""

11
archivebox/core/migrations/0069_alter_archiveresult_created_alter_snapshot_added_and_more.py
View file

@ -1,9 +1,10 @@
# Generated by Django 5.1 on 2024-08-28 09:40
import abid_utils.models
import django.utils.timezone
from django.db import migrations
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -15,21 +16,21 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='archiveresult',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='snapshot',
name='added',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='snapshot',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
),
migrations.AlterField(
model_name='tag',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=django.utils.timezone.now),
),
]

13
archivebox/core/migrations/0070_alter_archiveresult_created_by_alter_snapshot_added_and_more.py
View file

@ -1,10 +1,11 @@
# Generated by Django 5.1 on 2024-09-04 09:00
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -17,22 +18,22 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='archiveresult',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='snapshot',
name='added',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='snapshot',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='snapshot',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='snapshot',
@ -47,6 +48,6 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='tag',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
]

5
archivebox/core/migrations/0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more.py
View file

@ -1,10 +1,11 @@
# Generated by Django 5.1 on 2024-09-04 23:23
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import archivebox.base_models.models
class Migration(migrations.Migration):
@ -30,7 +31,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='archiveresult',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
field=archivebox.base_models.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='archiveresult',

24
archivebox/core/models.py
View file

@ -24,31 +24,17 @@ from django.conf import settings
import abx
from archivebox.config import CONSTANTS
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithOutputDir
from workers.models import ModelWithStateMachine
from workers.tasks import bg_archive_snapshot
from crawls.models import Crawl
# from machine.models import Machine, NetworkInterface
from archivebox.misc.system import get_dir_size
from archivebox.misc.util import parse_date, base_url
from archivebox.index.schema import Link
from archivebox.index.html import snapshot_icons
from archivebox.extractors import ARCHIVE_METHODS_INDEXING_PRECEDENCE
from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithOutputDir
# class BaseModel(models.Model):
# # TODO: migrate all models to a shared base class with all our standard fields and helpers:
# # ulid/created_at/modified_at/created_by/is_deleted/as_json/from_json/etc.
# #
# # id = models.AutoField(primary_key=True, serialize=False, verbose_name='ID')
# # ulid = models.CharField(max_length=26, null=True, blank=True, db_index=True, unique=True)
# class Meta(TypedModelMeta):
# abstract = True
from workers.models import ModelWithStateMachine
from workers.tasks import bg_archive_snapshot
from crawls.models import Crawl
# from machine.models import Machine, NetworkInterface

1
archivebox/core/settings.py
View file

@ -61,7 +61,6 @@ INSTALLED_APPS = [
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
# Our ArchiveBox-provided apps
# 'abid_utils', # handles ABID ID creation, handling, and models
'config', # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
'machine', # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
'workers', # handles starting and managing background workers and processes (orchestrators and actors)

2
archivebox/crawls/admin.py
View file

@ -5,7 +5,7 @@ from django.contrib import admin
from archivebox import DATA_DIR
from abid_utils.admin import ABIDModelAdmin
from archivebox.base_models.admin import ABIDModelAdmin
from core.models import Snapshot
from crawls.models import Seed, Crawl, CrawlSchedule

6
archivebox/crawls/models.py
View file

@ -11,10 +11,10 @@ from django.conf import settings
from django.urls import reverse_lazy
from django.utils import timezone
from workers.models import ModelWithStateMachine
from archivebox.config import CONSTANTS
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats, get_or_create_system_user_pk
from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats, get_or_create_system_user_pk
from workers.models import ModelWithStateMachine
if TYPE_CHECKING:
from core.models import Snapshot, ArchiveResult

2
archivebox/index/sql.py
View file

@ -35,7 +35,7 @@ def remove_from_sql_main_index(snapshots: QuerySet, atomic: bool=False, out_dir:
@enforce_types
def write_link_to_sql_index(link: Link, created_by_id: int | None=None):
from core.models import Snapshot, ArchiveResult
from abid_utils.models import get_or_create_system_user_pk
from archivebox.base_models.models import get_or_create_system_user_pk
info = {k: v for k, v in link._asdict().items() if k in Snapshot.keys}

2
archivebox/machine/admin.py
View file

@ -5,7 +5,7 @@ import abx
from django.contrib import admin
from django.utils.html import format_html
from abid_utils.admin import ABIDModelAdmin
from archivebox.base_models.admin import ABIDModelAdmin
from machine.models import Machine, NetworkInterface, InstalledBinary

6
archivebox/machine/migrations/0001_initial.py
View file

@ -1,6 +1,6 @@
# Generated by Django 5.1.1 on 2024-10-02 04:34
import archivebox.abid_utils.models
import archivebox.base_models.models
import charidfield.fields
import django.db.models.deletion
from django.db import migrations, models
@ -42,7 +42,7 @@ class Migration(migrations.Migration):
),
(
"created_at",
archivebox.abid_utils.models.AutoDateTimeField(
archivebox.base_models.models.AutoDateTimeField(
db_index=True, default=None
),
),
@ -99,7 +99,7 @@ class Migration(migrations.Migration):
),
(
"created_at",
archivebox.abid_utils.models.AutoDateTimeField(
archivebox.base_models.models.AutoDateTimeField(
db_index=True, default=None
),
),

4
archivebox/machine/migrations/0002_alter_machine_stats_installedbinary.py
View file

@ -1,6 +1,6 @@
# Generated by Django 5.1.1 on 2024-10-03 07:25
import archivebox.abid_utils.models
import archivebox.base_models.models
import charidfield.fields
import django.db.models.deletion
from django.db import migrations, models
@ -47,7 +47,7 @@ class Migration(migrations.Migration):
),
(
"created_at",
archivebox.abid_utils.models.AutoDateTimeField(
archivebox.base_models.models.AutoDateTimeField(
db_index=True, default=None
),
),

2
archivebox/machine/models.py
View file

@ -12,7 +12,7 @@ import abx
import archivebox
from abx_pkg import Binary, BinProvider
from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats

6
archivebox/personas/models.py
View file

@ -1,8 +1,8 @@
from django.db import models
# from django.db import models
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
# from archivebox.base_models.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
from django.conf import settings
# from django.conf import settings
# class Persona(ABIDModel, ModelWithHealthStats):