Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-21 19:53:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

merge queues and actors apps into new workers app

Browse source
This commit is contained in:
Nick Sweeting 2024-11-18 18:52:48 -08:00
parent e50f8cb3b6
commit e469c5a344
No known key found for this signature in database
37 changed files with 89 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/__init__.py
View file

@ -82,11 +82,10 @@ ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
# Load all built-in ArchiveBox plugins
ARCHIVEBOX_BUILTIN_PLUGINS = {
'config': PACKAGE_DIR / 'config',
'workers': PACKAGE_DIR / 'workers',
'core': PACKAGE_DIR / 'core',
'crawls': PACKAGE_DIR / 'crawls',
'queues': PACKAGE_DIR / 'queues',
'seeds': PACKAGE_DIR / 'seeds',
'actors': PACKAGE_DIR / 'actors',
# 'search': PACKAGE_DIR / 'search',
# 'core': PACKAGE_DIR / 'core',
}

2
archivebox/actors/__init__.py
View file

@ -1,2 +0,0 @@
__package__ = 'archivebox.actors'
__order__ = 100

3
archivebox/actors/admin.py
View file

@ -1,3 +0,0 @@
from django.contrib import admin
# Register your models here.

6
archivebox/actors/apps.py
View file

@ -1,6 +0,0 @@
from django.apps import AppConfig
class ActorsConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "actors"

202
archivebox/actors/templates/jobs_dashboard.html
View file

@ -1,202 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Job Dashboard</title>
<style>
body {
font-family: Arial, sans-serif;
line-height: 1.6;
color: #333;
width: 100%;
margin: 0 auto;
padding: 20px;
}
@keyframes pulse {
0% { opacity: 1; }
48% { opacity: 0.2; }
52% { opacity: 1; }
100% { opacity: 1; }
}
h1 {
text-align: center;
}
h1 a {
animation: pulse 1s;
}
.dashboard {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 20px;
}
.card {
border: 1px solid #ddd;
border-radius: 8px;
padding: 15px;
background-color: #f9f9f9;
}
.card h2 {
margin-top: 0;
border-bottom: 2px solid #ddd;
padding-bottom: 10px;
font-family: monospace;
}
.scroll-area {
/*height: 800px;
overflow-y: scroll; */
height: auto;
border: 1px solid #ddd;
padding: 10px;
background-color: #fff;
}
.job-item {
border: 1px solid #eee;
border-radius: 4px;
padding: 10px;
margin-bottom: 10px;
}
.job-item:last-child {
margin-bottom: 0;
}
.badge {
display: inline-block;
padding: 3px 7px;
border-radius: 3px;
font-size: 12px;
font-weight: bold;
}
.badge-started {
background-color: #4CAF50;
color: white;
}
.badge-queued {
background-color: #2196F3;
color: white;
}
.badge-failed {
background-color: #f44336;
color: white;
}
.badge-succeeded {
background-color: #666;
color: white;
}
.badge-sealed {
background-color: #666;
color: white;
}
.date {
font-size: 16px;
color: #666;
float: right;
}
</style>
</head>
<body>
<h1>Job Dashboard <small><a href="?refresh=true" id="current-time">♻️ {{now}}}</a></small></h1>
<div id="dashboard" class="dashboard"></div>
<script>
function formatDate(dateString) {
const now = Date.now()
const date = new Date(dateString)
// return new Date(dateString).toLocaleString();
// return date.toISOString().split('T').at(-1).replace('Z', '');
const seconds_diff = Math.round((date - now) / 1000, 0)
if (seconds_diff < 0) {
return `${seconds_diff}s ago`;
} else {
return `${seconds_diff}s in the future`;
}
}
function createJobElement(job) {
const jobElement = document.createElement('div');
jobElement.className = 'job-item';
jobElement.innerHTML = `
<p><a href="/api/v1/core/any/${job.abid}?api_key={{api_token|default:'NONE PROVIDED BY VIEW'}}"><code>${job.abid}</code></a></p>
<p>
<span class="badge badge-${job.status}">${job.status}</span>
<span class="date">♻️ ${formatDate(job.retry_at)}</span>
</p>
<p style="font-size: 12px; color: #666;">${job.description}</p>
`;
return jobElement;
}
function updateDashboard(data) {
const currentTime = document.getElementById('current-time');
window.now = new Date();
currentTime.innerHTML = `♻️ ${window.now.toISOString().split('T').at(-1).replace('Z', '')}`;
const dashboard = document.getElementById('dashboard');
dashboard.innerHTML = '';
data.forEach(actor => {
const card = document.createElement('div');
card.className = 'card';
card.innerHTML = `
<h2>${actor.model}</h2>
<hr/>
Future
<div class="scroll-area" style="background-color: white;" id="future-${actor.model}"></div>
<hr/>
Pending
<div class="scroll-area" style="background-color: lightblue;" id="pending-${actor.model}"></div>
<hr/>
Stalled
<div class="scroll-area" style="background-color: lightcoral;" id="stalled-${actor.model}"></div>
<hr/>
Active
<div class="scroll-area" style="background-color: lightgreen;" id="active-${actor.model}"></div>
<hr/>
Past
<div class="scroll-area" style="background-color: lightgrey;" id="past-${actor.model}"></div>
`;
dashboard.appendChild(card);
const futureContainer = document.getElementById(`future-${actor.model}`);
actor.future.forEach(job => {
futureContainer.appendChild(createJobElement(job));
});
const pendingContainer = document.getElementById(`pending-${actor.model}`);
actor.pending.forEach(job => {
pendingContainer.appendChild(createJobElement(job));
});
const stalledContainer = document.getElementById(`stalled-${actor.model}`);
actor.stalled.forEach(job => {
stalledContainer.appendChild(createJobElement(job));
});
const activeContainer = document.getElementById(`active-${actor.model}`);
actor.active.forEach(job => {
activeContainer.appendChild(createJobElement(job));
});
const pastContainer = document.getElementById(`past-${actor.model}`);
actor.past.forEach(job => {
pastContainer.appendChild(createJobElement(job));
});
});
}
function fetchData() {
fetch('/api/v1/jobs/actors', {
headers: {
'Authorization': `Bearer {{api_token|default:'NONE PROVIDED BY VIEW'}}`
}
})
.then(response => response.json())
.then(data => updateDashboard(data))
.catch(error => console.error('Error fetching data:', error));
}
fetchData();
setInterval(fetchData, 750);
</script>
</body>
</html>

2
archivebox/api/v1_api.py
View file

@ -41,7 +41,7 @@ def register_urls(api: NinjaAPI) -> NinjaAPI:
api.add_router('/core/', 'api.v1_core.router')
api.add_router('/crawls/', 'api.v1_crawls.router')
api.add_router('/cli/', 'api.v1_cli.router')
api.add_router('/jobs/', 'api.v1_actors.router')
api.add_router('/workers/', 'api.v1_workers.router')
return api

8
archivebox/api/v1_actors.py → archivebox/api/v1_workers.py
View file

@ -31,7 +31,7 @@ class TaskSchema(Schema):
class ActorSchema(Schema):
# TYPE: str = 'actors.actor.ActorType'
# TYPE: str = 'workers.actor.ActorType'
# name: str
#pid: int | None
@ -97,7 +97,7 @@ class ActorSchema(Schema):
class OrchestratorSchema(Schema):
# TYPE: str = 'actors.orchestrator.Orchestrator'
# TYPE: str = 'workers.orchestrator.Orchestrator'
#pid: int | None
exit_on_idle: bool
@ -114,7 +114,7 @@ class OrchestratorSchema(Schema):
def get_orchestrators(request):
"""List all the task orchestrators (aka Orchestrators) that are currently running"""
from actors.orchestrator import Orchestrator
from workers.orchestrator import Orchestrator
orchestrator = Orchestrator()
return [orchestrator]
@ -124,6 +124,6 @@ def get_orchestrators(request):
def get_actors(request):
"""List all the task consumer workers (aka Actors) that are currently running"""
from actors.orchestrator import Orchestrator
from workers.orchestrator import Orchestrator
orchestrator = Orchestrator()
return orchestrator.actor_types.values()

2
archivebox/cli/archivebox_add.py
View file

@ -53,7 +53,7 @@ def add(urls: str | list[str],
from seeds.models import Seed
from crawls.models import Crawl
from actors.orchestrator import Orchestrator
from workers.orchestrator import Orchestrator
from abid_utils.models import get_or_create_system_user_pk

2
archivebox/cli/archivebox_update.py
View file

@ -28,7 +28,7 @@ def update():
from archivebox.config.django import setup_django
setup_django()
from actors.orchestrator import Orchestrator
from workers.orchestrator import Orchestrator
orchestrator = Orchestrator(exit_on_idle=False)
orchestrator.start()

13
archivebox/config/views.py
View file

@ -7,7 +7,6 @@ from typing import Any, List, Dict, cast
from benedict import benedict
from django.http import HttpRequest
from django.conf import settings
from django.utils import timezone
from django.utils.html import format_html, mark_safe
@ -304,7 +303,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
"Exit Status": [],
}
from queues.supervisor_util import get_existing_supervisord_process
from workers.supervisor_util import get_existing_supervisord_process
supervisor = get_existing_supervisord_process()
if supervisor is None:
@ -374,8 +373,10 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
from queues.supervisor_util import get_existing_supervisord_process, get_worker
from queues.settings import SUPERVISORD_CONFIG_FILE
from workers.supervisor_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
SOCK_FILE = get_sock_file()
CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
supervisor = get_existing_supervisord_process()
if supervisor is None:
@ -388,7 +389,7 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
if key == 'supervisord':
relevant_config = SUPERVISORD_CONFIG_FILE.read_text()
relevant_config = CONFIG_FILE.read_text()
relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
@ -475,8 +476,6 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
from django.conf import settings
log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]

2
archivebox/core/admin_snapshots.py
View file

@ -25,7 +25,7 @@ from archivebox.extractors import archive_links
from archivebox.main import remove
from archivebox.abid_utils.admin import ABIDModelAdmin
from archivebox.queues.tasks import bg_archive_links, bg_add
from archivebox.workers.tasks import bg_archive_links, bg_add
from core.models import Tag
from core.admin_tags import TagInline

4
archivebox/core/models.py
View file

@ -26,8 +26,8 @@ import abx
from archivebox.config import CONSTANTS
from abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithOutputDir
from actors.models import ModelWithStateMachine
from queues.tasks import bg_archive_snapshot
from workers.models import ModelWithStateMachine
from workers.tasks import bg_archive_snapshot
from crawls.models import Crawl
# from machine.models import Machine, NetworkInterface

5
archivebox/core/settings.py
View file

@ -61,11 +61,10 @@ INSTALLED_APPS = [
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
# Our ArchiveBox-provided apps
# 'abid_utils', # handles ABID ID creation, handling, and models
# 'abid_utils', # handles ABID ID creation, handling, and models
'config', # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
'machine', # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
'actors', # handles starting and managing background workers and processes (orchestrators and actors)
'queues', # handles starting and managing background workers and processes (supervisord)
'workers', # handles starting and managing background workers and processes (orchestrators and actors)
'seeds', # handles Seed model and URL source management
'crawls', # handles Crawl and CrawlSchedule models and management
'personas', # handles Persona and session management

2
archivebox/core/statemachines.py
View file

@ -8,7 +8,7 @@ from django.utils import timezone
from statemachine import State, StateMachine
from actors.actor import ActorType
from workers.actor import ActorType
from core.models import Snapshot, ArchiveResult

2
archivebox/core/urls.py
View file

@ -10,7 +10,7 @@ from archivebox.misc.serve_static import serve_static
from core.admin_site import archivebox_admin
from core.views import HomepageView, SnapshotView, PublicIndexView, AddView, HealthCheckView
from actors.views import JobsDashboardView
from workers.views import JobsDashboardView
# GLOBAL_CONTEXT doesn't work as-is, disabled for now: https://github.com/ArchiveBox/ArchiveBox/discussions/1306
# from archivebox.config import VERSION, VERSIONS_AVAILABLE, CAN_UPGRADE

2
archivebox/core/views.py
View file

@ -25,7 +25,7 @@ import archivebox
from core.models import Snapshot
from core.forms import AddLinkForm
from queues.tasks import bg_add
from workers.tasks import bg_add
from archivebox.config import CONSTANTS_CONFIG, DATA_DIR, VERSION
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG

2
archivebox/crawls/models.py
View file

@ -10,7 +10,7 @@ from django.conf import settings
from django.urls import reverse_lazy
from django.utils import timezone
from actors.models import ModelWithStateMachine
from workers.models import ModelWithStateMachine
if TYPE_CHECKING:
from core.models import Snapshot, ArchiveResult

2
archivebox/crawls/statemachines.py
View file

@ -6,7 +6,7 @@ from django.utils import timezone
from statemachine import State, StateMachine
from actors.actor import ActorType
from workers.actor import ActorType
from crawls.models import Crawl

8
archivebox/main.py
View file

@ -675,8 +675,8 @@ def add(urls: Union[str, List[str]],
"""Add a new URL or list of URLs to your archive"""
from core.models import Snapshot, Tag
# from queues.supervisor_util import start_cli_workers, tail_worker_logs
# from queues.tasks import bg_archive_link
# from workers.supervisor_util import start_cli_workers, tail_worker_logs
# from workers.tasks import bg_archive_link
assert depth in (0, 1), 'Depth must be 0 or 1 (depth >1 is not supported yet)'
@ -873,7 +873,7 @@ def update(resume: Optional[float]=None,
from core.models import ArchiveResult
from .search import index_links
# from .queues.supervisor_util import start_cli_workers
# from workers.supervisor_util import start_cli_workers
check_data_folder()
@ -1494,7 +1494,7 @@ def server(runserver_args: Optional[List[str]]=None,
runserver_args.append('--noreload') # '--insecure'
call_command("runserver", *runserver_args)
else:
from queues.supervisor_util import start_server_workers
from workers.supervisor_util import start_server_workers
print()
start_server_workers(host=host, port=port, daemonize=False)

40
archivebox/pkgs/abx-plugin-singlefile/abx_plugin_singlefile/actors.py
View file

@ -1,27 +1,27 @@
__package__ = 'abx_plugin_singlefile'
# __package__ = 'abx_plugin_singlefile'
from typing import ClassVar
from django.db.models import QuerySet
from django.utils.functional import classproperty
# from typing import ClassVar
# from django.db.models import QuerySet
# from django.utils.functional import classproperty
from actors.actor import ActorType
# from workers.actor import ActorType
from .models import SinglefileResult
# from .models import SinglefileResult
class SinglefileActor(ActorType[SinglefileResult]):
CLAIM_ORDER: ClassVar[str] = 'created_at DESC'
CLAIM_WHERE: ClassVar[str] = 'status = "queued" AND extractor = "favicon"'
CLAIM_SET: ClassVar[str] = 'status = "started"'
# class SinglefileActor(ActorType[SinglefileResult]):
# CLAIM_ORDER: ClassVar[str] = 'created_at DESC'
# CLAIM_WHERE: ClassVar[str] = 'status = "queued" AND extractor = "favicon"'
# CLAIM_SET: ClassVar[str] = 'status = "started"'
@classproperty
def QUERYSET(cls) -> QuerySet:
return SinglefileResult.objects.filter(status='queued')
# @classproperty
# def QUERYSET(cls) -> QuerySet:
# return SinglefileResult.objects.filter(status='queued')
def tick(self, obj: SinglefileResult):
print(f'[grey53]{self}.tick({obj.abid or obj.id}, status={obj.status}) remaining:[/grey53]', self.get_queue().count())
updated = SinglefileResult.objects.filter(id=obj.id, status='started').update(status='success') == 1
if not updated:
raise Exception(f'Failed to update {obj.abid or obj.id}, interrupted by another actor writing to the same object')
obj.refresh_from_db()
obj.save()
# def tick(self, obj: SinglefileResult):
# print(f'[grey53]{self}.tick({obj.abid or obj.id}, status={obj.status}) remaining:[/grey53]', self.get_queue().count())
# updated = SinglefileResult.objects.filter(id=obj.id, status='started').update(status='success') == 1
# if not updated:
# raise Exception(f'Failed to update {obj.abid or obj.id}, interrupted by another actor writing to the same object')
# obj.refresh_from_db()
# obj.save()

10
archivebox/pkgs/abx-plugin-wget/abx_plugin_wget/extractors.py
View file

@ -13,11 +13,11 @@ class WgetExtractor(BaseExtractor):
name: ExtractorName = 'wget'
binary: BinName = WGET_BINARY.name
def get_output_path(self, snapshot) -> Path | None:
wget_index_path = wget_output_path(snapshot.as_link())
if wget_index_path:
return Path(wget_index_path)
return None
def get_output_path(self, snapshot) -> str:
# wget_index_path = wget_output_path(snapshot.as_link())
# if wget_index_path:
# return Path(wget_index_path)
return 'wget'
WGET_EXTRACTOR = WgetExtractor()

20
archivebox/pkgs/abx-spec-config/abx_spec_config/__init__.py
View file

@ -65,16 +65,16 @@ class ConfigPluginSpec:
def get_SCOPE_CONFIG(extra=None, archiveresult=None, snapshot=None, crawl=None, user=None, request=None, collection=..., environment=..., machine=..., default=...) -> dict[ConfigKeyStr, Any]:
"""Get the config as it applies to you right now, based on the current context"""
return benedict({
**pm.hook.get_default_config(default=default),
**pm.hook.get_machine_config(machine=machine),
**pm.hook.get_environment_config(environment=environment),
**pm.hook.get_collection_config(collection=collection),
**pm.hook.get_user_config(user=user),
**pm.hook.get_crawl_config(crawl=crawl),
**pm.hook.get_snapshot_config(snapshot=snapshot),
**pm.hook.get_archiveresult_config(archiveresult=archiveresult),
**pm.hook.get_request_config(request=request),
**(extra or {}),
**pm.hook.get_default_config(default=default), # schema defaults defined in source code
**pm.hook.get_machine_config(machine=machine), # machine defaults set on the Machine model
**pm.hook.get_environment_config(environment=environment), # env config set for just this run on this machine
**pm.hook.get_collection_config(collection=collection), # collection defaults set in ArchiveBox.conf
**pm.hook.get_user_config(user=user), # user config set on User model
**pm.hook.get_request_config(request=request), # extra config derived from the current request
**pm.hook.get_crawl_config(crawl=crawl), # extra config set on the Crawl model
**pm.hook.get_snapshot_config(snapshot=snapshot), # extra config set on the Snapshot model
**pm.hook.get_archiveresult_config(archiveresult=archiveresult), # extra config set on the ArchiveResult model
**(extra or {}), # extra config passed in by the caller
})
@staticmethod

8
archivebox/queues/__init__.py
View file

@ -1,8 +0,0 @@
__package__ = 'archivebox.queues'
import abx
@abx.hookimpl
def register_admin(admin_site):
from queues.admin import register_admin
register_admin(admin_site)

0
archivebox/queues/migrations/__init__.py
View file

9
archivebox/workers/__init__.py Normal file
View file

@ -0,0 +1,9 @@
__package__ = 'archivebox.workers'
__order__ = 100
import abx
@abx.hookimpl
def register_admin(admin_site):
from workers.admin import register_admin
register_admin(admin_site)

2
archivebox/actors/actor.py → archivebox/workers/actor.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.actors'
__package__ = 'archivebox.workers'
import os
import time

2
archivebox/queues/admin.py → archivebox/workers/admin.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.queues'
__package__ = 'archivebox.workers'
import abx

4
archivebox/queues/apps.py → archivebox/workers/apps.py
View file

@ -1,7 +1,7 @@
from django.apps import AppConfig
class QueuesConfig(AppConfig):
class WorkersConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'queues'
name = 'workers'

2
archivebox/actors/management/commands/orchestrator.py → archivebox/workers/management/commands/orchestrator.py
View file

@ -2,7 +2,7 @@
from django.core.management.base import BaseCommand
from actors.orchestrator import ArchivingOrchestrator
from workers.orchestrator import ArchivingOrchestrator
class Command(BaseCommand):

0
archivebox/actors/migrations/__init__.py → archivebox/workers/migrations/__init__.py
View file

18
archivebox/actors/models.py → archivebox/workers/models.py
View file

@ -60,7 +60,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices',
hint=f'{cls.__name__}.{field.name}.choices = {getattr(field, "choices", None)!r}',
obj=cls,
id='actors.E011',
id='workers.E011',
))
if getattr(field, '_is_retry_at_field', False):
if cls.retry_at_field_name == field.name:
@ -73,14 +73,14 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.state_field_name must be defined and point to a StatusField()',
hint=f'{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField',
obj=cls,
id='actors.E012',
id='workers.E012',
))
if not found_retry_at_field:
errors.append(checks.Error(
f'{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()',
hint=f'{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField',
obj=cls,
id='actors.E013',
id='workers.E013',
))
if not found_id_field:
@ -88,7 +88,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__} must have an id field that is a primary key',
hint=f'{cls.__name__}.id = {cls.id!r}',
obj=cls,
id='actors.E014',
id='workers.E014',
))
if not isinstance(cls.state_machine_name, str):
@ -96,7 +96,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class',
hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
obj=cls,
id='actors.E015',
id='workers.E015',
))
try:
@ -106,7 +106,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass',
hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
obj=cls,
id='actors.E016',
id='workers.E016',
))
if cls.INITIAL_STATE not in cls.StatusChoices.values:
@ -114,7 +114,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices',
hint=f'{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}',
obj=cls,
id='actors.E017',
id='workers.E017',
))
if cls.ACTIVE_STATE not in cls.StatusChoices.values:
@ -122,7 +122,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices',
hint=f'{cls.__name__}.active_state = {cls.active_state!r}',
obj=cls,
id='actors.E018',
id='workers.E018',
))
@ -132,7 +132,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
f'{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices',
hint=f'{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}',
obj=cls,
id='actors.E019',
id='workers.E019',
))
break
return errors

2
archivebox/actors/orchestrator.py → archivebox/workers/orchestrator.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.actors'
__package__ = 'archivebox.workers'
import os
import time

0
archivebox/queues/semaphores.py → archivebox/workers/semaphores.py
View file

2
archivebox/queues/supervisor_util.py → archivebox/workers/supervisor_util.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.queues'
__package__ = 'archivebox.workers'
import sys
import time

2
archivebox/queues/tasks.py → archivebox/workers/tasks.py
View file

@ -1,4 +1,4 @@
__package__ = 'archivebox.queues'
__package__ = 'archivebox.workers'
from functools import wraps
# from django.utils import timezone

0
archivebox/actors/tests.py → archivebox/workers/tests.py
View file

0
archivebox/actors/views.py → archivebox/workers/views.py
View file