change plugins to have both a .register that runs at import and .ready that runs later

archivebox/builtin_plugins/npm/apps.py

@@ -3,6 +3,7 @@ __package__ = 'archivebox.builtin_plugins.npm'
 from typing import List, Optional
 from pydantic import InstanceOf, Field
 
+from django.conf import settings
 
 from pydantic_pkgr import BinProvider, NpmProvider, BinName, PATHStr
 from plugantic.base_plugin import BasePlugin
@@ -65,4 +66,5 @@ class NpmPlugin(BasePlugin):
 
 
 PLUGIN = NpmPlugin()
+PLUGIN.register(settings)
 DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/pip/apps.py

@@ -9,6 +9,7 @@ import django
 
 from django.db.backends.sqlite3.base import Database as sqlite3     # type: ignore[import-type]
 from django.core.checks import Error, Tags
+from django.conf import settings
 
 from pydantic_pkgr import BinProvider, PipProvider, BinName, PATHStr, BinProviderName, ProviderLookupDict, SemVer
 from plugantic.base_plugin import BasePlugin
@@ -139,4 +140,5 @@ class PipPlugin(BasePlugin):
     ]
 
 PLUGIN = PipPlugin()
+PLUGIN.register(settings)
 DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/singlefile/apps.py

@@ -1,6 +1,8 @@
 from pathlib import Path
 from typing import List, Dict, Optional
 
+from django.conf import settings
+
 # Depends on other PyPI/vendor packages:
 from pydantic import InstanceOf, Field
 from pydantic_pkgr import BinProvider, BinProviderName, ProviderLookupDict, BinName
@@ -101,9 +103,11 @@ class SinglefilePlugin(BasePlugin):
         SINGLEFILE_CONFIG,
         SINGLEFILE_BINARY,
         SINGLEFILE_EXTRACTOR,
+        SINGLEFILE_QUEUE,
     ]
 
 
 
 PLUGIN = SinglefilePlugin()
+PLUGIN.register(settings)
 DJANGO_APP = PLUGIN.AppConfig

archivebox/builtin_plugins/ytdlp/apps.py

@@ -2,6 +2,7 @@ from typing import List, Dict
 from subprocess import run, PIPE
 from pydantic import InstanceOf, Field
 
+from django.conf import settings
 
 from pydantic_pkgr import BinProvider, BinName, BinProviderName, ProviderLookupDict
 from plugantic.base_plugin import BasePlugin
@@ -74,4 +75,5 @@ class YtdlpPlugin(BasePlugin):
 
 
 PLUGIN = YtdlpPlugin()
+PLUGIN.register(settings)
 DJANGO_APP = PLUGIN.AppConfig

archivebox/plugantic/base_hook.py

@@ -1,14 +1,15 @@
 __package__ = 'archivebox.plugantic'
 
-import json
+import inspect
+from huey.api import TaskWrapper
+
+from pathlib import Path
 from typing import List, Literal
 from pydantic import BaseModel, ConfigDict, Field, computed_field
 
 
-HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
+HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
-hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
+hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
-
-
 
 class BaseHook(BaseModel):
     """
@@ -56,24 +57,37 @@ class BaseHook(BaseModel):
         validate_defaults=True,
         validate_assignment=False,
         revalidate_instances="subclass-instances",
+        ignored_types=(TaskWrapper, ),
     )
     
     # verbose_name: str = Field()
+    
+    is_registered: bool = False
+    is_ready: bool = False
 
 
     @computed_field
     @property
     def id(self) -> str:
         return self.__class__.__name__
-    
+
     @computed_field
     @property
     def hook_module(self) -> str:
+        """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
         return f'{self.__module__}.{self.__class__.__name__}'
-    
+
+    @property
+    def plugin_module(self) -> str:
+        """e.g. builtin_plugins.singlefile"""
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
+        
+    @computed_field
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+
     hook_type: HookType = Field()
-    
-    
 
     def register(self, settings, parent_plugin=None):
         """Load a record of an installed hook into global Django settings.HOOKS at runtime."""
@@ -83,5 +97,20 @@ class BaseHook(BaseModel):
 
         # record installed hook in settings.HOOKS
         settings.HOOKS[self.id] = self
+        
+        if settings.HOOKS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
+
+        settings.HOOKS[self.id].is_registered = True
 
         # print("REGISTERED HOOK:", self.hook_module)
+
+    def ready(self, settings):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS."
+
+        if settings.HOOKS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!")
+
+        settings.HOOKS[self.id].is_ready = True

archivebox/plugantic/base_plugin.py

@@ -34,6 +34,9 @@ class BasePlugin(BaseModel):
     # All the hooks the plugin will install:
     hooks: List[InstanceOf[BaseHook]] = Field(default=[])
     
+    is_registered: bool = False
+    is_ready: bool = False
+    
     @computed_field
     @property
     def id(self) -> str:
@@ -81,7 +84,7 @@ class BasePlugin(BaseModel):
 
             def ready(self):
                 from django.conf import settings
-                plugin_self.register(settings)
+                plugin_self.ready(settings)
 
         return PluginAppConfig
 
@@ -97,9 +100,8 @@ class BasePlugin(BaseModel):
             hooks[hook.hook_type][hook.id] = hook
         return hooks
 
-
     def register(self, settings=None):
-        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
+        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called)."""
 
         if settings is None:
             from django.conf import settings as django_settings
@@ -112,11 +114,34 @@ class BasePlugin(BaseModel):
         ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
         settings.PLUGINS[self.id] = self
 
+        if settings.PLUGINS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!")
+
         for hook in self.hooks:
             hook.register(settings, parent_plugin=self)
 
+        settings.PLUGINS[self.id].is_registered = True
         # print('√ REGISTERED PLUGIN:', self.plugin_module)
 
+    def ready(self, settings=None):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        assert (
+            self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered
+        ), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS."
+
+        if settings.PLUGINS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!")
+
+        for hook in self.hooks:
+            hook.ready(settings)
+        
+        settings.PLUGINS[self.id].is_ready = True
+
     # @validate_call
     # def install_binaries(self) -> Self:
     #     new_binaries = []