more config improvements, move away from settings GLOBALS to getters

archivebox/abx/archivebox/base_binary.py

@@ -14,11 +14,8 @@ from pydantic_pkgr import (
     EnvProvider,
 )
 
-from archivebox.config import CONSTANTS
 from archivebox.config.permissions import ARCHIVEBOX_USER
 
-import abx
-
 
 class BaseBinProvider(BinProvider):
     
@@ -29,10 +26,6 @@ class BaseBinProvider(BinProvider):
         # e.g. /admin/environment/binproviders/NpmBinProvider/   TODO
         return "/admin/environment/binaries/"
 
-    @abx.hookimpl
-    def get_BINPROVIDERS(self):
-        return [self]
-
 class BaseBinary(Binary):
 
     @staticmethod
@@ -96,10 +89,6 @@ class BaseBinary(Binary):
         # e.g. /admin/environment/config/LdapConfig/
         return f"/admin/environment/binaries/{self.name}/"
 
-    @abx.hookimpl
-    def get_BINARIES(self):
-        return [self]
-
 
 class AptBinProvider(AptProvider, BaseBinProvider):
     name: BinProviderName = "apt"

archivebox/config/__init__.py

@@ -36,3 +36,7 @@ def get_CONFIG():
         'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
     }
 
+@abx.hookimpl
+def ready():
+    for config in get_CONFIG().values():
+        config.validate()

archivebox/config/common.py

@@ -7,13 +7,13 @@ from typing import Dict, Optional, List
 from pathlib import Path
 
 from rich import print
-from pydantic import Field, field_validator, computed_field
+from pydantic import Field, field_validator
 from django.utils.crypto import get_random_string
 
 from abx.archivebox.base_configset import BaseConfigSet
 
 from .constants import CONSTANTS
-from .version import get_COMMIT_HASH, get_BUILD_TIME
+from .version import get_COMMIT_HASH, get_BUILD_TIME, VERSION
 from .permissions import IN_DOCKER
 
 ###################### Config ##########################
@@ -31,25 +31,22 @@ class ShellConfig(BaseConfigSet):
 
     ANSI: Dict[str, str]                = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
 
-    VERSIONS_AVAILABLE: bool = False             # .check_for_update.get_versions_available_on_github(c)},
-    CAN_UPGRADE: bool = False                    # .check_for_update.can_upgrade(c)},
-
-    @computed_field
     @property
     def TERM_WIDTH(self) -> int:
         if not self.IS_TTY:
             return 200
         return shutil.get_terminal_size((140, 10)).columns
     
-    @computed_field
     @property
     def COMMIT_HASH(self) -> Optional[str]:
         return get_COMMIT_HASH()
     
-    @computed_field
     @property
     def BUILD_TIME(self) -> str:
         return get_BUILD_TIME()
+ 
+    # def VERSIONS_AVAILABLE() -> bool             # .check_for_update.get_versions_available_on_github(c)},
+    # def CAN_UPGRADE() -> bool                    # .check_for_update.can_upgrade(c)},
 
 SHELL_CONFIG = ShellConfig()
 
@@ -79,7 +76,6 @@ STORAGE_CONFIG = StorageConfig()
 class GeneralConfig(BaseConfigSet):
     TAG_SEPARATOR_PATTERN: str          = Field(default=r'[,]')
 
-
 GENERAL_CONFIG = GeneralConfig()
 
 
@@ -90,6 +86,7 @@ class ServerConfig(BaseConfigSet):
     CSRF_TRUSTED_ORIGINS: str           = Field(default=lambda c: 'http://localhost:8000,http://127.0.0.1:8000,http://0.0.0.0:8000,http://{}'.format(c.BIND_ADDR))
     
     SNAPSHOTS_PER_PAGE: int             = Field(default=40)
+    PREVIEW_ORIGINALS: bool             = Field(default=True)
     FOOTER_INFO: str                    = Field(default='Content is hosted for personal archiving purposes only.  Contact server owner for any takedown requests.')
     # CUSTOM_TEMPLATES_DIR: Path          = Field(default=None)  # this is now a constant
 
@@ -99,10 +96,10 @@ class ServerConfig(BaseConfigSet):
     
     ADMIN_USERNAME: str                 = Field(default=None)
     ADMIN_PASSWORD: str                 = Field(default=None)
+    
     REVERSE_PROXY_USER_HEADER: str      = Field(default='Remote-User')
     REVERSE_PROXY_WHITELIST: str        = Field(default='')
     LOGOUT_REDIRECT_URL: str            = Field(default='/')
-    PREVIEW_ORIGINALS: bool             = Field(default=True)
     
 SERVER_CONFIG = ServerConfig()
 
@@ -116,7 +113,7 @@ class ArchivingConfig(BaseConfigSet):
     MEDIA_MAX_SIZE: str                   = Field(default='750m')
     RESOLUTION: str                       = Field(default='1440,2000')
     CHECK_SSL_VALIDITY: bool              = Field(default=True)
-    USER_AGENT: str                       = Field(default='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
+    USER_AGENT: str                       = Field(default=f'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 ArchiveBox/{VERSION} (+https://github.com/ArchiveBox/ArchiveBox/)')
     COOKIES_FILE: Path | None             = Field(default=None)
     
     URL_DENYLIST: str                     = Field(default=r'\.(css|js|otf|ttf|woff|woff2|gstatic\.com|googleapis\.com/css)(\?.*)?$', alias='URL_BLACKLIST')
@@ -134,17 +131,15 @@ class ArchivingConfig(BaseConfigSet):
     # CHROME_HEADLESS: bool               = Field(default=True)
     # CHROME_SANDBOX: bool                = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
 
-    @field_validator('TIMEOUT', mode='after')
+    def validate(self):
-    def validate_timeout(cls, v):
+        if int(self.TIMEOUT) < 5:
-        if int(v) < 5:
+            print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]', file=sys.stderr)
-            print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={v} seconds)[/red]', file=sys.stderr)
             print('    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.', file=sys.stderr)
             print('    (Setting it to somewhere between 30 and 3000 seconds is recommended)', file=sys.stderr)
             print(file=sys.stderr)
             print('    If you want to make ArchiveBox run faster, disable specific archive methods instead:', file=sys.stderr)
             print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles', file=sys.stderr)
             print(file=sys.stderr)
-        return v
     
     @field_validator('CHECK_SSL_VALIDITY', mode='after')
     def validate_check_ssl_validity(cls, v):
@@ -167,15 +162,17 @@ class ArchivingConfig(BaseConfigSet):
     @property
     def SAVE_ALLOWLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
         return {
-            re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
+            # regexp: methods list
-            for k, v in self.SAVE_ALLOWLIST.items()
+            re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
+            for key, val in self.SAVE_ALLOWLIST.items()
         } if self.SAVE_ALLOWLIST else {}
     
     @property
     def SAVE_DENYLIST_PTNS(self) -> Dict[re.Pattern, List[str]]:
         return {
-            re.compile(k, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): v
+            # regexp: methods list
-            for k, v in self.SAVE_DENYLIST.items()
+            re.compile(key, CONSTANTS.ALLOWDENYLIST_REGEX_FLAGS): val
+            for key, val in self.SAVE_DENYLIST.items()
         } if self.SAVE_DENYLIST else {}
 
 ARCHIVING_CONFIG = ArchivingConfig()

archivebox/core/admin_snapshots.py

@@ -13,7 +13,7 @@ from django.template import Template, RequestContext
 from django.contrib.admin.helpers import ActionForm
 from django.contrib.admin.widgets import FilteredSelectMultiple
 
-from archivebox.config import DATA_DIR, VERSION
+from archivebox.config import DATA_DIR
 from archivebox.config.common import SERVER_CONFIG
 from archivebox.misc.util import htmldecode, urldecode
 from archivebox.misc.paginators import AccelleratedPaginator
@@ -32,8 +32,8 @@ from core.admin_tags import TagInline
 from core.admin_archiveresults import ArchiveResultInline, result_url
 
 
-GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
+# GLOBAL_CONTEXT = {'VERSION': VERSION, 'VERSIONS_AVAILABLE': [], 'CAN_UPGRADE': False}
-
+GLOBAL_CONTEXT = {}
 
 
 class SnapshotActionForm(ActionForm):

archivebox/index/json.py

@@ -33,7 +33,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
             'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
             'source': 'https://github.com/ArchiveBox/ArchiveBox',
             'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
-            'dependencies': settings.BINARIES.to_dict(),
+            'dependencies': settings.BINARIES,
         },
     }
     

archivebox/main.py

@@ -1075,11 +1075,11 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     
     package_manager_names = ', '.join(
         f'[yellow]{binprovider.name}[/yellow]'
-        for binprovider in reversed(list(settings.BINPROVIDERS.values()))
+        for binprovider in list(settings.BINPROVIDERS.values())
         if not binproviders or (binproviders and binprovider.name in binproviders)
     )
     print(f'[+] Setting up package managers {package_manager_names}...')
-    for binprovider in reversed(list(settings.BINPROVIDERS.values())):
+    for binprovider in list(settings.BINPROVIDERS.values()):
         if binproviders and binprovider.name not in binproviders:
             continue
         try:
@@ -1092,7 +1092,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
     
     print()
     
-    for binary in reversed(list(settings.BINARIES.values())):
+    for binary in list(settings.BINARIES.values()):
         if binary.name in ('archivebox', 'django', 'sqlite', 'python'):
             # obviously must already be installed if we are running
             continue

archivebox/plugins_extractor/chrome/config.py

@@ -189,6 +189,10 @@ class ChromeConfig(BaseConfigSet):
         #   cmd_args += ('--timeout={}'.format(options.CHROME_TIMEOUT * 1000),)
     
         if options.CHROME_USER_DATA_DIR:
+            # remove SingletonLock file
+            lockfile = options.CHROME_USER_DATA_DIR / options.CHROME_PROFILE_NAME / 'SingletonLock'
+            lockfile.unlink(missing_ok=True)
+            
             cmd_args.append('--user-data-dir={}'.format(options.CHROME_USER_DATA_DIR))
             cmd_args.append('--profile-directory={}'.format(options.CHROME_PROFILE_NAME or 'Default'))
         

archivebox/templates/admin/base.html

@@ -148,6 +148,7 @@
             {% block footer %}<div id="footer"></div>{% endblock %}
         </div>
 
+        {% comment %} 
         {% if user.is_authenticated and user.is_superuser and CAN_UPGRADE %}
             <script>
                 if (!localStorage.getItem("bannerDismissed")) {
@@ -184,6 +185,7 @@
                 }
             </script>
         {% endif %}
+        {% endcomment %}
         
         <script>
             $ = django.jQuery;