2019-04-11 07:40:37 +00:00
|
|
|
__package__ = 'archivebox.core'
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2019-04-17 07:49:18 +00:00
|
|
|
import os
|
2019-04-22 23:07:39 +00:00
|
|
|
import sys
|
2024-08-27 03:14:47 +00:00
|
|
|
import inspect
|
2020-10-31 07:08:03 +00:00
|
|
|
|
2020-09-30 19:54:32 +00:00
|
|
|
from pathlib import Path
|
2024-09-06 06:19:21 +00:00
|
|
|
|
2020-04-23 01:14:43 +00:00
|
|
|
from django.utils.crypto import get_random_string
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
import abx
|
2024-09-27 07:41:21 +00:00
|
|
|
import abx.archivebox
|
|
|
|
import abx.archivebox.use
|
|
|
|
import abx.django.use
|
|
|
|
|
2024-10-08 06:45:11 +00:00
|
|
|
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS
|
|
|
|
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG # noqa
|
2019-04-17 09:42:09 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
|
|
|
|
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
|
2019-04-22 23:07:39 +00:00
|
|
|
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2024-09-23 02:26:26 +00:00
|
|
|
|
2024-09-24 22:13:54 +00:00
|
|
|
|
2024-09-03 07:58:50 +00:00
|
|
|
################################################################################
|
|
|
|
### ArchiveBox Plugin Settings
|
|
|
|
################################################################################
|
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
PLUGIN_HOOKSPECS = [
|
2024-09-27 07:41:21 +00:00
|
|
|
'abx.django.hookspec',
|
|
|
|
'abx.pydantic_pkgr.hookspec',
|
|
|
|
'abx.archivebox.hookspec',
|
2024-09-26 09:43:12 +00:00
|
|
|
]
|
|
|
|
abx.register_hookspecs(PLUGIN_HOOKSPECS)
|
|
|
|
|
|
|
|
BUILTIN_PLUGIN_DIRS = {
|
2024-09-30 22:59:05 +00:00
|
|
|
'archivebox': PACKAGE_DIR,
|
|
|
|
'plugins_pkg': PACKAGE_DIR / 'plugins_pkg',
|
|
|
|
'plugins_auth': PACKAGE_DIR / 'plugins_auth',
|
|
|
|
'plugins_search': PACKAGE_DIR / 'plugins_search',
|
|
|
|
'plugins_extractor': PACKAGE_DIR / 'plugins_extractor',
|
2024-09-26 09:43:12 +00:00
|
|
|
}
|
|
|
|
USER_PLUGIN_DIRS = {
|
2024-10-15 04:50:47 +00:00
|
|
|
# 'user_plugins': DATA_DIR / 'user_plugins',
|
2024-09-03 07:58:50 +00:00
|
|
|
}
|
2024-09-06 08:48:18 +00:00
|
|
|
|
2024-10-02 21:17:28 +00:00
|
|
|
# Discover ArchiveBox plugins
|
2024-09-26 09:43:12 +00:00
|
|
|
BUILTIN_PLUGINS = abx.get_plugins_in_dirs(BUILTIN_PLUGIN_DIRS)
|
|
|
|
PIP_PLUGINS = abx.get_pip_installed_plugins(group='archivebox')
|
|
|
|
USER_PLUGINS = abx.get_plugins_in_dirs(USER_PLUGIN_DIRS)
|
|
|
|
ALL_PLUGINS = {**BUILTIN_PLUGINS, **PIP_PLUGINS, **USER_PLUGINS}
|
|
|
|
|
2024-10-02 21:17:28 +00:00
|
|
|
# Load ArchiveBox plugins
|
2024-09-26 09:43:12 +00:00
|
|
|
PLUGIN_MANAGER = abx.pm
|
2024-10-15 04:50:47 +00:00
|
|
|
abx.archivebox.load_archivebox_plugins(PLUGIN_MANAGER, ALL_PLUGINS)
|
|
|
|
PLUGINS = abx.archivebox.use.get_PLUGINS()
|
2024-09-27 07:41:21 +00:00
|
|
|
|
2024-10-02 21:17:28 +00:00
|
|
|
# Load ArchiveBox config from plugins
|
2024-09-27 07:41:21 +00:00
|
|
|
CONFIGS = abx.archivebox.use.get_CONFIGS()
|
2024-10-15 04:50:47 +00:00
|
|
|
CONFIG = FLAT_CONFIG = abx.archivebox.use.get_FLAT_CONFIG()
|
2024-09-27 07:41:21 +00:00
|
|
|
BINPROVIDERS = abx.archivebox.use.get_BINPROVIDERS()
|
|
|
|
BINARIES = abx.archivebox.use.get_BINARIES()
|
|
|
|
EXTRACTORS = abx.archivebox.use.get_EXTRACTORS()
|
|
|
|
SEARCHBACKENDS = abx.archivebox.use.get_SEARCHBACKENDS()
|
2024-10-15 04:50:47 +00:00
|
|
|
# REPLAYERS = abx.archivebox.use.get_REPLAYERS()
|
|
|
|
# ADMINDATAVIEWS = abx.archivebox.use.get_ADMINDATAVIEWS()
|
2024-09-27 07:41:21 +00:00
|
|
|
|
2024-09-03 07:58:50 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
################################################################################
|
|
|
|
### Django Core Settings
|
|
|
|
################################################################################
|
|
|
|
|
|
|
|
WSGI_APPLICATION = 'core.wsgi.application'
|
2024-09-06 04:43:42 +00:00
|
|
|
ASGI_APPLICATION = "core.asgi.application"
|
2020-10-31 07:08:03 +00:00
|
|
|
ROOT_URLCONF = 'core.urls'
|
|
|
|
|
|
|
|
LOGIN_URL = '/accounts/login/'
|
2022-03-31 19:40:14 +00:00
|
|
|
LOGOUT_REDIRECT_URL = os.environ.get('LOGOUT_REDIRECT_URL', '/')
|
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
PASSWORD_RESET_URL = '/accounts/password_reset/'
|
|
|
|
APPEND_SLASH = True
|
2020-04-23 01:14:43 +00:00
|
|
|
|
2024-10-01 06:21:34 +00:00
|
|
|
DEBUG = SHELL_CONFIG.DEBUG or ('--debug' in sys.argv)
|
2021-01-30 11:07:35 +00:00
|
|
|
|
2024-05-18 03:48:11 +00:00
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
INSTALLED_APPS = [
|
2024-09-06 04:45:43 +00:00
|
|
|
'daphne',
|
2024-09-13 10:25:46 +00:00
|
|
|
|
2024-09-03 07:58:50 +00:00
|
|
|
# Django default apps
|
2019-04-17 07:49:18 +00:00
|
|
|
'django.contrib.auth',
|
|
|
|
'django.contrib.contenttypes',
|
|
|
|
'django.contrib.sessions',
|
|
|
|
'django.contrib.messages',
|
|
|
|
'django.contrib.staticfiles',
|
2020-07-28 03:26:45 +00:00
|
|
|
'django.contrib.admin',
|
2024-05-18 03:48:11 +00:00
|
|
|
|
2024-09-03 07:58:50 +00:00
|
|
|
# 3rd-party apps from PyPI
|
2024-09-06 06:19:21 +00:00
|
|
|
'django_jsonform', # handles rendering Pydantic models to Django HTML widgets/forms https://github.com/bhch/django-jsonform
|
|
|
|
'signal_webhooks', # handles REST API outbound webhooks https://github.com/MrThearMan/django-signal-webhooks
|
|
|
|
'django_object_actions', # provides easy Django Admin action buttons on change views https://github.com/crccheck/django-object-actions
|
2024-09-13 10:25:46 +00:00
|
|
|
|
2024-09-06 08:48:18 +00:00
|
|
|
# Our ArchiveBox-provided apps
|
2024-10-15 04:50:47 +00:00
|
|
|
'config', # ArchiveBox config settings (loaded as a plugin, don't need to add it here)
|
2024-10-02 04:46:35 +00:00
|
|
|
'machine', # handles collecting and storing information about the host machine, network interfaces, installed binaries, etc.
|
2024-09-10 07:04:39 +00:00
|
|
|
'queues', # handles starting and managing background workers and processes
|
2024-09-03 07:58:50 +00:00
|
|
|
'abid_utils', # handles ABID ID creation, handling, and models
|
|
|
|
'core', # core django model with Snapshot, ArchiveResult, etc.
|
|
|
|
'api', # Django-Ninja-based Rest API interfaces, config, APIToken model, etc.
|
|
|
|
|
|
|
|
# ArchiveBox plugins
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_INSTALLED_APPS(), # all plugin django-apps found in archivebox/plugins_* and data/user_plugins,
|
2024-09-03 07:58:50 +00:00
|
|
|
|
|
|
|
# 3rd-party apps from PyPI that need to be loaded last
|
|
|
|
'admin_data_views', # handles rendering some convenient automatic read-only views of data in Django admin
|
|
|
|
'django_extensions', # provides Django Debug Toolbar (and other non-debug helpers)
|
2024-09-10 07:04:39 +00:00
|
|
|
'django_huey', # provides multi-queue support for django huey https://github.com/gaiacoop/django-huey
|
|
|
|
'bx_django_utils', # needed for huey_monitor https://github.com/boxine/bx_django_utils
|
|
|
|
'huey_monitor', # adds an admin UI for monitoring background huey tasks https://github.com/boxine/django-huey-monitor
|
2024-09-26 09:43:12 +00:00
|
|
|
|
|
|
|
# load plugins last so all other apps are already .ready() when we call plugins.ready()
|
|
|
|
'abx',
|
2019-04-02 22:53:21 +00:00
|
|
|
]
|
2021-02-16 21:23:09 +00:00
|
|
|
|
2023-09-14 09:41:27 +00:00
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
|
2021-02-16 21:23:09 +00:00
|
|
|
MIDDLEWARE = [
|
2021-04-10 08:16:12 +00:00
|
|
|
'core.middleware.TimezoneMiddleware',
|
2021-02-16 21:23:09 +00:00
|
|
|
'django.middleware.security.SecurityMiddleware',
|
|
|
|
'django.contrib.sessions.middleware.SessionMiddleware',
|
|
|
|
'django.middleware.common.CommonMiddleware',
|
|
|
|
'django.middleware.csrf.CsrfViewMiddleware',
|
|
|
|
'django.contrib.auth.middleware.AuthenticationMiddleware',
|
2021-09-30 15:40:13 +00:00
|
|
|
'core.middleware.ReverseProxyAuthMiddleware',
|
2021-02-16 21:23:09 +00:00
|
|
|
'django.contrib.messages.middleware.MessageMiddleware',
|
2021-04-10 08:16:12 +00:00
|
|
|
'core.middleware.CacheControlMiddleware',
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_MIDDLEWARES(),
|
2021-02-16 21:23:09 +00:00
|
|
|
]
|
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
|
2023-08-17 02:53:49 +00:00
|
|
|
################################################################################
|
|
|
|
### Authentication Settings
|
|
|
|
################################################################################
|
|
|
|
|
2024-08-20 08:58:36 +00:00
|
|
|
# AUTH_USER_MODEL = 'auth.User' # cannot be easily changed unfortunately
|
|
|
|
|
2021-02-16 21:23:09 +00:00
|
|
|
AUTHENTICATION_BACKENDS = [
|
2021-09-30 15:40:13 +00:00
|
|
|
'django.contrib.auth.backends.RemoteUserBackend',
|
2021-02-16 21:23:09 +00:00
|
|
|
'django.contrib.auth.backends.ModelBackend',
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_AUTHENTICATION_BACKENDS(),
|
2021-02-16 21:23:09 +00:00
|
|
|
]
|
|
|
|
|
2023-08-17 02:53:49 +00:00
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
# from ..plugins_auth.ldap.settings import LDAP_CONFIG
|
|
|
|
|
|
|
|
# if LDAP_CONFIG.LDAP_ENABLED:
|
|
|
|
# AUTH_LDAP_BIND_DN = LDAP_CONFIG.LDAP_BIND_DN
|
|
|
|
# AUTH_LDAP_SERVER_URI = LDAP_CONFIG.LDAP_SERVER_URI
|
|
|
|
# AUTH_LDAP_BIND_PASSWORD = LDAP_CONFIG.LDAP_BIND_PASSWORD
|
|
|
|
# AUTH_LDAP_USER_ATTR_MAP = LDAP_CONFIG.LDAP_USER_ATTR_MAP
|
|
|
|
# AUTH_LDAP_USER_SEARCH = LDAP_CONFIG.AUTH_LDAP_USER_SEARCH
|
2024-09-24 08:25:55 +00:00
|
|
|
|
2024-09-26 09:43:12 +00:00
|
|
|
# AUTHENTICATION_BACKENDS = LDAP_CONFIG.AUTHENTICATION_BACKENDS
|
2024-04-26 00:59:54 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
################################################################################
|
|
|
|
### Staticfile and Template Settings
|
|
|
|
################################################################################
|
|
|
|
|
|
|
|
STATIC_URL = '/static/'
|
2024-09-25 12:10:09 +00:00
|
|
|
TEMPLATES_DIR_NAME = 'templates'
|
2024-10-08 10:02:34 +00:00
|
|
|
CUSTOM_TEMPLATES_ENABLED = os.access(CONSTANTS.CUSTOM_TEMPLATES_DIR, os.R_OK) and CONSTANTS.CUSTOM_TEMPLATES_DIR.is_dir()
|
2020-10-31 07:08:03 +00:00
|
|
|
STATICFILES_DIRS = [
|
2024-10-08 10:02:34 +00:00
|
|
|
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_ENABLED else []),
|
2024-09-26 09:43:12 +00:00
|
|
|
# *[
|
|
|
|
# str(plugin_dir / 'static')
|
|
|
|
# for plugin_dir in PLUGIN_DIRS.values()
|
|
|
|
# if (plugin_dir / 'static').is_dir()
|
|
|
|
# ],
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_STATICFILES_DIRS(),
|
2024-09-25 12:10:09 +00:00
|
|
|
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / 'static'),
|
2020-10-31 07:08:03 +00:00
|
|
|
]
|
|
|
|
|
|
|
|
TEMPLATE_DIRS = [
|
2024-10-08 10:02:34 +00:00
|
|
|
*([str(CONSTANTS.CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_ENABLED else []),
|
2024-09-26 09:43:12 +00:00
|
|
|
# *[
|
|
|
|
# str(plugin_dir / 'templates')
|
|
|
|
# for plugin_dir in PLUGIN_DIRS.values()
|
|
|
|
# if (plugin_dir / 'templates').is_dir()
|
|
|
|
# ],
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_TEMPLATE_DIRS(),
|
2024-09-25 12:10:09 +00:00
|
|
|
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / 'core'),
|
|
|
|
str(PACKAGE_DIR / TEMPLATES_DIR_NAME / 'admin'),
|
|
|
|
str(PACKAGE_DIR / TEMPLATES_DIR_NAME),
|
2020-10-31 07:08:03 +00:00
|
|
|
]
|
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
TEMPLATES = [
|
|
|
|
{
|
|
|
|
'BACKEND': 'django.template.backends.django.DjangoTemplates',
|
2020-10-31 07:08:03 +00:00
|
|
|
'DIRS': TEMPLATE_DIRS,
|
2019-04-02 22:53:21 +00:00
|
|
|
'APP_DIRS': True,
|
|
|
|
'OPTIONS': {
|
|
|
|
'context_processors': [
|
|
|
|
'django.template.context_processors.debug',
|
|
|
|
'django.template.context_processors.request',
|
|
|
|
'django.contrib.auth.context_processors.auth',
|
|
|
|
'django.contrib.messages.context_processors.messages',
|
|
|
|
],
|
|
|
|
},
|
|
|
|
},
|
|
|
|
]
|
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
|
|
|
|
################################################################################
|
|
|
|
### External Service Settings
|
|
|
|
################################################################################
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2024-09-25 12:10:09 +00:00
|
|
|
# CACHE_DB_FILENAME = 'cache.sqlite3'
|
|
|
|
# CACHE_DB_PATH = CONSTANTS.CACHE_DIR / CACHE_DB_FILENAME
|
|
|
|
# CACHE_DB_TABLE = 'django_cache'
|
2024-05-12 08:42:34 +00:00
|
|
|
|
2024-09-25 12:10:09 +00:00
|
|
|
DATABASE_NAME = os.environ.get("ARCHIVEBOX_DATABASE_NAME", str(CONSTANTS.DATABASE_FILE))
|
2020-12-08 23:05:37 +00:00
|
|
|
|
2024-09-13 10:25:46 +00:00
|
|
|
SQLITE_CONNECTION_OPTIONS = {
|
2024-09-27 07:41:21 +00:00
|
|
|
"ENGINE": "django.db.backends.sqlite3",
|
2024-09-26 09:43:12 +00:00
|
|
|
"TIME_ZONE": CONSTANTS.TIMEZONE,
|
2024-09-13 10:25:46 +00:00
|
|
|
"OPTIONS": {
|
|
|
|
# https://gcollazo.com/optimal-sqlite-settings-for-django/
|
2024-09-27 07:41:21 +00:00
|
|
|
# https://litestream.io/tips/#busy-timeout
|
|
|
|
# https://docs.djangoproject.com/en/5.1/ref/databases/#setting-pragma-options
|
2024-09-13 10:25:46 +00:00
|
|
|
"timeout": 5,
|
|
|
|
"check_same_thread": False,
|
|
|
|
"transaction_mode": "IMMEDIATE",
|
|
|
|
"init_command": (
|
|
|
|
"PRAGMA foreign_keys=ON;"
|
|
|
|
"PRAGMA journal_mode = WAL;"
|
|
|
|
"PRAGMA synchronous = NORMAL;"
|
|
|
|
"PRAGMA temp_store = MEMORY;"
|
|
|
|
"PRAGMA mmap_size = 134217728;"
|
|
|
|
"PRAGMA journal_size_limit = 67108864;"
|
|
|
|
"PRAGMA cache_size = 2000;"
|
|
|
|
),
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
DATABASES = {
|
2024-09-10 07:04:39 +00:00
|
|
|
"default": {
|
|
|
|
"NAME": DATABASE_NAME,
|
2024-09-27 07:41:21 +00:00
|
|
|
**SQLITE_CONNECTION_OPTIONS,
|
2024-05-12 08:42:34 +00:00
|
|
|
},
|
2024-09-10 07:04:39 +00:00
|
|
|
"queue": {
|
2024-10-05 02:38:36 +00:00
|
|
|
"NAME": CONSTANTS.QUEUE_DATABASE_FILE,
|
2024-09-13 10:25:46 +00:00
|
|
|
**SQLITE_CONNECTION_OPTIONS,
|
2024-09-10 07:04:39 +00:00
|
|
|
},
|
2024-05-13 12:12:12 +00:00
|
|
|
# 'cache': {
|
|
|
|
# 'NAME': CACHE_DB_PATH,
|
2024-09-13 10:25:46 +00:00
|
|
|
# **SQLITE_CONNECTION_OPTIONS,
|
2024-05-13 12:12:12 +00:00
|
|
|
# },
|
2019-04-02 22:53:21 +00:00
|
|
|
}
|
2024-05-13 12:12:12 +00:00
|
|
|
MIGRATION_MODULES = {'signal_webhooks': None}
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2024-05-13 09:37:48 +00:00
|
|
|
# as much as I'd love this to be a UUID or ULID field, it's not supported yet as of Django 5.0
|
|
|
|
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
|
|
|
|
|
2021-02-17 23:25:23 +00:00
|
|
|
|
2024-09-10 07:04:39 +00:00
|
|
|
HUEY = {
|
|
|
|
"huey_class": "huey.SqliteHuey",
|
2024-10-05 02:38:36 +00:00
|
|
|
"filename": CONSTANTS.QUEUE_DATABASE_FILENAME,
|
2024-09-10 07:04:39 +00:00
|
|
|
"name": "system_tasks",
|
|
|
|
"results": True,
|
|
|
|
"store_none": True,
|
|
|
|
"immediate": False,
|
|
|
|
"utc": True,
|
|
|
|
"consumer": {
|
|
|
|
"workers": 1,
|
|
|
|
"worker_type": "thread",
|
|
|
|
"initial_delay": 0.1, # Smallest polling interval, same as -d.
|
|
|
|
"backoff": 1.15, # Exponential backoff using this rate, -b.
|
|
|
|
"max_delay": 10.0, # Max possible polling interval, -m.
|
|
|
|
"scheduler_interval": 1, # Check schedule every second, -s.
|
|
|
|
"periodic": True, # Enable crontab feature.
|
|
|
|
"check_worker_health": True, # Enable worker health checks.
|
|
|
|
"health_check_interval": 1, # Check worker health every second.
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
# https://huey.readthedocs.io/en/latest/contrib.html#setting-things-up
|
|
|
|
# https://github.com/gaiacoop/django-huey
|
|
|
|
DJANGO_HUEY = {
|
|
|
|
"default": "system_tasks",
|
|
|
|
"queues": {
|
|
|
|
HUEY["name"]: HUEY.copy(),
|
|
|
|
# more registered here at plugin import-time by BaseQueue.register()
|
2024-10-05 02:38:36 +00:00
|
|
|
**abx.django.use.get_DJANGO_HUEY_QUEUES(QUEUE_DATABASE_NAME=CONSTANTS.QUEUE_DATABASE_FILENAME),
|
2024-09-10 07:04:39 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
|
|
|
class HueyDBRouter:
|
2024-09-11 23:50:44 +00:00
|
|
|
"""
|
|
|
|
A router to store all the Huey result k:v / Huey Monitor models in the queue.sqlite3 database.
|
|
|
|
We keep the databases separate because the queue database receives many more reads/writes per second
|
|
|
|
and we want to avoid single-write lock contention with the main database. Also all the in-progress task
|
|
|
|
data is ephemeral/not-important-long-term. This makes it easier to for the user to clear non-critical
|
|
|
|
temp data by just deleting queue.sqlite3 and leaving index.sqlite3.
|
|
|
|
"""
|
2024-09-10 07:04:39 +00:00
|
|
|
|
|
|
|
route_app_labels = {"huey_monitor", "django_huey", "djhuey"}
|
|
|
|
|
|
|
|
def db_for_read(self, model, **hints):
|
|
|
|
if model._meta.app_label in self.route_app_labels:
|
|
|
|
return "queue"
|
|
|
|
return 'default'
|
|
|
|
|
|
|
|
def db_for_write(self, model, **hints):
|
|
|
|
if model._meta.app_label in self.route_app_labels:
|
|
|
|
return "queue"
|
|
|
|
return 'default'
|
|
|
|
|
|
|
|
def allow_relation(self, obj1, obj2, **hints):
|
|
|
|
if obj1._meta.app_label in self.route_app_labels or obj2._meta.app_label in self.route_app_labels:
|
|
|
|
return obj1._meta.app_label == obj2._meta.app_label
|
|
|
|
return None
|
|
|
|
|
|
|
|
def allow_migrate(self, db, app_label, model_name=None, **hints):
|
|
|
|
if app_label in self.route_app_labels:
|
|
|
|
return db == "queue"
|
|
|
|
return db == "default"
|
|
|
|
|
|
|
|
DATABASE_ROUTERS = ['core.settings.HueyDBRouter']
|
|
|
|
|
2021-02-17 23:25:23 +00:00
|
|
|
CACHES = {
|
2024-05-13 12:12:12 +00:00
|
|
|
'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'},
|
|
|
|
# 'sqlite': {'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'cache'},
|
|
|
|
# 'dummy': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
|
|
|
|
# 'filebased': {"BACKEND": "django.core.cache.backends.filebased.FileBasedCache", "LOCATION": CACHE_DIR / 'cache_filebased'},
|
2021-02-17 23:25:23 +00:00
|
|
|
}
|
2021-02-16 11:18:39 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
|
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2024-05-06 13:58:03 +00:00
|
|
|
STORAGES = {
|
2024-05-06 14:14:01 +00:00
|
|
|
"default": {
|
|
|
|
"BACKEND": "django.core.files.storage.FileSystemStorage",
|
|
|
|
},
|
|
|
|
"staticfiles": {
|
|
|
|
"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
|
|
|
|
},
|
2024-05-06 13:58:03 +00:00
|
|
|
"archive": {
|
|
|
|
"BACKEND": "django.core.files.storage.FileSystemStorage",
|
|
|
|
"OPTIONS": {
|
|
|
|
"base_url": "/archive/",
|
2024-09-25 12:10:09 +00:00
|
|
|
"location": ARCHIVE_DIR,
|
2024-05-06 13:58:03 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
# "personas": {
|
|
|
|
# "BACKEND": "django.core.files.storage.FileSystemStorage",
|
|
|
|
# "OPTIONS": {
|
|
|
|
# "base_url": "/personas/",
|
|
|
|
# "location": PERSONAS_DIR,
|
|
|
|
# },
|
|
|
|
# },
|
|
|
|
}
|
|
|
|
|
2019-04-22 23:07:39 +00:00
|
|
|
################################################################################
|
|
|
|
### Security Settings
|
|
|
|
################################################################################
|
2020-10-31 07:08:03 +00:00
|
|
|
|
2024-10-01 06:21:34 +00:00
|
|
|
SECRET_KEY = SERVER_CONFIG.SECRET_KEY or get_random_string(50, 'abcdefghijklmnopqrstuvwxyz0123456789_')
|
2020-10-31 07:08:03 +00:00
|
|
|
|
2024-10-01 06:21:34 +00:00
|
|
|
ALLOWED_HOSTS = SERVER_CONFIG.ALLOWED_HOSTS.split(',')
|
|
|
|
CSRF_TRUSTED_ORIGINS = list(set(SERVER_CONFIG.CSRF_TRUSTED_ORIGINS.split(',')))
|
2024-08-23 01:40:47 +00:00
|
|
|
|
|
|
|
# automatically fix case when user sets ALLOWED_HOSTS (e.g. to archivebox.example.com)
|
|
|
|
# but forgets to add https://archivebox.example.com to CSRF_TRUSTED_ORIGINS
|
2024-08-23 09:01:40 +00:00
|
|
|
for hostname in ALLOWED_HOSTS:
|
|
|
|
https_endpoint = f'https://{hostname}'
|
|
|
|
if hostname != '*' and https_endpoint not in CSRF_TRUSTED_ORIGINS:
|
|
|
|
print(f'[!] WARNING: {https_endpoint} from ALLOWED_HOSTS should be added to CSRF_TRUSTED_ORIGINS')
|
|
|
|
CSRF_TRUSTED_ORIGINS.append(https_endpoint)
|
2020-10-31 07:08:03 +00:00
|
|
|
|
2019-04-22 23:07:39 +00:00
|
|
|
SECURE_BROWSER_XSS_FILTER = True
|
|
|
|
SECURE_CONTENT_TYPE_NOSNIFF = True
|
2021-04-10 08:19:30 +00:00
|
|
|
SECURE_REFERRER_POLICY = 'strict-origin-when-cross-origin'
|
2020-10-31 07:08:03 +00:00
|
|
|
|
2019-04-22 23:07:39 +00:00
|
|
|
CSRF_COOKIE_SECURE = False
|
2020-10-31 07:08:03 +00:00
|
|
|
SESSION_COOKIE_SECURE = False
|
2024-09-03 21:16:44 +00:00
|
|
|
SESSION_COOKIE_HTTPONLY = True
|
2019-04-22 23:07:39 +00:00
|
|
|
SESSION_COOKIE_DOMAIN = None
|
2020-10-31 07:08:03 +00:00
|
|
|
SESSION_COOKIE_AGE = 1209600 # 2 weeks
|
2019-04-22 23:07:39 +00:00
|
|
|
SESSION_EXPIRE_AT_BROWSER_CLOSE = False
|
2024-09-03 21:16:44 +00:00
|
|
|
SESSION_SAVE_EVERY_REQUEST = False
|
2019-04-22 23:07:39 +00:00
|
|
|
|
2021-02-16 11:18:39 +00:00
|
|
|
SESSION_ENGINE = "django.contrib.sessions.backends.db"
|
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
AUTH_PASSWORD_VALIDATORS = [
|
|
|
|
{'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator'},
|
|
|
|
{'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator'},
|
|
|
|
{'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator'},
|
|
|
|
{'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator'},
|
|
|
|
]
|
|
|
|
|
2024-08-27 03:14:47 +00:00
|
|
|
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
|
2024-09-10 01:42:59 +00:00
|
|
|
DATA_UPLOAD_MAX_MEMORY_SIZE = 26_214_400 # 25MB
|
2024-08-27 03:14:47 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
################################################################################
|
|
|
|
### Shell Settings
|
|
|
|
################################################################################
|
2019-04-22 23:07:39 +00:00
|
|
|
|
|
|
|
SHELL_PLUS = 'ipython'
|
|
|
|
SHELL_PLUS_PRINT_SQL = False
|
|
|
|
IPYTHON_ARGUMENTS = ['--no-confirm-exit', '--no-banner']
|
|
|
|
IPYTHON_KERNEL_DISPLAY_NAME = 'ArchiveBox Django Shell'
|
|
|
|
if IS_SHELL:
|
2024-10-15 00:35:28 +00:00
|
|
|
os.environ['PYTHONSTARTUP'] = str(PACKAGE_DIR / 'misc' / 'shell_welcome_message.py')
|
2019-04-22 23:07:39 +00:00
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
|
2020-10-31 07:08:03 +00:00
|
|
|
################################################################################
|
|
|
|
### Internationalization & Localization Settings
|
|
|
|
################################################################################
|
|
|
|
|
2019-04-02 22:53:21 +00:00
|
|
|
LANGUAGE_CODE = 'en-us'
|
2021-04-10 08:19:30 +00:00
|
|
|
USE_I18N = True
|
|
|
|
USE_TZ = True
|
2024-09-05 06:43:25 +00:00
|
|
|
DATETIME_FORMAT = 'Y-m-d h:i:s A'
|
|
|
|
SHORT_DATETIME_FORMAT = 'Y-m-d h:i:s A'
|
2024-09-26 09:43:12 +00:00
|
|
|
TIME_ZONE = CONSTANTS.TIMEZONE # django convention is TIME_ZONE, archivebox config uses TIMEZONE, they are equivalent
|
2022-06-09 03:00:29 +00:00
|
|
|
|
2021-02-16 01:51:23 +00:00
|
|
|
|
2024-08-21 01:31:21 +00:00
|
|
|
from django.conf.locale.en import formats as en_formats # type: ignore
|
2021-04-10 08:19:30 +00:00
|
|
|
|
2024-09-24 22:13:54 +00:00
|
|
|
en_formats.DATETIME_FORMAT = DATETIME_FORMAT # monkey patch en_format default with our preferred format
|
2021-04-10 08:19:30 +00:00
|
|
|
en_formats.SHORT_DATETIME_FORMAT = SHORT_DATETIME_FORMAT
|
|
|
|
|
2021-02-16 01:51:23 +00:00
|
|
|
|
|
|
|
################################################################################
|
|
|
|
### Logging Settings
|
|
|
|
################################################################################
|
|
|
|
|
2021-02-16 06:23:01 +00:00
|
|
|
|
2024-09-24 22:13:54 +00:00
|
|
|
from .settings_logging import SETTINGS_LOGGING, LOGS_DIR, ERROR_LOG
|
2024-09-06 04:45:43 +00:00
|
|
|
|
2024-09-24 22:13:54 +00:00
|
|
|
LOGGING = SETTINGS_LOGGING
|
2024-05-06 13:58:03 +00:00
|
|
|
|
|
|
|
|
2024-08-27 03:14:47 +00:00
|
|
|
################################################################################
|
|
|
|
### REST API Outbound Webhooks settings
|
|
|
|
################################################################################
|
|
|
|
|
2024-05-06 13:58:03 +00:00
|
|
|
# Add default webhook configuration to the User model
|
2024-05-13 09:36:15 +00:00
|
|
|
SIGNAL_WEBHOOKS_CUSTOM_MODEL = 'api.models.OutboundWebhook'
|
2024-05-06 13:58:03 +00:00
|
|
|
SIGNAL_WEBHOOKS = {
|
|
|
|
"HOOKS": {
|
2024-05-13 09:36:15 +00:00
|
|
|
# ... is a special sigil value that means "use the default autogenerated hooks"
|
|
|
|
"django.contrib.auth.models.User": ...,
|
2024-05-06 14:13:54 +00:00
|
|
|
"core.models.Snapshot": ...,
|
|
|
|
"core.models.ArchiveResult": ...,
|
|
|
|
"core.models.Tag": ...,
|
|
|
|
"api.models.APIToken": ...,
|
2024-05-06 13:58:03 +00:00
|
|
|
},
|
|
|
|
}
|
2024-05-06 18:06:42 +00:00
|
|
|
|
2024-08-27 03:14:47 +00:00
|
|
|
################################################################################
|
|
|
|
### Admin Data View Settings
|
|
|
|
################################################################################
|
2024-05-06 18:06:42 +00:00
|
|
|
|
|
|
|
ADMIN_DATA_VIEWS = {
|
2024-05-18 03:13:54 +00:00
|
|
|
"NAME": "Environment",
|
2024-05-06 18:06:42 +00:00
|
|
|
"URLS": [
|
|
|
|
{
|
2024-05-18 03:13:54 +00:00
|
|
|
"route": "config/",
|
2024-05-06 18:06:42 +00:00
|
|
|
"view": "core.views.live_config_list_view",
|
2024-05-18 03:13:54 +00:00
|
|
|
"name": "Configuration",
|
2024-05-06 18:06:42 +00:00
|
|
|
"items": {
|
|
|
|
"route": "<str:key>/",
|
|
|
|
"view": "core.views.live_config_value_view",
|
2024-05-18 03:13:54 +00:00
|
|
|
"name": "config_val",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"route": "binaries/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.binaries_list_view",
|
2024-10-03 10:10:22 +00:00
|
|
|
"name": "Dependencies",
|
2024-05-18 03:13:54 +00:00
|
|
|
"items": {
|
|
|
|
"route": "<str:key>/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.binary_detail_view",
|
2024-05-18 03:13:54 +00:00
|
|
|
"name": "binary",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"route": "plugins/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.plugins_list_view",
|
2024-05-18 03:13:54 +00:00
|
|
|
"name": "Plugins",
|
|
|
|
"items": {
|
|
|
|
"route": "<str:key>/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.plugin_detail_view",
|
2024-05-18 03:13:54 +00:00
|
|
|
"name": "plugin",
|
2024-05-06 18:06:42 +00:00
|
|
|
},
|
|
|
|
},
|
2024-09-10 07:05:01 +00:00
|
|
|
{
|
|
|
|
"route": "workers/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.worker_list_view",
|
2024-09-10 07:05:01 +00:00
|
|
|
"name": "Workers",
|
|
|
|
"items": {
|
|
|
|
"route": "<str:key>/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.worker_detail_view",
|
2024-09-10 07:05:01 +00:00
|
|
|
"name": "worker",
|
|
|
|
},
|
|
|
|
},
|
|
|
|
{
|
|
|
|
"route": "logs/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.log_list_view",
|
2024-09-10 07:05:01 +00:00
|
|
|
"name": "Logs",
|
|
|
|
"items": {
|
|
|
|
"route": "<str:key>/",
|
2024-09-30 22:59:05 +00:00
|
|
|
"view": "archivebox.config.views.log_detail_view",
|
2024-09-10 07:05:01 +00:00
|
|
|
"name": "log",
|
|
|
|
},
|
|
|
|
},
|
2024-09-27 07:41:21 +00:00
|
|
|
*abx.django.use.get_ADMIN_DATA_VIEWS_URLS(),
|
2024-05-06 18:06:42 +00:00
|
|
|
],
|
|
|
|
}
|
2024-08-27 03:14:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
################################################################################
|
|
|
|
### Debug Settings
|
|
|
|
################################################################################
|
|
|
|
|
|
|
|
# only enable debug toolbar when in DEBUG mode with --nothreading (it doesnt work in multithreaded mode)
|
2024-08-27 03:20:33 +00:00
|
|
|
DEBUG_TOOLBAR = False
|
2024-08-27 03:14:47 +00:00
|
|
|
DEBUG_TOOLBAR = DEBUG_TOOLBAR and DEBUG and ('--nothreading' in sys.argv) and ('--reload' not in sys.argv)
|
|
|
|
if DEBUG_TOOLBAR:
|
|
|
|
try:
|
|
|
|
import debug_toolbar # noqa
|
|
|
|
DEBUG_TOOLBAR = True
|
|
|
|
except ImportError:
|
|
|
|
DEBUG_TOOLBAR = False
|
|
|
|
|
|
|
|
if DEBUG_TOOLBAR:
|
|
|
|
INSTALLED_APPS = [*INSTALLED_APPS, 'debug_toolbar']
|
|
|
|
INTERNAL_IPS = ['0.0.0.0', '127.0.0.1', '*']
|
|
|
|
DEBUG_TOOLBAR_CONFIG = {
|
|
|
|
"SHOW_TOOLBAR_CALLBACK": lambda request: True,
|
|
|
|
"RENDER_PANELS": True,
|
|
|
|
}
|
|
|
|
DEBUG_TOOLBAR_PANELS = [
|
|
|
|
'debug_toolbar.panels.history.HistoryPanel',
|
|
|
|
'debug_toolbar.panels.versions.VersionsPanel',
|
|
|
|
'debug_toolbar.panels.timer.TimerPanel',
|
|
|
|
'debug_toolbar.panels.settings.SettingsPanel',
|
|
|
|
'debug_toolbar.panels.headers.HeadersPanel',
|
|
|
|
'debug_toolbar.panels.request.RequestPanel',
|
|
|
|
'debug_toolbar.panels.sql.SQLPanel',
|
|
|
|
'debug_toolbar.panels.staticfiles.StaticFilesPanel',
|
|
|
|
# 'debug_toolbar.panels.templates.TemplatesPanel',
|
|
|
|
'debug_toolbar.panels.cache.CachePanel',
|
|
|
|
'debug_toolbar.panels.signals.SignalsPanel',
|
|
|
|
'debug_toolbar.panels.logging.LoggingPanel',
|
|
|
|
'debug_toolbar.panels.redirects.RedirectsPanel',
|
|
|
|
'debug_toolbar.panels.profiling.ProfilingPanel',
|
|
|
|
'djdt_flamegraph.FlamegraphPanel',
|
|
|
|
]
|
|
|
|
MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
|
|
|
|
|
|
|
|
if DEBUG:
|
|
|
|
from django_autotyping.typing import AutotypingSettingsDict
|
|
|
|
|
|
|
|
INSTALLED_APPS += ['django_autotyping']
|
|
|
|
AUTOTYPING: AutotypingSettingsDict = {
|
|
|
|
"STUBS_GENERATION": {
|
2024-09-23 02:26:26 +00:00
|
|
|
"LOCAL_STUBS_DIR": PACKAGE_DIR / "typings",
|
2024-08-27 03:14:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# https://github.com/bensi94/Django-Requests-Tracker (improved version of django-debug-toolbar)
|
|
|
|
# Must delete archivebox/templates/admin to use because it relies on some things we override
|
|
|
|
# visit /__requests_tracker__/ to access
|
|
|
|
DEBUG_REQUESTS_TRACKER = True
|
2024-08-27 03:20:33 +00:00
|
|
|
DEBUG_REQUESTS_TRACKER = DEBUG_REQUESTS_TRACKER and DEBUG
|
2024-08-27 03:14:47 +00:00
|
|
|
if DEBUG_REQUESTS_TRACKER:
|
|
|
|
import requests_tracker
|
|
|
|
|
|
|
|
INSTALLED_APPS += ["requests_tracker"]
|
|
|
|
MIDDLEWARE += ["requests_tracker.middleware.requests_tracker_middleware"]
|
|
|
|
INTERNAL_IPS = ["127.0.0.1", "10.0.2.2", "0.0.0.0", "*"]
|
|
|
|
|
|
|
|
TEMPLATE_DIRS.insert(0, str(Path(inspect.getfile(requests_tracker)).parent / "templates"))
|
|
|
|
|
|
|
|
REQUESTS_TRACKER_CONFIG = {
|
|
|
|
"TRACK_SQL": True,
|
|
|
|
"ENABLE_STACKTRACES": False,
|
|
|
|
"IGNORE_PATHS_PATTERNS": (
|
|
|
|
r".*/favicon\.ico",
|
|
|
|
r".*\.png",
|
|
|
|
r"/admin/jsi18n/",
|
|
|
|
),
|
|
|
|
"IGNORE_SQL_PATTERNS": (
|
|
|
|
r"^SELECT .* FROM django_migrations WHERE app = 'requests_tracker'",
|
|
|
|
r"^SELECT .* FROM django_migrations WHERE app = 'auth'",
|
|
|
|
),
|
|
|
|
}
|
|
|
|
|
2024-10-04 00:39:39 +00:00
|
|
|
# # https://docs.pydantic.dev/logfire/integrations/django/ (similar to DataDog / NewRelic / etc.)
|
|
|
|
# DEBUG_LOGFIRE = False
|
2024-10-08 10:02:34 +00:00
|
|
|
# DEBUG_LOGFIRE = DEBUG_LOGFIRE and os.access(DATA_DIR / '.logfire', os.W_OK) and (DATA_DIR / '.logfire').is_dir()
|
2024-09-03 07:58:50 +00:00
|
|
|
|
|
|
|
|
|
|
|
# For usage with https://www.jetadmin.io/integrations/django
|
|
|
|
# INSTALLED_APPS += ['jet_django']
|
|
|
|
# JET_PROJECT = 'archivebox'
|
|
|
|
# JET_TOKEN = 'some-api-token-here'
|
2024-09-26 09:43:12 +00:00
|
|
|
|
|
|
|
|
2024-09-27 07:41:21 +00:00
|
|
|
abx.django.use.register_checks()
|
2024-10-15 04:50:47 +00:00
|
|
|
# abx.archivebox.use.register_all_hooks(globals())
|
2024-09-26 09:43:12 +00:00
|
|
|
|
2024-09-27 07:41:21 +00:00
|
|
|
# import ipdb; ipdb.set_trace()
|