Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 04:03:06 +00:00
Code Issues Projects Releases Packages Wiki Activity

relocate LIB_DIR and TMP_DIR inside docker so it doesnt clash with outside docker

Browse source
This commit is contained in:
Nick Sweeting 2024-10-03 03:43:02 -07:00
parent f4f1d7893c
commit b36e89d086
No known key found for this signature in database
4 changed files with 28 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
archivebox/config/constants.py
View file

@ -18,6 +18,7 @@ PACKAGE_DIR: Path = Path(__file__).resolve().parent.parent # archivebox sourc
DATA_DIR: Path = Path(os.curdir).resolve() # archivebox user data dir
ARCHIVE_DIR: Path = DATA_DIR / 'archive' # archivebox snapshot data dir
IN_DOCKER = os.environ.get('IN_DOCKER', False) in ('1', 'true', 'True', 'yes')
def _detect_installed_version(PACKAGE_DIR: Path):
"""Autodetect the installed archivebox version by using pip package metadata, pyproject.toml file, or package.json file"""
@ -79,8 +80,8 @@ class ConstantsDict(Mapping):
PERSONAS_DIR: Path = DATA_DIR / PERSONAS_DIR_NAME
CACHE_DIR: Path = DATA_DIR / CACHE_DIR_NAME
LOGS_DIR: Path = DATA_DIR / LOGS_DIR_NAME
LIB_DIR: Path = DATA_DIR / LIB_DIR_NAME
TMP_DIR: Path = DATA_DIR / TMP_DIR_NAME
LIB_DIR: Path = (Path('/tmp') if IN_DOCKER else DATA_DIR) / LIB_DIR_NAME
TMP_DIR: Path = (Path('/tmp') if IN_DOCKER else DATA_DIR) / TMP_DIR_NAME
CUSTOM_TEMPLATES_DIR: Path = DATA_DIR / CUSTOM_TEMPLATES_DIR_NAME
USER_PLUGINS_DIR: Path = DATA_DIR / USER_PLUGINS_DIR_NAME

4
archivebox/config/legacy.py
View file

@ -756,7 +756,9 @@ def setup_django(out_dir: Path | None=None, check_db=False, config: benedict=CON
global DJANGO_SET_UP
if DJANGO_SET_UP:
raise Exception('django is already set up!')
# raise Exception('django is already set up!')
# TODO: figure out why CLI entrypoints with init_pending are running this twice sometimes
return
with Progress(transient=True, expand=True, console=CONSOLE) as INITIAL_STARTUP_PROGRESS:
INITIAL_STARTUP_PROGRESS_TASK = INITIAL_STARTUP_PROGRESS.add_task("[green]Loading modules...", total=25)

7
archivebox/queues/settings.py
View file

@ -1,13 +1,14 @@
from pathlib import Path
from archivebox.config import DATA_DIR, CONSTANTS
from archivebox.config import CONSTANTS
DATA_DIR = CONSTANTS.DATA_DIR
LOGS_DIR = CONSTANTS.LOGS_DIR
TMP_DIR = CONSTANTS.TMP_DIR
Path.mkdir(TMP_DIR, exist_ok=True)
CONFIG_FILE = TMP_DIR / "supervisord.conf"
SUPERVISORD_CONFIG_FILE = TMP_DIR / "supervisord.conf"
PID_FILE = TMP_DIR / "supervisord.pid"
SOCK_FILE = TMP_DIR / "supervisord.sock"
LOG_FILE = TMP_DIR / "supervisord.log"
WORKER_DIR = TMP_DIR / "workers"
WORKERS_DIR = TMP_DIR / "workers"

32
archivebox/queues/supervisor_util.py
View file

@ -12,7 +12,7 @@ from typing import Dict, cast
from supervisor.xmlrpc import SupervisorTransport
from xmlrpc.client import ServerProxy
from .settings import CONFIG_FILE, PID_FILE, SOCK_FILE, LOG_FILE, WORKER_DIR, TMP_DIR, LOGS_DIR
from .settings import SUPERVISORD_CONFIG_FILE, DATA_DIR, PID_FILE, SOCK_FILE, LOG_FILE, WORKERS_DIR, TMP_DIR, LOGS_DIR
from typing import Iterator
@ -36,38 +36,39 @@ def create_supervisord_config():
[supervisord]
nodaemon = true
environment = IS_SUPERVISORD_PARENT="true"
pidfile = %(here)s/{PID_FILE.name}
logfile = %(here)s/../{LOGS_DIR.name}/{LOG_FILE.name}
childlogdir = %(here)s/../{LOGS_DIR.name}
directory = %(here)s/..
pidfile = {TMP_DIR}/{PID_FILE.name}
logfile = {LOGS_DIR}/{LOG_FILE.name}
childlogdir = {LOGS_DIR}
directory = {DATA_DIR}
strip_ansi = true
nocleanup = true
[unix_http_server]
file = %(here)s/{SOCK_FILE.name}
file = {TMP_DIR}/{SOCK_FILE.name}
chmod = 0700
[supervisorctl]
serverurl = unix://%(here)s/{SOCK_FILE.name}
serverurl = unix://{TMP_DIR}/{SOCK_FILE.name}
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[include]
files = %(here)s/{WORKER_DIR.name}/*.conf
files = {WORKERS_DIR}/*.conf
"""
CONFIG_FILE.write_text(config_content)
SUPERVISORD_CONFIG_FILE.write_text(config_content)
def create_worker_config(daemon):
Path.mkdir(WORKER_DIR, exist_ok=True)
Path.mkdir(WORKERS_DIR, exist_ok=True)
name = daemon['name']
configfile = WORKER_DIR / f"{name}.conf"
configfile = WORKERS_DIR / f"{name}.conf"
config_content = f"[program:{name}]\n"
for key, value in daemon.items():
if key == 'name': continue
if key == 'name':
continue
config_content += f"{key}={value}\n"
config_content += "\n"
@ -117,7 +118,7 @@ def start_new_supervisord_process(daemonize=False):
# Start supervisord
subprocess.Popen(
f"supervisord --configuration={CONFIG_FILE}",
f"supervisord --configuration={SUPERVISORD_CONFIG_FILE}",
stdin=None,
shell=True,
start_new_session=daemonize,
@ -146,8 +147,11 @@ def get_or_create_supervisord_process(daemonize=False):
if supervisor is None:
stop_existing_supervisord_process()
supervisor = start_new_supervisord_process(daemonize=daemonize)
time.sleep(0.5)
assert supervisor and supervisor.getPID(), "Failed to start supervisord or connect to it!"
assert supervisor, "Failed to start supervisord or connect to it!"
supervisor.getPID() # make sure it doesn't throw an exception
return supervisor
def start_worker(supervisor, daemon, lazy=False):