Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-22 20:23:12 +00:00
Code Issues Projects Releases Packages Wiki Activity

wip refactoring

Browse source
This commit is contained in:
Nick Sweeting 2024-01-19 07:09:24 -08:00
parent ea2c5a21ff
commit 3234a366d5
2 changed files with 79 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
archivebox/core/settings.py
View file

@ -64,6 +64,9 @@ INSTALLED_APPS = [
# Plugins
'plugins.replaywebpage',
'plugins.gallerydl',
# 'plugins.browsertrix',
# 'plugins.playwright',
# ...
# someday we may have enough plugins to justify dynamic loading:
# *(path.parent.name for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/apps.py')),,
@ -71,6 +74,64 @@ INSTALLED_APPS = [
'django_extensions',
]
################################################################################
### Staticfile and Template Settings
################################################################################
STATIC_URL = '/static/'
STATIC_ROOT = Path(PACKAGE_DIR) / 'collected_static'
STATICFILES_DIRS = [
*([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
# Plugins
str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/static'),
str(Path(PACKAGE_DIR) / 'plugins/gallerydl/static'),
# str(Path(PACKAGE_DIR) / 'plugins/browsertrix/static'),
# str(Path(PACKAGE_DIR) / 'plugins/playwright/static'),
# ...
# someday if there are many more plugins / user-addable plugins:
# *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/static')),
]
MEDIA_URL = '/archive/'
MEDIA_ROOT = OUTPUT_DIR / 'archive'
TEMPLATE_DIRS = [
*([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
# Plugins
str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/templates'),
str(Path(PACKAGE_DIR) / 'plugins/gallerydl/templates'),
# str(Path(PACKAGE_DIR) / 'plugins/browsertrix/templates'),
# str(Path(PACKAGE_DIR) / 'plugins/playwright/templates'),
# ...
#
# someday if there are many more plugins / user-addable plugins:
# *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/templates')),
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': TEMPLATE_DIRS,
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
# For usage with https://www.jetadmin.io/integrations/django
# INSTALLED_APPS += ['jet_django']
@ -178,58 +239,6 @@ if DEBUG_TOOLBAR:
]
MIDDLEWARE = [*MIDDLEWARE, 'debug_toolbar.middleware.DebugToolbarMiddleware']
################################################################################
### Staticfile and Template Settings
################################################################################
STATIC_URL = '/static/'
STATIC_ROOT = Path(PACKAGE_DIR) / 'collected_static'
STATICFILES_DIRS = [
*([str(CUSTOM_TEMPLATES_DIR / 'static')] if CUSTOM_TEMPLATES_DIR else []),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'static'),
# Plugins
str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/static'),
# ...
# someday if there are many more plugins / user-addable plugins:
# *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/static')),
]
MEDIA_URL = '/archive/'
MEDIA_ROOT = OUTPUT_DIR / 'archive'
TEMPLATE_DIRS = [
*([str(CUSTOM_TEMPLATES_DIR)] if CUSTOM_TEMPLATES_DIR else []),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'core'),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME / 'admin'),
str(Path(PACKAGE_DIR) / TEMPLATES_DIR_NAME),
# Plugins
str(Path(PACKAGE_DIR) / 'plugins/replaywebpage/templates')
# ...
#
# someday if there are many more plugins / user-addable plugins:
# *(str(path) for path in (Path(PACKAGE_DIR) / 'plugins').glob('*/templates')),
]
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': TEMPLATE_DIRS,
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
################################################################################

23
archivebox/plugins/gallerydl/models.py
View file

@ -5,6 +5,9 @@ class GalleryDLDependency(SingletonModel):
GALLERYDL_ENABLED = models.BooleanField(default=True)
GALLERYDL_BINARY = models.CharField(max_length=255, default='gallery-dl')
# GALLERYDL_WORKERS = models.IntegerField(default='{NUM_CORES}')
def __str__(self):
return "GalleryDL Dependency Configuration"
@ -28,6 +31,20 @@ class GalleryDLDependency(SingletonModel):
return self.GALLERYDL_ENABLED and self.is_valid
def run(args, pwd, timeout):
errors = None
timer = TimedProgress(timeout, prefix=' ')
try:
proc = run(cmd=[self.bin_path, *args]=True, pwd=pwd, timeout=timeout)run(cmd=[self.bin_path, *args]=True, pwd=pwd, timeout=timeout)
except Exception as err:
errors = err
finally:
timer.end()
return proc, timer, errors
def pretty_version(self):
if self.enabled:
if self.is_valid:
@ -142,9 +159,8 @@ class GalleryDLExtractor(SingletonModel):
]
status, stdout, stderr, output_path = 'failed', '', '', None
timer = TimedProgress(timeout, prefix=' ')
try:
proc = run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT, text=True)
proc, timer, errors = self.GALLERYDL_DEPENDENCY.run(cmd, cwd=extractor_dir, timeout=self.GALLERYDL_TIMEOUT)
stdout, stderr = proc.stdout, proc.stderr
if 'ERROR: Unsupported URL' in stderr:
@ -154,11 +170,8 @@ class GalleryDLExtractor(SingletonModel):
if proc.returncode == 0 and 'finished' in stdout:
output_path = extractor_dir / 'index.html'
status = 'succeeded'
except Exception as err:
stderr += err
finally:
timer.end()
num_bytes, num_dirs, num_files = get_dir_size(extractor_dir)