Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-25 13:40:20 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1146 from mrled/add-favicon-provider

Browse source
This commit is contained in:
Nick Sweeting 2023-05-05 19:00:15 -07:00 committed by GitHub
commit 5bbc5228cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/config.py
View file

@ -183,7 +183,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'--compressed' '--compressed'
]}, ]},
'GIT_ARGS': {'type': list, 'default': ['--recursive']}, 'GIT_ARGS': {'type': list, 'default': ['--recursive']},
'SINGLEFILE_ARGS': {'type': list, 'default' : None} 'SINGLEFILE_ARGS': {'type': list, 'default' : None},
'FAVICON_PROVIDER': {'type': str, 'default': 'https://www.google.com/s2/favicons?domain={}'},
}, },
'SEARCH_BACKEND_CONFIG' : { 'SEARCH_BACKEND_CONFIG' : {

3
archivebox/extractors/favicon.py
View file

@ -10,6 +10,7 @@ from ..util import enforce_types, domain
from ..config import ( from ..config import (
TIMEOUT, TIMEOUT,
SAVE_FAVICON, SAVE_FAVICON,
FAVICON_PROVIDER,
CURL_BINARY, CURL_BINARY,
CURL_ARGS, CURL_ARGS,
CURL_VERSION, CURL_VERSION,
@ -40,7 +41,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
'--output', str(output), '--output', str(output),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []), *(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']), *([] if CHECK_SSL_VALIDITY else ['--insecure']),
'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)), FAVICON_PROVIDER.format(domain(link.url)),
] ]
status = 'failed' status = 'failed'
timer = TimedProgress(timeout, prefix=' ') timer = TimedProgress(timeout, prefix=' ')

4
archivebox/index/schema.py
View file

@ -20,7 +20,7 @@ from django.utils.functional import cached_property
from ..system import get_dir_size from ..system import get_dir_size
from ..util import ts_to_date_str, parse_date from ..util import ts_to_date_str, parse_date
from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME, FAVICON_PROVIDER
class ArchiveError(Exception): class ArchiveError(Exception):
def __init__(self, message, hints=None): def __init__(self, message, hints=None):
@ -423,7 +423,7 @@ class Link:
canonical = { canonical = {
'index_path': 'index.html', 'index_path': 'index.html',
'favicon_path': 'favicon.ico', 'favicon_path': 'favicon.ico',
'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain), 'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
'wget_path': wget_output_path(self), 'wget_path': wget_output_path(self),
'warc_path': 'warc/', 'warc_path': 'warc/',
'singlefile_path': 'singlefile.html', 'singlefile_path': 'singlefile.html',