Mirrors/ArchiveBox
2
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox synced 2024-11-26 06:00:22 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #1120 from fa0311/dev

Browse source
This commit is contained in:
Nick Sweeting 2023-03-15 17:05:46 -06:00 committed by GitHub
commit f5e631dbfa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 9 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
archivebox/config.py
View file

@ -139,6 +139,7 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'COOKIES_FILE': {'type': str, 'default': None}, 'COOKIES_FILE': {'type': str, 'default': None},
'CHROME_USER_DATA_DIR': {'type': str, 'default': None}, 'CHROME_USER_DATA_DIR': {'type': str, 'default': None},
'CHROME_TIMEOUT': {'type': int, 'default': 0},
'CHROME_HEADLESS': {'type': bool, 'default': True}, 'CHROME_HEADLESS': {'type': bool, 'default': True},
'CHROME_SANDBOX': {'type': bool, 'default': lambda c: not c['IN_DOCKER']}, 'CHROME_SANDBOX': {'type': bool, 'default': lambda c: not c['IN_DOCKER']},
'YOUTUBEDL_ARGS': {'type': list, 'default': lambda c: [ 'YOUTUBEDL_ARGS': {'type': list, 'default': lambda c: [
@ -981,6 +982,7 @@ def get_chrome_info(config: ConfigDict) -> ConfigValue:
'RESOLUTION': config['RESOLUTION'], 'RESOLUTION': config['RESOLUTION'],
'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'], 'CHECK_SSL_VALIDITY': config['CHECK_SSL_VALIDITY'],
'CHROME_BINARY': bin_path(config['CHROME_BINARY']), 'CHROME_BINARY': bin_path(config['CHROME_BINARY']),
'CHROME_TIMEOUT': config['CHROME_TIMEOUT'],
'CHROME_HEADLESS': config['CHROME_HEADLESS'], 'CHROME_HEADLESS': config['CHROME_HEADLESS'],
'CHROME_SANDBOX': config['CHROME_SANDBOX'], 'CHROME_SANDBOX': config['CHROME_SANDBOX'],
'CHROME_USER_AGENT': config['CHROME_USER_AGENT'], 'CHROME_USER_AGENT': config['CHROME_USER_AGENT'],

1
archivebox/config_stubs.py
View file

@ -74,6 +74,7 @@ class ConfigDict(BaseConfig, total=False):
CHROME_USER_AGENT: str CHROME_USER_AGENT: str
COOKIES_FILE: Union[str, Path, None] COOKIES_FILE: Union[str, Path, None]
CHROME_USER_DATA_DIR: Union[str, Path, None] CHROME_USER_DATA_DIR: Union[str, Path, None]
CHROME_TIMEOUT: int
CHROME_HEADLESS: bool CHROME_HEADLESS: bool
CHROME_SANDBOX: bool CHROME_SANDBOX: bool

2
archivebox/extractors/dom.py
View file

@ -39,7 +39,7 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
output: ArchiveOutput = 'output.html' output: ArchiveOutput = 'output.html'
output_path = out_dir / output output_path = out_dir / output
cmd = [ cmd = [
*chrome_args(TIMEOUT=timeout), *chrome_args(),
'--dump-dom', '--dump-dom',
link.url link.url
] ]

2
archivebox/extractors/pdf.py
View file

@ -37,7 +37,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
out_dir = out_dir or Path(link.link_dir) out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = 'output.pdf' output: ArchiveOutput = 'output.pdf'
cmd = [ cmd = [
*chrome_args(TIMEOUT=timeout), *chrome_args(),
'--print-to-pdf', '--print-to-pdf',
link.url, link.url,
] ]

2
archivebox/extractors/screenshot.py
View file

@ -37,7 +37,7 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
out_dir = out_dir or Path(link.link_dir) out_dir = out_dir or Path(link.link_dir)
output: ArchiveOutput = 'screenshot.png' output: ArchiveOutput = 'screenshot.png'
cmd = [ cmd = [
*chrome_args(TIMEOUT=timeout), *chrome_args(),
'--screenshot', '--screenshot',
link.url, link.url,
] ]

2
archivebox/extractors/singlefile.py
View file

@ -42,7 +42,7 @@ def save_singlefile(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
out_dir = out_dir or Path(link.link_dir) out_dir = out_dir or Path(link.link_dir)
output = "singlefile.html" output = "singlefile.html"
browser_args = chrome_args(TIMEOUT=0) browser_args = chrome_args(CHROME_TIMEOUT=0)
# SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli # SingleFile CLI Docs: https://github.com/gildas-lormeau/SingleFile/tree/master/cli
browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:])) browser_args = '--browser-args={}'.format(json.dumps(browser_args[1:]))

4
archivebox/util.py
View file

@ -260,8 +260,8 @@ def chrome_args(**options) -> List[str]:
if options['RESOLUTION']: if options['RESOLUTION']:
cmd_args += ('--window-size={}'.format(options['RESOLUTION']),) cmd_args += ('--window-size={}'.format(options['RESOLUTION']),)
#if options['TIMEOUT']: if options['CHROME_TIMEOUT']:
# cmd_args += ('--timeout={}'.format(options['TIMEOUT'] * 1000),) cmd_args += ('--timeout={}'.format(options['CHROME_TIMEOUT'] * 1000),)
if options['CHROME_USER_DATA_DIR']: if options['CHROME_USER_DATA_DIR']:
cmd_args.append('--user-data-dir={}'.format(options['CHROME_USER_DATA_DIR'])) cmd_args.append('--user-data-dir={}'.format(options['CHROME_USER_DATA_DIR']))