From 06e57990f7b12328a68471309ae460f320006354 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Tue, 14 Dec 2021 22:33:47 +0530 Subject: [PATCH] Allow multiple and nested configuration files --- test/test_options.py | 26 ------ test/test_utils.py | 10 +++ yt_dlp/options.py | 189 +++++++++++++++++-------------------------- yt_dlp/utils.py | 88 ++++++++++++++++++++ 4 files changed, 172 insertions(+), 141 deletions(-) delete mode 100644 test/test_options.py diff --git a/test/test_options.py b/test/test_options.py deleted file mode 100644 index 42d9183a96..0000000000 --- a/test/test_options.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding: utf-8 - -from __future__ import unicode_literals - -# Allow direct execution -import os -import sys -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from yt_dlp.options import _hide_login_info - - -class TestOptions(unittest.TestCase): - def test_hide_login_info(self): - self.assertEqual(_hide_login_info(['-u', 'foo', '-p', 'bar']), - ['-u', 'PRIVATE', '-p', 'PRIVATE']) - self.assertEqual(_hide_login_info(['-u']), ['-u']) - self.assertEqual(_hide_login_info(['-u', 'foo', '-u', 'bar']), - ['-u', 'PRIVATE', '-u', 'PRIVATE']) - self.assertEqual(_hide_login_info(['--username=foo']), - ['--username=PRIVATE']) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index 2e33308c75..1a9f71947b 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -23,6 +23,7 @@ from yt_dlp.utils import ( caesar, clean_html, clean_podcast_url, + Config, date_from_str, datetime_from_str, DateRange, @@ -1701,6 +1702,15 @@ Line 1 self.assertEqual(format_bytes(1024**7), '1.00ZiB') self.assertEqual(format_bytes(1024**8), '1.00YiB') + def test_hide_login_info(self): + self.assertEqual(Config.hide_login_info(['-u', 'foo', '-p', 'bar']), + ['-u', 'PRIVATE', '-p', 'PRIVATE']) + self.assertEqual(Config.hide_login_info(['-u']), ['-u']) + self.assertEqual(Config.hide_login_info(['-u', 'foo', '-u', 'bar']), + ['-u', 'PRIVATE', '-u', 'PRIVATE']) + self.assertEqual(Config.hide_login_info(['--username=foo']), + ['--username=PRIVATE']) + if __name__ == '__main__': unittest.main() diff --git a/yt_dlp/options.py b/yt_dlp/options.py index a96fb82a2b..51b8a28962 100644 --- a/yt_dlp/options.py +++ b/yt_dlp/options.py @@ -13,11 +13,11 @@ from .compat import ( compat_shlex_split, ) from .utils import ( + Config, expand_path, get_executable_path, OUTTMPL_TYPES, POSTPROCESS_WHEN, - preferredencoding, remove_end, write_string, ) @@ -35,39 +35,16 @@ from .postprocessor import ( from .postprocessor.modify_chapters import DEFAULT_SPONSORBLOCK_CHAPTER_TITLE -def _hide_login_info(opts): - PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']) - eqre = re.compile('^(?P' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$') +def parseOpts(overrideArguments=None, ignore_config_files='if_override'): + parser = create_parser() + root = Config(parser) - def _scrub_eq(o): - m = eqre.match(o) - if m: - return m.group('key') + '=PRIVATE' - else: - return o - - opts = list(map(_scrub_eq, opts)) - for idx, opt in enumerate(opts): - if opt in PRIVATE_OPTS and idx + 1 < len(opts): - opts[idx + 1] = 'PRIVATE' - return opts - - -def parseOpts(overrideArguments=None): - def _readOptions(filename_bytes, default=[]): - try: - optionf = open(filename_bytes) - except IOError: - return default # silently skip if file is not present - try: - # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56 - contents = optionf.read() - if sys.version_info < (3,): - contents = contents.decode(preferredencoding()) - res = compat_shlex_split(contents, comments=True) - finally: - optionf.close() - return res + if ignore_config_files == 'if_override': + ignore_config_files = overrideArguments is not None + if overrideArguments: + root.append_config(overrideArguments, label='Override') + else: + root.append_config(sys.argv[1:], label='Command-line') def _readUserConf(package_name, default=[]): # .config @@ -75,7 +52,7 @@ def parseOpts(overrideArguments=None): userConfFile = os.path.join(xdg_config_home, package_name, 'config') if not os.path.isfile(userConfFile): userConfFile = os.path.join(xdg_config_home, '%s.conf' % package_name) - userConf = _readOptions(userConfFile, default=None) + userConf = Config.read_file(userConfFile, default=None) if userConf is not None: return userConf, userConfFile @@ -83,24 +60,64 @@ def parseOpts(overrideArguments=None): appdata_dir = compat_getenv('appdata') if appdata_dir: userConfFile = os.path.join(appdata_dir, package_name, 'config') - userConf = _readOptions(userConfFile, default=None) + userConf = Config.read_file(userConfFile, default=None) if userConf is None: userConfFile += '.txt' - userConf = _readOptions(userConfFile, default=None) + userConf = Config.read_file(userConfFile, default=None) if userConf is not None: return userConf, userConfFile # home userConfFile = os.path.join(compat_expanduser('~'), '%s.conf' % package_name) - userConf = _readOptions(userConfFile, default=None) + userConf = Config.read_file(userConfFile, default=None) if userConf is None: userConfFile += '.txt' - userConf = _readOptions(userConfFile, default=None) + userConf = Config.read_file(userConfFile, default=None) if userConf is not None: return userConf, userConfFile return default, None + def add_config(label, path, user=False): + """ Adds config and returns whether to continue """ + if root.parse_args()[0].ignoreconfig: + return False + # Multiple package names can be given here + # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for + # the configuration file of any of these three packages + for package in ('yt-dlp',): + if user: + args, current_path = _readUserConf(package, default=None) + else: + current_path = os.path.join(path, '%s.conf' % package) + args = Config.read_file(current_path, default=None) + if args is not None: + root.append_config(args, current_path, label=label) + return True + return True + + def load_configs(): + yield not ignore_config_files + yield add_config('Portable', get_executable_path()) + yield add_config('Home', expand_path(root.parse_args()[0].paths.get('home', '')).strip()) + yield add_config('User', None, user=True) + yield add_config('System', '/etc') + + if all(load_configs()): + # If ignoreconfig is found inside the system configuration file, + # the user configuration is removed + if root.parse_args()[0].ignoreconfig: + user_conf = next((i for i, conf in enumerate(root.configs) if conf.label == 'User'), None) + if user_conf is not None: + root.configs.pop(user_conf) + + opts, args = root.parse_args() + if opts.verbose: + write_string(f'\n{root}'.replace('\n| ', '\n[debug] ')[1:] + '\n') + return parser, opts, args + + +def create_parser(): def _format_option_string(option): ''' ('-o', '--option') -> -o, --format METAVAR''' @@ -244,14 +261,20 @@ def parseOpts(overrideArguments=None): '--ignore-config', '--no-config', action='store_true', dest='ignoreconfig', help=( - 'Disable loading any configuration files except the one provided by --config-location. ' - 'When given inside a configuration file, no further configuration files are loaded. ' - 'Additionally, (for backward compatibility) if this option is found inside the ' - 'system configuration file, the user configuration is not loaded')) + 'Disable loading any further configuration files except the one provided by --config-locations. ' + 'For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded')) general.add_option( - '--config-location', - dest='config_location', metavar='PATH', - help='Location of the main configuration file; either the path to the config or its containing directory') + '--no-config-locations', + action='store_const', dest='config_locations', const=[], + help=( + 'Do not load any custom configuration files (default). When given inside a ' + 'configuration file, ignore all previous --config-locations defined in the current file')) + general.add_option( + '--config-locations', + dest='config_locations', metavar='PATH', action='append', + help=( + 'Location of the main configuration file; either the path to the config or its containing directory. ' + 'Can be used multiple times and inside other configuration files')) general.add_option( '--flat-playlist', action='store_const', dest='extract_flat', const='in_playlist', default=False, @@ -1634,75 +1657,11 @@ def parseOpts(overrideArguments=None): parser.add_option_group(sponsorblock) parser.add_option_group(extractor) - if overrideArguments is not None: - opts, args = parser.parse_args(overrideArguments) - if opts.verbose: - write_string('[debug] Override config: ' + repr(overrideArguments) + '\n') - else: - def compat_conf(conf): - if sys.version_info < (3,): - return [a.decode(preferredencoding(), 'replace') for a in conf] - return conf + return parser - configs = { - 'command-line': compat_conf(sys.argv[1:]), - 'custom': [], 'home': [], 'portable': [], 'user': [], 'system': []} - paths = {'command-line': False} - def read_options(name, path, user=False): - ''' loads config files and returns ignoreconfig ''' - # Multiple package names can be given here - # Eg: ('yt-dlp', 'youtube-dlc', 'youtube-dl') will look for - # the configuration file of any of these three packages - for package in ('yt-dlp',): - if user: - config, current_path = _readUserConf(package, default=None) - else: - current_path = os.path.join(path, '%s.conf' % package) - config = _readOptions(current_path, default=None) - if config is not None: - current_path = os.path.realpath(current_path) - if current_path in paths.values(): - return False - configs[name], paths[name] = config, current_path - return parser.parse_args(config)[0].ignoreconfig - return False - - def get_configs(): - opts, _ = parser.parse_args(configs['command-line']) - if opts.config_location is not None: - location = compat_expanduser(opts.config_location) - if os.path.isdir(location): - location = os.path.join(location, 'yt-dlp.conf') - if not os.path.exists(location): - parser.error('config-location %s does not exist.' % location) - config = _readOptions(location, default=None) - if config: - configs['custom'], paths['custom'] = config, location - - if opts.ignoreconfig: - return - if parser.parse_args(configs['custom'])[0].ignoreconfig: - return - if read_options('portable', get_executable_path()): - return - opts, _ = parser.parse_args(configs['portable'] + configs['custom'] + configs['command-line']) - if read_options('home', expand_path(opts.paths.get('home', '')).strip()): - return - if read_options('system', '/etc'): - return - if read_options('user', None, user=True): - configs['system'], paths['system'] = [], None - - get_configs() - argv = configs['system'] + configs['user'] + configs['home'] + configs['portable'] + configs['custom'] + configs['command-line'] - opts, args = parser.parse_args(argv) - if opts.verbose: - for label in ('Command-line', 'Custom', 'Portable', 'Home', 'User', 'System'): - key = label.lower() - if paths.get(key): - write_string(f'[debug] {label} config file: {paths[key]}\n') - if paths.get(key) is not None: - write_string(f'[debug] {label} config: {_hide_login_info(configs[key])!r}\n') - - return parser, opts, args +def _hide_login_info(opts): + write_string( + 'DeprecationWarning: "yt_dlp.options._hide_login_info" is deprecated and may be removed in a future version. ' + 'Use "yt_dlp.utils.Config.hide_login_info" instead\n') + return Config.hide_login_info(opts) diff --git a/yt_dlp/utils.py b/yt_dlp/utils.py index 1fd85de8eb..c1295c4b2e 100644 --- a/yt_dlp/utils.py +++ b/yt_dlp/utils.py @@ -58,6 +58,7 @@ from .compat import ( compat_kwargs, compat_os_name, compat_parse_qs, + compat_shlex_split, compat_shlex_quote, compat_str, compat_struct_pack, @@ -5100,3 +5101,90 @@ def join_nonempty(*values, delim='-', from_dict=None): if from_dict is not None: values = map(from_dict.get, values) return delim.join(map(str, filter(None, values))) + + +class Config: + own_args = None + filename = None + __initialized = False + + def __init__(self, parser, label=None): + self._parser, self.label = parser, label + self._loaded_paths, self.configs = set(), [] + + def init(self, args=None, filename=None): + assert not self.__initialized + if filename: + location = os.path.realpath(filename) + if location in self._loaded_paths: + return False + self._loaded_paths.add(location) + + self.__initialized = True + self.own_args, self.filename = args, filename + for location in self._parser.parse_args(args)[0].config_locations or []: + location = compat_expanduser(location) + if os.path.isdir(location): + location = os.path.join(location, 'yt-dlp.conf') + if not os.path.exists(location): + self._parser.error(f'config location {location} does not exist') + self.append_config(self.read_file(location), location) + return True + + def __str__(self): + label = join_nonempty( + self.label, 'config', f'"{self.filename}"' if self.filename else '', + delim=' ') + return join_nonempty( + self.own_args is not None and f'{label[0].upper()}{label[1:]}: {self.hide_login_info(self.own_args)}', + *(f'\n{c}'.replace('\n', '\n| ')[1:] for c in self.configs), + delim='\n') + + @staticmethod + def read_file(filename, default=[]): + try: + optionf = open(filename) + except IOError: + return default # silently skip if file is not present + try: + # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56 + contents = optionf.read() + if sys.version_info < (3,): + contents = contents.decode(preferredencoding()) + res = compat_shlex_split(contents, comments=True) + finally: + optionf.close() + return res + + @staticmethod + def hide_login_info(opts): + PRIVATE_OPTS = set(['-p', '--password', '-u', '--username', '--video-password', '--ap-password', '--ap-username']) + eqre = re.compile('^(?P' + ('|'.join(re.escape(po) for po in PRIVATE_OPTS)) + ')=.+$') + + def _scrub_eq(o): + m = eqre.match(o) + if m: + return m.group('key') + '=PRIVATE' + else: + return o + + opts = list(map(_scrub_eq, opts)) + for idx, opt in enumerate(opts): + if opt in PRIVATE_OPTS and idx + 1 < len(opts): + opts[idx + 1] = 'PRIVATE' + return opts + + def append_config(self, *args, label=None): + config = type(self)(self._parser, label) + config._loaded_paths = self._loaded_paths + if config.init(*args): + self.configs.append(config) + + @property + def all_args(self): + for config in reversed(self.configs): + yield from config.all_args + yield from self.own_args or [] + + def parse_args(self): + return self._parser.parse_args(list(self.all_args))