[cleanup] Refactor some code

yt_dlp/YoutubeDL.py

@@ -126,6 +126,7 @@ from .extractor import (
 )
 from .extractor.openload import PhantomJSwrapper
 from .downloader import (
+    FFmpegFD,
     get_suitable_downloader,
     shorten_protocol_name
 )
@@ -2690,20 +2691,15 @@ class YoutubeDL(object):
                     info_dict['__real_download'] = False
 
                     _protocols = set(determine_protocol(f) for f in requested_formats)
-                    if len(_protocols) == 1:
+                    if len(_protocols) == 1:  # All requested formats have same protocol
                         info_dict['protocol'] = _protocols.pop()
-                    directly_mergable = (
+                    directly_mergable = FFmpegFD.can_merge_formats(info_dict)
-                        'no-direct-merge' not in self.params.get('compat_opts', [])
+                    if dl_filename is not None:
-                        and info_dict.get('protocol') is not None  # All requested formats have same protocol
+                        pass
-                        and not self.params.get('allow_unplayable_formats')
+                    elif (directly_mergable and get_suitable_downloader(info_dict, self.params) == FFmpegFD):
-                        and get_suitable_downloader(info_dict, self.params).__name__ == 'FFmpegFD')
+                        info_dict['url'] = '\n'.join(f['url'] for f in requested_formats)
-                    if directly_mergable:
+                        success, real_download = self.dl(temp_filename, info_dict)
-                        info_dict['url'] = requested_formats[0]['url']
+                        info_dict['__real_download'] = real_download
-                        # Treat it as a single download
-                        dl_filename = existing_file(full_filename, temp_filename)
-                        if dl_filename is None:
-                            success, real_download = self.dl(temp_filename, info_dict)
-                            info_dict['__real_download'] = real_download
                     else:
                         downloaded = []
                         merger = FFmpegMergerPP(self)
@@ -2717,28 +2713,25 @@ class YoutubeDL(object):
                                 'You have requested merging of multiple formats but ffmpeg is not installed. '
                                 'The formats won\'t be merged.')
 
-                        if dl_filename is None:
+                        for f in requested_formats:
-                            for f in requested_formats:
+                            new_info = dict(info_dict)
-                                new_info = dict(info_dict)
+                            del new_info['requested_formats']
-                                del new_info['requested_formats']
+                            new_info.update(f)
-                                new_info.update(f)
+                            fname = prepend_extension(temp_filename, 'f%s' % f['format_id'], new_info['ext'])
-                                fname = prepend_extension(
+                            if not self._ensure_dir_exists(fname):
-                                    self.prepare_filename(new_info, 'temp'),
+                                return
-                                    'f%s' % f['format_id'], new_info['ext'])
+                            downloaded.append(fname)
-                                if not self._ensure_dir_exists(fname):
+                            partial_success, real_download = self.dl(fname, new_info)
-                                    return
+                            info_dict['__real_download'] = info_dict['__real_download'] or real_download
-                                downloaded.append(fname)
+                            success = success and partial_success
-                                partial_success, real_download = self.dl(fname, new_info)
+                        if merger.available and not self.params.get('allow_unplayable_formats'):
-                                info_dict['__real_download'] = info_dict['__real_download'] or real_download
+                            info_dict['__postprocessors'].append(merger)
-                                success = success and partial_success
+                            info_dict['__files_to_merge'] = downloaded
-                            if merger.available and not self.params.get('allow_unplayable_formats'):
+                            # Even if there were no downloads, it is being merged only now
-                                info_dict['__postprocessors'].append(merger)
+                            info_dict['__real_download'] = True
-                                info_dict['__files_to_merge'] = downloaded
+                        else:
-                                # Even if there were no downloads, it is being merged only now
+                            for file in downloaded:
-                                info_dict['__real_download'] = True
+                                files_to_move[file] = None
-                            else:
-                                for file in downloaded:
-                                    files_to_move[file] = None
                 else:
                     # Just a single file
                     dl_filename = existing_file(full_filename, temp_filename)

yt_dlp/downloader/__init__.py

@@ -3,17 +3,19 @@ from __future__ import unicode_literals
 from ..compat import compat_str
 from ..utils import (
     determine_protocol,
+    NO_DEFAULT
 )
 
 
-def _get_real_downloader(info_dict, protocol=None, *args, **kwargs):
+def get_suitable_downloader(info_dict, params={}, default=NO_DEFAULT, protocol=None):
+    info_dict['protocol'] = determine_protocol(info_dict)
     info_copy = info_dict.copy()
     if protocol:
         info_copy['protocol'] = protocol
-    return get_suitable_downloader(info_copy, *args, **kwargs)
+    return _get_suitable_downloader(info_copy, params, default)
 
 
-# Some of these require _get_real_downloader
+# Some of these require get_suitable_downloader
 from .common import FileDownloader
 from .dash import DashSegmentsFD
 from .f4m import F4mFD
@@ -69,14 +71,15 @@ def shorten_protocol_name(proto, simplify=False):
     return short_protocol_names.get(proto, proto)
 
 
-def get_suitable_downloader(info_dict, params={}, default=HttpFD):
+def _get_suitable_downloader(info_dict, params, default):
     """Get the downloader class that can handle the info dict."""
-    protocol = determine_protocol(info_dict)
+    if default is NO_DEFAULT:
-    info_dict['protocol'] = protocol
+        default = HttpFD
 
     # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
     #     return FFmpegFD
 
+    protocol = info_dict['protocol']
     downloaders = params.get('external_downloader')
     external_downloader = (
         downloaders if isinstance(downloaders, compat_str) or downloaders is None
@@ -94,7 +97,7 @@ def get_suitable_downloader(info_dict, params={}, default=HttpFD):
             return FFmpegFD
         elif external_downloader == 'native':
             return HlsFD
-        elif _get_real_downloader(info_dict, 'm3u8_frag_urls', params, None):
+        elif get_suitable_downloader(info_dict, params, None, protocol='m3u8_frag_urls'):
             return HlsFD
         elif params.get('hls_prefer_native') is True:
             return HlsFD

yt_dlp/downloader/dash.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 
-from ..downloader import _get_real_downloader
+from ..downloader import get_suitable_downloader
 from .fragment import FragmentFD
 
 from ..utils import urljoin
@@ -15,11 +15,14 @@ class DashSegmentsFD(FragmentFD):
     FD_NAME = 'dashsegments'
 
     def real_download(self, filename, info_dict):
+        if info_dict.get('is_live'):
+            self.report_error('Live DASH videos are not supported')
+
         fragment_base_url = info_dict.get('fragment_base_url')
         fragments = info_dict['fragments'][:1] if self.params.get(
             'test', False) else info_dict['fragments']
 
-        real_downloader = _get_real_downloader(info_dict, 'dash_frag_urls', self.params, None)
+        real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='dash_frag_urls')
 
         ctx = {
             'filename': filename,
@@ -54,9 +57,6 @@ class DashSegmentsFD(FragmentFD):
             info_copy = info_dict.copy()
             info_copy['fragments'] = fragments_to_download
             fd = real_downloader(self.ydl, self.params)
-            # TODO: Make progress updates work without hooking twice
-            # for ph in self._progress_hooks:
-            #     fd.add_progress_hook(ph)
             return fd.real_download(filename, info_copy)
 
         return self.download_and_append_fragments(ctx, fragments_to_download, info_dict)

yt_dlp/downloader/external.py

@@ -345,12 +345,22 @@ class FFmpegFD(ExternalFD):
     @classmethod
     def available(cls, path=None):
         # TODO: Fix path for ffmpeg
+        # Fixme: This may be wrong when --ffmpeg-location is used
         return FFmpegPostProcessor().available
 
     def on_process_started(self, proc, stdin):
         """ Override this in subclasses  """
         pass
 
+    @classmethod
+    def can_merge_formats(cls, info_dict, params={}):
+        return (
+            info_dict.get('requested_formats')
+            and info_dict.get('protocol')
+            and not params.get('allow_unplayable_formats')
+            and 'no-direct-merge' not in params.get('compat_opts', [])
+            and cls.can_download(info_dict))
+
     def _call_downloader(self, tmpfilename, info_dict):
         urls = [f['url'] for f in info_dict.get('requested_formats', [])] or [info_dict['url']]
         ffpp = FFmpegPostProcessor(downloader=self)

yt_dlp/downloader/hls.py

@@ -4,7 +4,7 @@ import re
 import io
 import binascii
 
-from ..downloader import _get_real_downloader
+from ..downloader import get_suitable_downloader
 from .fragment import FragmentFD, can_decrypt_frag
 from .external import FFmpegFD
 
@@ -80,16 +80,13 @@ class HlsFD(FragmentFD):
             fd = FFmpegFD(self.ydl, self.params)
             self.report_warning(
                 '%s detected unsupported features; extraction will be delegated to %s' % (self.FD_NAME, fd.get_basename()))
-            # TODO: Make progress updates work without hooking twice
-            # for ph in self._progress_hooks:
-            #     fd.add_progress_hook(ph)
             return fd.real_download(filename, info_dict)
 
         is_webvtt = info_dict['ext'] == 'vtt'
         if is_webvtt:
             real_downloader = None  # Packing the fragments is not currently supported for external downloader
         else:
-            real_downloader = _get_real_downloader(info_dict, 'm3u8_frag_urls', self.params, None)
+            real_downloader = get_suitable_downloader(info_dict, self.params, None, protocol='m3u8_frag_urls')
         if real_downloader and not real_downloader.supports_manifest(s):
             real_downloader = None
         if real_downloader:

yt_dlp/downloader/niconico.py

@@ -4,7 +4,7 @@ from __future__ import unicode_literals
 import threading
 
 from .common import FileDownloader
-from ..downloader import _get_real_downloader
+from ..downloader import get_suitable_downloader
 from ..extractor.niconico import NiconicoIE
 from ..compat import compat_urllib_request
 
@@ -20,7 +20,7 @@ class NiconicoDmcFD(FileDownloader):
         ie = NiconicoIE(self.ydl)
         info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
 
-        fd = _get_real_downloader(info_dict, params=self.params)(self.ydl, self.params)
+        fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
 
         success = download_complete = False
         timer = [None]

yt_dlp/extractor/common.py

@@ -1298,7 +1298,7 @@ class InfoExtractor(object):
         # JSON-LD may be malformed and thus `fatal` should be respected.
         # At the same time `default` may be passed that assumes `fatal=False`
         # for _search_regex. Let's simulate the same behavior here as well.
-        fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
+        fatal = kwargs.get('fatal', True) if default is NO_DEFAULT else False
         json_ld = []
         for mobj in json_ld_list:
             json_ld_item = self._parse_json(
@@ -1522,7 +1522,7 @@ class InfoExtractor(object):
             'size': {'type': 'combined', 'same_limit': True, 'field': ('filesize', 'fs_approx')},
             'ext': {'type': 'combined', 'field': ('vext', 'aext')},
             'res': {'type': 'multiple', 'field': ('height', 'width'),
-            'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
+                    'function': lambda it: (lambda l: min(l) if l else 0)(tuple(filter(None, it)))},
 
             # Most of these exist only for compatibility reasons
             'dimension': {'type': 'alias', 'field': 'res'},

yt_dlp/options.py

@@ -706,9 +706,8 @@ def parseOpts(overrideArguments=None):
         callback_kwargs={
             'allowed_keys': 'http|ftp|m3u8|dash|rtsp|rtmp|mms',
             'default_key': 'default',
-            'process': lambda x: x.strip()
+            'process': str.strip
-        },
+        }, help=(
-        help=(
             'Name or path of the external downloader to use (optionally) prefixed by '
             'the protocols (http, ftp, m3u8, dash, rstp, rtmp, mms) to use it for. '
             'Currently supports native, %s (Recommended: aria2c). '
@@ -724,8 +723,7 @@ def parseOpts(overrideArguments=None):
             'allowed_keys': '|'.join(list_external_downloaders()),
             'default_key': 'default',
             'process': compat_shlex_split
-        },
+        }, help=(
-        help=(
             'Give these arguments to the external downloader. '
             'Specify the downloader name and the arguments separated by a colon ":". '
             'You can use this option multiple times to give different arguments to different downloaders '
@@ -944,8 +942,7 @@ def parseOpts(overrideArguments=None):
         callback_kwargs={
             'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
             'default_key': 'default'
-        },
+        }, help='Output filename template; see "OUTPUT TEMPLATE" for details')
-        help='Output filename template; see "OUTPUT TEMPLATE" for details')
     filesystem.add_option(
         '--output-na-placeholder',
         dest='outtmpl_na_placeholder', metavar='TEXT', default='NA',
@@ -1191,8 +1188,7 @@ def parseOpts(overrideArguments=None):
             'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
             'process': compat_shlex_split,
             'multiple_keys': False
-        },
+        }, help=(
-        help=(
             'Give these arguments to the postprocessors. '
             'Specify the postprocessor/executable name and the arguments separated by a colon ":" '
             'to give the argument to the specified postprocessor/executable. Supported PP are: '
@@ -1385,8 +1381,7 @@ def parseOpts(overrideArguments=None):
             'multiple_keys': False,
             'process': lambda val: dict(
                 _extractor_arg_parser(*arg.split('=', 1)) for arg in val.split(';'))
-        },
+        }, help=(
-        help=(
             'Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. '
             'You can use this option multiple times to give arguments for different extractors'))
     extractor.add_option(

yt_dlp/utils.py

@@ -4544,7 +4544,7 @@ def parse_codecs(codecs_str):
     if not codecs_str:
         return {}
     split_codecs = list(filter(None, map(
-        lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
+        str.strip, codecs_str.strip().strip(',').split(','))))
     vcodec, acodec = None, None
     for full_codec in split_codecs:
         codec = full_codec.split('.')[0]
@@ -6246,7 +6246,7 @@ def traverse_obj(
     # TODO: Write tests
     '''
     if not casesense:
-        _lower = lambda k: k.lower() if isinstance(k, str) else k
+        _lower = lambda k: (k.lower() if isinstance(k, str) else k)
         path_list = (map(_lower, variadic(path)) for path in path_list)
 
     def _traverse_obj(obj, path, _current_depth=0):