Mirrors/yt-dlp
2
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-10 15:14:57 +00:00
Code Issues Projects Releases Packages Wiki Activity

Parse metadata from multiple fields

Browse source 
Closes #196
This commit is contained in:
pukkandan 2021-03-25 03:32:15 +05:30
parent 3700c7ef10
commit 143db31d48
No known key found for this signature in database
GPG key ID: 0F00D95A001F4698
5 changed files with 143 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
README.md
View file

@ -670,18 +670,24 @@ Then simply run `make`. You can also run `make yt-dlp` instead to compile only t
--add-metadata Write metadata to the video file --add-metadata Write metadata to the video file
--no-add-metadata Do not write metadata (default) --no-add-metadata Do not write metadata (default)
--parse-metadata FIELD:FORMAT Parse additional metadata like title/artist --parse-metadata FIELD:FORMAT Parse additional metadata like title/artist
from other fields. Give field name to from other fields. Give a template or field
extract data from, and format of the field name to extract data from and the format to
seperated by a ":". Either regular interpret it as, seperated by a ":". Either
expression with named capture groups or a regular expression with named capture
similar syntax to the output template can groups or a similar syntax to the output
also be used. The parsed parameters replace template can be used for the FORMAT.
any existing values and can be use in Similarly, the syntax for output template
output template. This option can be used can be used for FIELD to parse the data
multiple times. Example: --parse-metadata from multiple fields. The parsed parameters
"title:%(artist)s - %(title)s" matches a replace any existing values and can be used
title like "Coldplay - Paradise". Example in output templates. This option can be
(regex): --parse-metadata used multiple times. Example: --parse-
metadata "title:%(artist)s - %(title)s"
matches a title like "Coldplay - Paradise".
Example: --parse-metadata "%(series)s
%(episode_number)s:%(title)s" sets the
title using series and episode number.
Example (regex): --parse-metadata
"description:Artist - (?P<artist>.+?)" "description:Artist - (?P<artist>.+?)"
--xattrs Write metadata to the video file's xattrs --xattrs Write metadata to the video file's xattrs
(using dublin core and xdg standards) (using dublin core and xdg standards)

46
yt_dlp/YoutubeDL.py
View file

@ -67,6 +67,7 @@ from .utils import (
float_or_none, float_or_none,
format_bytes, format_bytes,
format_field, format_field,
FORMAT_RE,
formatSeconds, formatSeconds,
GeoRestrictedError, GeoRestrictedError,
int_or_none, int_or_none,
@ -772,20 +773,26 @@ class YoutubeDL(object):
'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.') 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
return outtmpl_dict return outtmpl_dict
def _prepare_filename(self, info_dict, tmpl_type='default'): def prepare_outtmpl(self, outtmpl, info_dict, sanitize=None):
try: """ Make the template and info_dict suitable for substitution (outtmpl % info_dict)"""
template_dict = dict(info_dict) template_dict = dict(info_dict)
# duration_string
template_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs template_dict['duration_string'] = ( # %(duration>%H-%M-%S)s is wrong if duration > 24hrs
formatSeconds(info_dict['duration'], '-') formatSeconds(info_dict['duration'], '-')
if info_dict.get('duration', None) is not None if info_dict.get('duration', None) is not None
else None) else None)
# epoch
template_dict['epoch'] = int(time.time()) template_dict['epoch'] = int(time.time())
# autonumber
autonumber_size = self.params.get('autonumber_size') autonumber_size = self.params.get('autonumber_size')
if autonumber_size is None: if autonumber_size is None:
autonumber_size = 5 autonumber_size = 5
template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads template_dict['autonumber'] = self.params.get('autonumber_start', 1) - 1 + self._num_downloads
# resolution if not defined
if template_dict.get('resolution') is None: if template_dict.get('resolution') is None:
if template_dict.get('width') and template_dict.get('height'): if template_dict.get('width') and template_dict.get('height'):
template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height']) template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
@ -794,19 +801,14 @@ class YoutubeDL(object):
elif template_dict.get('width'): elif template_dict.get('width'):
template_dict['resolution'] = '%dx?' % template_dict['width'] template_dict['resolution'] = '%dx?' % template_dict['width']
sanitize = lambda k, v: sanitize_filename( if sanitize is None:
compat_str(v), sanitize = lambda k, v: v
restricted=self.params.get('restrictfilenames'),
is_id=(k == 'id' or k.endswith('_id')))
template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v)) template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
for k, v in template_dict.items() for k, v in template_dict.items()
if v is not None and not isinstance(v, (list, tuple, dict))) if v is not None and not isinstance(v, (list, tuple, dict)))
na = self.params.get('outtmpl_na_placeholder', 'NA') na = self.params.get('outtmpl_na_placeholder', 'NA')
template_dict = collections.defaultdict(lambda: na, template_dict) template_dict = collections.defaultdict(lambda: na, template_dict)
outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
force_ext = OUTTMPL_TYPES.get(tmpl_type)
# For fields playlist_index and autonumber convert all occurrences # For fields playlist_index and autonumber convert all occurrences
# of %(field)s to %(field)0Nd for backward compatibility # of %(field)s to %(field)0Nd for backward compatibility
field_size_compat_map = { field_size_compat_map = {
@ -821,20 +823,6 @@ class YoutubeDL(object):
r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')], r'%%(\1)0%dd' % field_size_compat_map[mobj.group('field')],
outtmpl) outtmpl)
# As of [1] format syntax is:
# %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
# 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
FORMAT_RE = r'''(?x)
(?<!%)
%
\({0}\) # mapping key
(?:[#0\-+ ]+)? # conversion flags (optional)
(?:\d+)? # minimum field width (optional)
(?:\.\d+)? # precision (optional)
[hlL]? # length modifier (optional)
(?P<type>[diouxXeEfFgGcrs%]) # conversion type
'''
numeric_fields = list(self._NUMERIC_FIELDS) numeric_fields = list(self._NUMERIC_FIELDS)
# Format date # Format date
@ -862,6 +850,17 @@ class YoutubeDL(object):
FORMAT_RE.format(re.escape(numeric_field)), FORMAT_RE.format(re.escape(numeric_field)),
r'%({0})s'.format(numeric_field), outtmpl) r'%({0})s'.format(numeric_field), outtmpl)
return outtmpl, template_dict
def _prepare_filename(self, info_dict, tmpl_type='default'):
try:
sanitize = lambda k, v: sanitize_filename(
compat_str(v),
restricted=self.params.get('restrictfilenames'),
is_id=(k == 'id' or k.endswith('_id')))
outtmpl = self.outtmpl_dict.get(tmpl_type, self.outtmpl_dict['default'])
outtmpl, template_dict = self.prepare_outtmpl(outtmpl, info_dict, sanitize)
# expand_path translates '%%' into '%' and '$$' into '$' # expand_path translates '%%' into '%' and '$$' into '$'
# correspondingly that is not what we want since we need to keep # correspondingly that is not what we want since we need to keep
# '%%' intact for template dict substitution step. Working around # '%%' intact for template dict substitution step. Working around
@ -875,6 +874,7 @@ class YoutubeDL(object):
# title "Hello $PATH", we don't want `$PATH` to be expanded. # title "Hello $PATH", we don't want `$PATH` to be expanded.
filename = expand_path(outtmpl).replace(sep, '') % template_dict filename = expand_path(outtmpl).replace(sep, '') % template_dict
force_ext = OUTTMPL_TYPES.get(tmpl_type)
if force_ext is not None: if force_ext is not None:
filename = replace_extension(filename, force_ext, template_dict.get('ext')) filename = replace_extension(filename, force_ext, template_dict.get('ext'))

11
yt_dlp/options.py
View file

@ -1147,13 +1147,18 @@ def parseOpts(overrideArguments=None):
metavar='FIELD:FORMAT', dest='metafromfield', action='append', metavar='FIELD:FORMAT', dest='metafromfield', action='append',
help=( help=(
'Parse additional metadata like title/artist from other fields. ' 'Parse additional metadata like title/artist from other fields. '
'Give field name to extract data from, and format of the field seperated by a ":". ' 'Give a template or field name to extract data from and the '
'format to interpret it as, seperated by a ":". '
'Either regular expression with named capture groups or a ' 'Either regular expression with named capture groups or a '
'similar syntax to the output template can also be used. ' 'similar syntax to the output template can be used for the FORMAT. '
'The parsed parameters replace any existing values and can be use in output template. ' 'Similarly, the syntax for output template can be used for FIELD '
'to parse the data from multiple fields. '
'The parsed parameters replace any existing values and can be used in output templates. '
'This option can be used multiple times. ' 'This option can be used multiple times. '
'Example: --parse-metadata "title:%(artist)s - %(title)s" matches a title like ' 'Example: --parse-metadata "title:%(artist)s - %(title)s" matches a title like '
'"Coldplay - Paradise". ' '"Coldplay - Paradise". '
'Example: --parse-metadata "%(series)s %(episode_number)s:%(title)s" '
'sets the title using series and episode number. '
'Example (regex): --parse-metadata "description:Artist - (?P<artist>.+?)"')) 'Example (regex): --parse-metadata "description:Artist - (?P<artist>.+?)"'))
postproc.add_option( postproc.add_option(
'--xattrs', '--xattrs',

39
yt_dlp/postprocessor/metadatafromfield.py
View file

@ -8,7 +8,7 @@ from ..utils import str_or_none
class MetadataFromFieldPP(PostProcessor): class MetadataFromFieldPP(PostProcessor):
regex = r'(?P<field>\w+):(?P<format>.+)$' regex = r'(?P<in>.+):(?P<out>.+)$'
def __init__(self, downloader, formats): def __init__(self, downloader, formats):
PostProcessor.__init__(self, downloader) PostProcessor.__init__(self, downloader)
@ -19,11 +19,20 @@ class MetadataFromFieldPP(PostProcessor):
match = re.match(self.regex, f) match = re.match(self.regex, f)
assert match is not None assert match is not None
self._data.append({ self._data.append({
'field': match.group('field'), 'in': match.group('in'),
'format': match.group('format'), 'out': match.group('out'),
'regex': self.format_to_regex(match.group('format'))}) 'tmpl': self.field_to_template(match.group('in')),
'regex': self.format_to_regex(match.group('out')),
})
def format_to_regex(self, fmt): @staticmethod
def field_to_template(tmpl):
if re.match(r'\w+$', tmpl):
return '%%(%s)s' % tmpl
return tmpl
@staticmethod
def format_to_regex(fmt):
r""" r"""
Converts a string like Converts a string like
'%(title)s - %(artist)s' '%(title)s - %(artist)s'
@ -37,7 +46,7 @@ class MetadataFromFieldPP(PostProcessor):
# replace %(..)s with regex group and escape other string parts # replace %(..)s with regex group and escape other string parts
for match in re.finditer(r'%\((\w+)\)s', fmt): for match in re.finditer(r'%\((\w+)\)s', fmt):
regex += re.escape(fmt[lastpos:match.start()]) regex += re.escape(fmt[lastpos:match.start()])
regex += r'(?P<' + match.group(1) + r'>[^\r\n]+)' regex += r'(?P<%s>[^\r\n]+)' % match.group(1)
lastpos = match.end() lastpos = match.end()
if lastpos < len(fmt): if lastpos < len(fmt):
regex += re.escape(fmt[lastpos:]) regex += re.escape(fmt[lastpos:])
@ -45,22 +54,16 @@ class MetadataFromFieldPP(PostProcessor):
def run(self, info): def run(self, info):
for dictn in self._data: for dictn in self._data:
field, regex = dictn['field'], dictn['regex'] tmpl, info_copy = self._downloader.prepare_outtmpl(dictn['tmpl'], info)
if field not in info: data_to_parse = tmpl % info_copy
self.report_warning('Video doesnot have a %s' % field) self.write_debug('Searching for r"%s" in %s' % (dictn['regex'], tmpl))
continue match = re.search(dictn['regex'], data_to_parse)
data_to_parse = str_or_none(info[field])
if data_to_parse is None:
self.report_warning('Field %s cannot be parsed' % field)
continue
self.write_debug('Searching for r"%s" in %s' % (regex, field))
match = re.search(regex, data_to_parse)
if match is None: if match is None:
self.report_warning('Could not interpret video %s as "%s"' % (field, dictn['format'])) self.report_warning('Could not interpret video %s as "%s"' % (dictn['in'], dictn['out']))
continue continue
for attribute, value in match.groupdict().items(): for attribute, value in match.groupdict().items():
info[attribute] = value info[attribute] = value
self.to_screen('parsed %s from %s: %s' % (attribute, field, value if value is not None else 'NA')) self.to_screen('parsed %s from "%s": %s' % (attribute, dictn['in'], value if value is not None else 'NA'))
return [], info return [], info

14
yt_dlp/utils.py
View file

@ -4205,6 +4205,20 @@ OUTTMPL_TYPES = {
'pl_infojson': 'info.json', 'pl_infojson': 'info.json',
} }
# As of [1] format syntax is:
# %[mapping_key][conversion_flags][minimum_width][.precision][length_modifier]type
# 1. https://docs.python.org/2/library/stdtypes.html#string-formatting
FORMAT_RE = r'''(?x)
(?<!%)
%
\({0}\) # mapping key
(?:[#0\-+ ]+)? # conversion flags (optional)
(?:\d+)? # minimum field width (optional)
(?:\.\d+)? # precision (optional)
[hlL]? # length modifier (optional)
(?P<type>[diouxXeEfFgGcrs%]) # conversion type
'''
def limit_length(s, length): def limit_length(s, length):
""" Add ellipses to overly long strings """ """ Add ellipses to overly long strings """