Mirrors/yt-dlp
2
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-12-15 15:52:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Fix YoutubeChannelIE

Browse source 
- urls with query parameters now match
- fixes regex for identifying videos
- fixes pagination
This commit is contained in:
ispedals 2013-04-19 18:05:35 -04:00
parent dce9027045
commit 5a8d13199c
1 changed files with 43 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
youtube_dl/InfoExtractors.py
View file

@ -1823,15 +1823,23 @@ class YoutubePlaylistIE(InfoExtractor):
class YoutubeChannelIE(InfoExtractor): class YoutubeChannelIE(InfoExtractor):
"""Information Extractor for YouTube channels.""" """Information Extractor for YouTube channels."""
_VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$" _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
_TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en' _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
_MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}" _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
_MORE_PAGES_URL = 'http://www.youtube.com/channel_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
IE_NAME = u'youtube:channel' IE_NAME = u'youtube:channel'
def report_download_page(self, channel_id, pagenum): def report_download_page(self, channel_id, pagenum):
"""Report attempt to download channel page with given number.""" """Report attempt to download channel page with given number."""
self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum)) self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
def extract_videos_from_page(self, page):
ids_in_page = []
for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
if mobj.group(1) not in ids_in_page:
ids_in_page.append(mobj.group(1))
return ids_in_page
def _real_extract(self, url): def _real_extract(self, url):
# Extract channel id # Extract channel id
mobj = re.match(self._VALID_URL, url) mobj = re.match(self._VALID_URL, url)
@ -1839,31 +1847,45 @@ class YoutubeChannelIE(InfoExtractor):
self._downloader.report_error(u'invalid url: %s' % url) self._downloader.report_error(u'invalid url: %s' % url)
return return
# Download channel pages # Download channel page
channel_id = mobj.group(1) channel_id = mobj.group(1)
video_ids = [] video_ids = []
pagenum = 1 pagenum = 1
while True: self.report_download_page(channel_id, pagenum)
self.report_download_page(channel_id, pagenum) url = self._TEMPLATE_URL % (channel_id, pagenum)
url = self._TEMPLATE_URL % (channel_id, pagenum) request = compat_urllib_request.Request(url)
request = compat_urllib_request.Request(url) try:
try: page = compat_urllib_request.urlopen(request).read().decode('utf8')
page = compat_urllib_request.urlopen(request).read().decode('utf8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err)) return
return
# Extract video identifiers # Extract video identifiers
ids_in_page = [] ids_in_page = self.extract_videos_from_page(page)
for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page): video_ids.extend(ids_in_page)
if mobj.group(1) not in ids_in_page:
ids_in_page.append(mobj.group(1))
video_ids.extend(ids_in_page)
if self._MORE_PAGES_INDICATOR not in page: # Download any subsequent channel pages using the json-based channel_ajax query
break if self._MORE_PAGES_INDICATOR in page:
pagenum = pagenum + 1 while True:
pagenum = pagenum + 1
self.report_download_page(channel_id, pagenum)
url = self._MORE_PAGES_URL % (pagenum, channel_id)
request = compat_urllib_request.Request(url)
try:
page = compat_urllib_request.urlopen(request).read().decode('utf8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.report_error(u'unable to download webpage: %s' % compat_str(err))
return
page = json.loads(page)
ids_in_page = self.extract_videos_from_page(page['content_html'])
video_ids.extend(ids_in_page)
if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
break
self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))