From 723e04d0be85fbdbbbda52512f322331d8fda760 Mon Sep 17 00:00:00 2001
From: Adam <oneplusme@users.noreply.github.com>
Date: Fri, 29 Aug 2014 22:32:03 +0100
Subject: [PATCH 1/4] Add login support to Crunchyroll extractor
---
youtube_dl/extractor/crunchyroll.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 026a9177e7..7642b868e4 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -17,6 +17,7 @@ from ..utils import (
intlist_to_bytes,
unified_strdate,
clean_html,
+ urlencode_postdata,
)
from ..aes import (
aes_cbc_decrypt,
@@ -51,6 +52,24 @@ class CrunchyrollIE(InfoExtractor):
'1080': ('80', '108'),
}
+ def _login(self):
+ (username, password) = self._get_login_info()
+ if username is None:
+ return
+ self.report_login()
+ login_url = 'https://www.crunchyroll.com/?a=formhandler'
+ data = urlencode_postdata({
+ 'formname': 'RpcApiUser_Login',
+ 'name': username,
+ 'password': password,
+ })
+ login_request = compat_urllib_request.Request(login_url, data)
+ login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ self._download_webpage(login_request, None, False, 'Wrong login info')
+
+ def _real_initialize(self):
+ self._login()
+
def _decrypt_subtitles(self, data, iv, id):
data = bytes_to_intlist(data)
iv = bytes_to_intlist(iv)
From 78272a076e96bea48525e415e42fa4d95d4d66fa Mon Sep 17 00:00:00 2001
From: Adam <oneplusme@users.noreply.github.com>
Date: Sat, 30 Aug 2014 12:48:56 +0100
Subject: [PATCH 2/4] Add SSA (.ass) subtitle output for Crunchyroll
---
youtube_dl/extractor/crunchyroll.py | 78 ++++++++++++++++++++++++++++-
1 file changed, 77 insertions(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py
index 7642b868e4..4903764f70 100644
--- a/youtube_dl/extractor/crunchyroll.py
+++ b/youtube_dl/extractor/crunchyroll.py
@@ -5,6 +5,7 @@ import re
import json
import base64
import zlib
+import xml.etree.ElementTree
from hashlib import sha1
from math import pow, sqrt, floor
@@ -67,9 +68,11 @@ class CrunchyrollIE(InfoExtractor):
login_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
self._download_webpage(login_request, None, False, 'Wrong login info')
+
def _real_initialize(self):
self._login()
+
def _decrypt_subtitles(self, data, iv, id):
data = bytes_to_intlist(data)
iv = bytes_to_intlist(iv)
@@ -116,6 +119,75 @@ class CrunchyrollIE(InfoExtractor):
output += '%d\n%s --> %s\n%s\n\n' % (i, start, end, text)
return output
+ def _convert_subtitles_to_ass(self, subtitles):
+ output = ''
+
+ def ass_bool(strvalue):
+ assvalue = '0'
+ if strvalue == '1':
+ assvalue = '-1'
+ return assvalue
+
+ sub_root = xml.etree.ElementTree.fromstring(subtitles)
+ if not sub_root:
+ return output
+
+ output = '[Script Info]\n'
+ output += 'Title: %s\n' % sub_root.attrib["title"]
+ output += 'ScriptType: v4.00+\n'
+ output += 'WrapStyle: %s\n' % sub_root.attrib["wrap_style"]
+ output += 'PlayResX: %s\n' % sub_root.attrib["play_res_x"]
+ output += 'PlayResY: %s\n' % sub_root.attrib["play_res_y"]
+ output += """ScaledBorderAndShadow: yes
+
+[V4+ Styles]
+Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
+"""
+ for style in sub_root.findall('./styles/style'):
+ output += 'Style: ' + style.attrib["name"]
+ output += ',' + style.attrib["font_name"]
+ output += ',' + style.attrib["font_size"]
+ output += ',' + style.attrib["primary_colour"]
+ output += ',' + style.attrib["secondary_colour"]
+ output += ',' + style.attrib["outline_colour"]
+ output += ',' + style.attrib["back_colour"]
+ output += ',' + ass_bool(style.attrib["bold"])
+ output += ',' + ass_bool(style.attrib["italic"])
+ output += ',' + ass_bool(style.attrib["underline"])
+ output += ',' + ass_bool(style.attrib["strikeout"])
+ output += ',' + style.attrib["scale_x"]
+ output += ',' + style.attrib["scale_y"]
+ output += ',' + style.attrib["spacing"]
+ output += ',' + style.attrib["angle"]
+ output += ',' + style.attrib["border_style"]
+ output += ',' + style.attrib["outline"]
+ output += ',' + style.attrib["shadow"]
+ output += ',' + style.attrib["alignment"]
+ output += ',' + style.attrib["margin_l"]
+ output += ',' + style.attrib["margin_r"]
+ output += ',' + style.attrib["margin_v"]
+ output += ',' + style.attrib["encoding"]
+ output += '\n'
+
+ output += """
+[Events]
+Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
+"""
+ for event in sub_root.findall('./events/event'):
+ output += 'Dialogue: 0'
+ output += ',' + event.attrib["start"]
+ output += ',' + event.attrib["end"]
+ output += ',' + event.attrib["style"]
+ output += ',' + event.attrib["name"]
+ output += ',' + event.attrib["margin_l"]
+ output += ',' + event.attrib["margin_r"]
+ output += ',' + event.attrib["margin_v"]
+ output += ',' + event.attrib["effect"]
+ output += ',' + event.attrib["text"]
+ output += '\n'
+
+ return output
+
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('video_id')
@@ -177,6 +249,7 @@ class CrunchyrollIE(InfoExtractor):
})
subtitles = {}
+ sub_format = self._downloader.params.get('subtitlesformat', 'srt')
for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage):
sub_page = self._download_webpage('http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\
video_id, note='Downloading subtitles for '+sub_name)
@@ -193,7 +266,10 @@ class CrunchyrollIE(InfoExtractor):
lang_code = self._search_regex(r'lang_code=["\']([^"\']+)', subtitle, 'subtitle_lang_code', fatal=False)
if not lang_code:
continue
- subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
+ if sub_format == 'ass':
+ subtitles[lang_code] = self._convert_subtitles_to_ass(subtitle)
+ else:
+ subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle)
return {
'id': video_id,
From 7eb21356f98315b15fc05d9e616951511253076f Mon Sep 17 00:00:00 2001
From: peugeot <peugeot@psa.fr>
Date: Sun, 31 Aug 2014 12:48:34 +0200
Subject: [PATCH 3/4] Add support for sunporno.com
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/sunporno.py | 57 ++++++++++++++++++++++++++++++++
2 files changed, 58 insertions(+)
create mode 100644 youtube_dl/extractor/sunporno.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 1479d998a2..72b01505de 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -322,6 +322,7 @@ from .stanfordoc import StanfordOpenClassroomIE
from .steam import SteamIE
from .streamcloud import StreamcloudIE
from .streamcz import StreamCZIE
+from .sunporno import SunPornoIE
from .swrmediathek import SWRMediathekIE
from .syfy import SyfyIE
from .sztvhu import SztvHuIE
diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
new file mode 100644
index 0000000000..2a193dee3d
--- /dev/null
+++ b/youtube_dl/extractor/sunporno.py
@@ -0,0 +1,57 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ parse_duration,
+ int_or_none,
+)
+
+
+class SunPornoIE(InfoExtractor):
+ _VALID_URL = r'https?://(?:www\.)?sunporno\.com/videos/(?P<id>\d+)'
+ _TEST = {
+ 'url': 'http://www.sunporno.com/videos/807778/',
+ 'md5': '6457d3c165fd6de062b99ef6c2ff4c86',
+ 'info_dict': {
+ 'id': '807778',
+ 'ext': 'flv',
+ 'title': 'md5:0a400058e8105d39e35c35e7c5184164',
+ 'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
+ 'duration': 302,
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = self._html_search_regex(
+ r'videoSource\s*=\s*\'<source\s*src="([^"]*)"', webpage, 'video URL')
+
+ title = self._html_search_regex(r'<title>([^<]*)</title>', webpage, 'title')
+
+ description = self._html_search_regex(
+ r'<meta name="description" content="([^"]*)"', webpage, 'description', fatal=False)
+
+ thumbnail = self._html_search_regex(
+ r'poster="([^"]*)"', webpage, 'thumbnail', fatal=False)
+
+ duration = parse_duration(self._search_regex(
+ r'<span>Duration: (\d+:\d+)</span>', webpage, 'duration', fatal=False))
+
+ view_count = int_or_none(self._html_search_regex(
+ r'<span class="views">(\d+)</span>', webpage, 'view count', fatal=False))
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'view_count': view_count,
+ }
From ae7246e7d57654381fb629be46662f8e4ded615f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 1 Sep 2014 01:08:19 +0700
Subject: [PATCH 4/4] [sunporno] Extract all formats and metadata
---
youtube_dl/extractor/sunporno.py | 31 +++++++++++++++++++++----------
1 file changed, 21 insertions(+), 10 deletions(-)
diff --git a/youtube_dl/extractor/sunporno.py b/youtube_dl/extractor/sunporno.py
index 2a193dee3d..c7a46eb718 100644
--- a/youtube_dl/extractor/sunporno.py
+++ b/youtube_dl/extractor/sunporno.py
@@ -6,6 +6,8 @@ from .common import InfoExtractor
from ..utils import (
parse_duration,
int_or_none,
+ qualities,
+ determine_ext,
)
@@ -19,6 +21,7 @@ class SunPornoIE(InfoExtractor):
'ext': 'flv',
'title': 'md5:0a400058e8105d39e35c35e7c5184164',
'description': 'md5:a31241990e1bd3a64e72ae99afb325fb',
+ 'thumbnail': 're:^https?://.*\.jpg$',
'duration': 302,
}
}
@@ -29,29 +32,37 @@ class SunPornoIE(InfoExtractor):
webpage = self._download_webpage(url, video_id)
- video_url = self._html_search_regex(
- r'videoSource\s*=\s*\'<source\s*src="([^"]*)"', webpage, 'video URL')
-
- title = self._html_search_regex(r'<title>([^<]*)</title>', webpage, 'title')
-
- description = self._html_search_regex(
- r'<meta name="description" content="([^"]*)"', webpage, 'description', fatal=False)
-
+ title = self._html_search_regex(r'<title>([^<]+)</title>', webpage, 'title')
+ description = self._html_search_meta('description', webpage, 'description')
thumbnail = self._html_search_regex(
- r'poster="([^"]*)"', webpage, 'thumbnail', fatal=False)
+ r'poster="([^"]+)"', webpage, 'thumbnail', fatal=False)
duration = parse_duration(self._search_regex(
r'<span>Duration: (\d+:\d+)</span>', webpage, 'duration', fatal=False))
view_count = int_or_none(self._html_search_regex(
r'<span class="views">(\d+)</span>', webpage, 'view count', fatal=False))
+ comment_count = int_or_none(self._html_search_regex(
+ r'(\d+)</b> Comments?', webpage, 'comment count', fatal=False))
+
+ formats = []
+ quality = qualities(['mp4', 'flv'])
+ for video_url in re.findall(r'<source src="([^"]+)"', webpage):
+ video_ext = determine_ext(video_url)
+ formats.append({
+ 'url': video_url,
+ 'format_id': video_ext,
+ 'quality': quality(video_ext),
+ })
+ self._sort_formats(formats)
return {
'id': video_id,
- 'url': video_url,
'title': title,
'description': description,
'thumbnail': thumbnail,
'duration': duration,
'view_count': view_count,
+ 'comment_count': comment_count,
+ 'formats': formats,
}