From e3aaace4005e7b767d28152e60d1cf6272c398ec Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:55:41 +0100
Subject: [PATCH 1/6] [tweakers] Add new extractor
---
youtube_dl/extractor/__init__.py | 1 +
youtube_dl/extractor/tweakers.py | 41 ++++++++++++++++++++++++++++++++
2 files changed, 42 insertions(+)
create mode 100644 youtube_dl/extractor/tweakers.py
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5866a76179..047f7002a3 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -475,6 +475,7 @@ from .tutv import TutvIE
from .tvigle import TvigleIE
from .tvp import TvpIE, TvpSeriesIE
from .tvplay import TVPlayIE
+from .tweakers import TweakersIE
from .twentyfourvideo import TwentyFourVideoIE
from .twitch import (
TwitchVideoIE,
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
new file mode 100644
index 0000000000..3080f9e467
--- /dev/null
+++ b/youtube_dl/extractor/tweakers.py
@@ -0,0 +1,41 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class TweakersIE(InfoExtractor):
+ _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+ _TEST = {
+ 'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
+ 'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+ 'info_dict': {
+ 'id': '9926',
+ 'ext': 'mp4',
+ 'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+ # TODO more properties, either as:
+ # * A value
+ # * MD5 checksum; start the string with md5:
+ # * A regular expression; start the string with re:
+ # * Any Python type (for example int or float)
+ }
+ }
+
+ def _real_extract(self, url):
+ splitted_url = re.split('.html|/', url)
+ del splitted_url[-1] # To remove extra '/' at the end
+ video_id = splitted_url[4]
+ title = splitted_url[5].title() # Retrieve title for URL and capitalize
+ splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+ player_url = '/'.join(splitted_url) + '.html'
+ player_page = self._download_webpage(player_url, video_id)
+
+ return {
+ 'id': video_id,
+ 'ext': 'mp4',
+ 'title': title,
+ 'url': re.findall('http.*mp4', player_page)[0],
+ 'player_url': player_url
+ }
From 3ee6e02564f7f6c54480fd4eeb94ae08a747b9b6 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 19:59:36 +0100
Subject: [PATCH 2/6] Edit Tweakers extractor
Fixed code conventions (mainly adding two or more spaces before making
an inline comment)
---
youtube_dl/extractor/tweakers.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 3080f9e467..29cd04f005 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -25,10 +25,10 @@ class TweakersIE(InfoExtractor):
def _real_extract(self, url):
splitted_url = re.split('.html|/', url)
- del splitted_url[-1] # To remove extra '/' at the end
+ del splitted_url[-1] # To remove extra '/' at the end
video_id = splitted_url[4]
- title = splitted_url[5].title() # Retrieve title for URL and capitalize
- splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
+ title = splitted_url[5].title() # Retrieve title for URL and capitalize
+ splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
player_url = '/'.join(splitted_url) + '.html'
player_page = self._download_webpage(player_url, video_id)
@@ -36,6 +36,6 @@ class TweakersIE(InfoExtractor):
'id': video_id,
'ext': 'mp4',
'title': title,
- 'url': re.findall('http.*mp4', player_page)[0],
+ 'url': re.findall('http.*mp4', player_page)[0],
'player_url': player_url
}
From c893d70805be9b8f8a10c7d7dbe39216f8976ea5 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:38:35 +0100
Subject: [PATCH 3/6] Remove player-url in tweakers.py
Player-url only needed for rmftp, not for regular URLs
---
youtube_dl/extractor/tweakers.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 29cd04f005..14145e617e 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -37,5 +37,4 @@ class TweakersIE(InfoExtractor):
'ext': 'mp4',
'title': title,
'url': re.findall('http.*mp4', player_page)[0],
- 'player_url': player_url
}
From bc63d56ccac8805ca8394e0b41890027e6d677d2 Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:40:18 +0100
Subject: [PATCH 4/6] Remove unnecessary TODO-comments
---
youtube_dl/extractor/tweakers.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index 14145e617e..a4e7498501 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -15,11 +15,6 @@ class TweakersIE(InfoExtractor):
'id': '9926',
'ext': 'mp4',
'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
- # TODO more properties, either as:
- # * A value
- # * MD5 checksum; start the string with md5:
- # * A regular expression; start the string with re:
- # * Any Python type (for example int or float)
}
}
From 67ce4f8820f94961c9b094f9c652f421c9d9f6fb Mon Sep 17 00:00:00 2001
From: robin <rderooij685@gmail.com>
Date: Thu, 5 Feb 2015 21:49:13 +0100
Subject: [PATCH 5/6] Use match_id method instead of splitted URL
---
youtube_dl/extractor/tweakers.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index a4e7498501..e332d46948 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -21,7 +21,7 @@ class TweakersIE(InfoExtractor):
def _real_extract(self, url):
splitted_url = re.split('.html|/', url)
del splitted_url[-1] # To remove extra '/' at the end
- video_id = splitted_url[4]
+ video_id = self._match_id(url)
title = splitted_url[5].title() # Retrieve title for URL and capitalize
splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
player_url = '/'.join(splitted_url) + '.html'
From 9c91a8fa70e3c2f29dd23809e90b3357b6c1cdc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 6 Feb 2015 03:23:42 +0600
Subject: [PATCH 6/6] [tweakers] Switch extraction to xspf playlist, extract
all formats and meta (#4881)
---
youtube_dl/extractor/tweakers.py | 58 ++++++++++++++++++++++++--------
1 file changed, 44 insertions(+), 14 deletions(-)
diff --git a/youtube_dl/extractor/tweakers.py b/youtube_dl/extractor/tweakers.py
index e332d46948..c80ec15cf1 100644
--- a/youtube_dl/extractor/tweakers.py
+++ b/youtube_dl/extractor/tweakers.py
@@ -1,35 +1,65 @@
-# coding: utf-8
from __future__ import unicode_literals
-import re
-
from .common import InfoExtractor
+from ..utils import (
+ xpath_text,
+ xpath_with_ns,
+ int_or_none,
+ float_or_none,
+)
class TweakersIE(InfoExtractor):
- _VALID_URL = r'https?://tweakers\.net/video/(?P<id>[0-9]+).*'
+ _VALID_URL = r'https?://tweakers\.net/video/(?P<id>\d+)'
_TEST = {
'url': 'https://tweakers.net/video/9926/new-nintendo-3ds-xl-op-alle-fronten-beter.html',
- 'md5': 'f7f7f3027166a7f32f024b4ae6571ced',
+ 'md5': '1b5afa817403bb5baa08359dca31e6df',
'info_dict': {
'id': '9926',
'ext': 'mp4',
- 'title': 'New-Nintendo-3Ds-Xl-Op-Alle-Fronten-Beter',
+ 'title': 'New Nintendo 3DS XL - Op alle fronten beter',
+ 'description': 'md5:f97324cc71e86e11c853f0763820e3ba',
+ 'thumbnail': 're:^https?://.*\.jpe?g$',
+ 'duration': 386,
}
}
def _real_extract(self, url):
- splitted_url = re.split('.html|/', url)
- del splitted_url[-1] # To remove extra '/' at the end
video_id = self._match_id(url)
- title = splitted_url[5].title() # Retrieve title for URL and capitalize
- splitted_url[3] = splitted_url[3] + '/player' # Add /player to get the player page
- player_url = '/'.join(splitted_url) + '.html'
- player_page = self._download_webpage(player_url, video_id)
+
+ playlist = self._download_xml(
+ 'https://tweakers.net/video/s1playlist/%s/playlist.xspf' % video_id,
+ video_id)
+
+ NS_MAP = {
+ 'xspf': 'http://xspf.org/ns/0/',
+ 's1': 'http://static.streamone.nl/player/ns/0',
+ }
+
+ track = playlist.find(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP))
+
+ title = xpath_text(
+ track, xpath_with_ns('./xspf:title', NS_MAP), 'title')
+ description = xpath_text(
+ track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
+ thumbnail = xpath_text(
+ track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
+ duration = float_or_none(
+ xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'),
+ 1000)
+
+ formats = [{
+ 'url': location.text,
+ 'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
+ 'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
+ 'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
+ } for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP))]
return {
'id': video_id,
- 'ext': 'mp4',
'title': title,
- 'url': re.findall('http.*mp4', player_page)[0],
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'duration': duration,
+ 'formats': formats,
}