[grooveshark,http] Make HTTP POST downloads work

youtube_dl/downloader/http.py

@@ -27,8 +27,16 @@ class HttpFD(FileDownloader):
             headers['Youtubedl-user-agent'] = info_dict['user_agent']
         if 'http_referer' in info_dict:
             headers['Referer'] = info_dict['http_referer']
-        basic_request = compat_urllib_request.Request(url, None, headers)
+        add_headers = info_dict.get('http_headers')
-        request = compat_urllib_request.Request(url, None, headers)
+        if add_headers:
+            headers.update(add_headers)
+        data = info_dict.get('http_post_data')
+        http_method = info_dict.get('http_method')
+        basic_request = compat_urllib_request.Request(url, data, headers)
+        request = compat_urllib_request.Request(url, data, headers)
+        if http_method is not None:
+            basic_request.get_method = lambda: http_method
+            request.get_method = lambda: http_method
 
         is_test = self.params.get('test', False)
 

youtube_dl/extractor/common.py

@@ -84,6 +84,12 @@ class InfoExtractor(object):
                                  format, irrespective of the file format.
                                  -1 for default (order by other properties),
                                  -2 or smaller for less than default.
+                    * http_referer  HTTP Referer header value to set.
+                    * http_method  HTTP method to use for the download.
+                    * http_headers  A dictionary of additional HTTP headers
+                                 to add to the request.
+                    * http_post_data  Additional data to send with a POST
+                                 request.
     url:            Final video URL.
     ext:            Video filename extension.
     format:         The video format, defaults to ext (used for --get-format)

youtube_dl/extractor/grooveshark.py

@@ -12,7 +12,7 @@ from .common import InfoExtractor
 from ..utils import ExtractorError, compat_urllib_request, compat_html_parser
 
 from ..utils import compat_urlparse
-urlparse = compat_urlparse.urlparse
+
 urlunparse = compat_urlparse.urlunparse
 urldefrag = compat_urlparse.urldefrag
 
@@ -52,7 +52,7 @@ class GroovesharkIE(InfoExtractor):
             'id': '6SS1DW',
             'title': 'Jolene (Tenth Key Remix ft. Will Sessions)',
             'ext': 'mp3',
-            'duration': 227
+            'duration': 227,
         }
     }
 
@@ -60,7 +60,7 @@ class GroovesharkIE(InfoExtractor):
     do_bootstrap_request = True
 
     def _parse_target(self, target):
-        uri = urlparse(target)
+        uri = compat_urlparse.urlparse(target)
         hash = uri.fragment[1:].split('?')[0]
         token = basename(hash.rstrip('/'))
         return (uri, hash, token)
@@ -123,22 +123,36 @@ class GroovesharkIE(InfoExtractor):
     def _get_playerpage(self, target):
         (_, _, token) = self._parse_target(target)
 
-        res = self._download_webpage(
+
+        webpage = self._download_webpage(
             target, token,
             note='Downloading player page',
             errnote='Unable to download player page',
             fatal=False)
 
-        if res is not None:
+        if webpage is not None:
-            o = GroovesharkHtmlParser.extract_object_tags(res)
+            # Search (for example German) error message
-            return (res, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+            error_msg = self._html_search_regex(
+                r'<div id="content">\s*<h2>(.*?)</h2>', webpage,
+                'error message', default=None)
+            if error_msg is not None:
+                error_msg = error_msg.replace('\n', ' ')
+                raise ExtractorError('Grooveshark said: %s' % error_msg)
 
-        return (res, None)
+        if webpage is not None:
+            o = GroovesharkHtmlParser.extract_object_tags(webpage)
+            return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed'])
+
+        return (webpage, None)
+
+    def _real_initialize(self):
+        self.ts = int(time.time() * 1000)  # timestamp in millis
 
     def _real_extract(self, url):
         (target_uri, _, token) = self._parse_target(url)
 
         # 1. Fill cookiejar by making a request to the player page
+        swf_referer = None
         if self.do_playerpage_request:
             (_, player_objs) = self._get_playerpage(url)
             if player_objs is not None:
@@ -162,11 +176,10 @@ class GroovesharkIE(InfoExtractor):
             'Content-Length': len(post_data),
             'Content-Type': 'application/x-www-form-urlencoded'
         }
-
+        if swf_referer is not None:
-        if 'swf_referer' in locals():
             headers['Referer'] = swf_referer
 
-        info_dict = {
+        return {
             'id': token,
             'title': meta['song']['Name'],
             'http_method': 'POST',
@@ -174,32 +187,6 @@ class GroovesharkIE(InfoExtractor):
             'ext': 'mp3',
             'format': 'mp3 audio',
             'duration': duration,
-
+            'http_post_data': post_data,
-            # various ways of supporting the download request.
+            'http_headers': headers,
-            # remove keys unnecessary to the eventual post implementation
-            'post_data': post_data,
-            'post_dict': post_dict,
-            'headers': headers
         }
-
-        if 'swf_referer' in locals():
-            info_dict['http_referer'] = swf_referer
-
-        return info_dict
-
-    def _real_initialize(self):
-        self.ts = int(time.time() * 1000)  # timestamp in millis
-
-    def _download_json(self, url_or_request, video_id,
-                       note=u'Downloading JSON metadata',
-                       errnote=u'Unable to download JSON metadata',
-                       fatal=True,
-                       transform_source=None):
-        try:
-            out = super(GroovesharkIE, self)._download_json(
-                url_or_request, video_id, note, errnote, transform_source)
-            return out
-        except ExtractorError as ee:
-            if fatal:
-                raise ee
-        return None