From efbed08dc20c530fe428256e4dcbea4dc4423d0d Mon Sep 17 00:00:00 2001
From: Yen Chi Hsuan <yan12125@gmail.com>
Date: Wed, 23 Mar 2016 22:24:52 +0800
Subject: [PATCH] [utils] Encode hostnames before passing to urllib
With IDN (Internationalized Domain Name) and a proxy, non-ascii URLs
are passed down to urllib/urllib2, causing UnicodeEncodeError
Fixes #8890
---
test/test_http.py | 10 ++++++++++
youtube_dl/utils.py | 1 +
2 files changed, 11 insertions(+)
diff --git a/test/test_http.py b/test/test_http.py
index fc59b1aed6..15e0ad369d 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
+# coding: utf-8
from __future__ import unicode_literals
# Allow direct execution
@@ -120,5 +121,14 @@ class TestProxy(unittest.TestCase):
response = ydl.urlopen(req).read().decode('utf-8')
self.assertEqual(response, 'cn: {0}'.format(url))
+ def test_proxy_with_idn(self):
+ ydl = YoutubeDL({
+ 'proxy': 'localhost:{0}'.format(self.port),
+ })
+ url = 'http://中文.tw/'
+ response = ydl.urlopen(url).read().decode('utf-8')
+ # b'xn--fiq228c' is '中文'.encode('idna')
+ self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
+
if __name__ == '__main__':
unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 067b8a184c..03bb7782f4 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1746,6 +1746,7 @@ def escape_url(url):
"""Escape URL as suggested by RFC 3986"""
url_parsed = compat_urllib_parse_urlparse(url)
return url_parsed._replace(
+ netloc=url_parsed.netloc.encode('idna').decode('ascii'),
path=escape_rfc3986(url_parsed.path),
params=escape_rfc3986(url_parsed.params),
query=escape_rfc3986(url_parsed.query),