From 1666c2709f575a20c7ae1d0c76935b855e872dc9 Mon Sep 17 00:00:00 2001
From: Dan Walker <dwalker@tripwire.com>
Date: Sun, 25 Oct 2020 06:08:57 -0700
Subject: [PATCH 1/4] Fixed tiktok.py The JSON formatting of the response
changed, corrected.
#TESTING
Successfully downloaded and played a tiktok video
---
youtube_dlc/extractor/tiktok.py | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/youtube_dlc/extractor/tiktok.py b/youtube_dlc/extractor/tiktok.py
index 52e5f4f1fa..f27aa42507 100644
--- a/youtube_dlc/extractor/tiktok.py
+++ b/youtube_dlc/extractor/tiktok.py
@@ -14,17 +14,17 @@ from ..utils import (
class TikTokBaseIE(InfoExtractor):
def _extract_aweme(self, video_data, webpage, url):
video_info = try_get(
- video_data, lambda x: x['videoData']['itemInfos'], dict)
+ video_data, lambda x: x['itemInfo']['itemStruct'], dict)
author_info = try_get(
- video_data, lambda x: x['videoData']['authorInfos'], dict)
- share_info = try_get(video_data, lambda x: x['shareMeta'], dict)
+ video_data, lambda x: x['itemInfo']['itemStruct']['author'], dict)
+ share_info = try_get(video_data, lambda x: x['itemInfo']['shareMeta'], dict)
unique_id = str_or_none(author_info.get('uniqueId'))
timestamp = try_get(video_info, lambda x: int(x['createTime']), int)
date = datetime.fromtimestamp(timestamp).strftime('%Y%m%d')
- height = try_get(video_info, lambda x: x['video']['videoMeta']['height'], int)
- width = try_get(video_info, lambda x: x['video']['videoMeta']['width'], int)
+ height = try_get(video_info, lambda x: x['video']['height'], int)
+ width = try_get(video_info, lambda x: x['video']['width'], int)
thumbnails = []
thumbnails.append({
'url': video_info.get('thumbnail') or self._og_search_thumbnail(webpage),
@@ -34,7 +34,7 @@ class TikTokBaseIE(InfoExtractor):
formats = []
formats.append({
- 'url': try_get(video_info, lambda x: x['video']['urls'][0]),
+ 'url': try_get(video_info, lambda x: x['video']['playAddr']),
'ext': 'mp4',
'height': height,
'width': width
From 4f5a0ad82214456c3d4832bbf4e15e416e1a16cb Mon Sep 17 00:00:00 2001
From: Unknown <blackjack4494@web.de>
Date: Mon, 26 Oct 2020 02:15:46 +0100
Subject: [PATCH 2/4] [tiktok] fallback url + cookie
---
youtube_dlc/extractor/tiktok.py | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/youtube_dlc/extractor/tiktok.py b/youtube_dlc/extractor/tiktok.py
index f27aa42507..0cfd2168a6 100644
--- a/youtube_dlc/extractor/tiktok.py
+++ b/youtube_dlc/extractor/tiktok.py
@@ -12,7 +12,8 @@ from ..utils import (
class TikTokBaseIE(InfoExtractor):
- def _extract_aweme(self, video_data, webpage, url):
+ def _extract_aweme(self, props_data, webpage, url):
+ video_data = try_get(props_data, lambda x: x['pageProps'], expected_type=dict)
video_info = try_get(
video_data, lambda x: x['itemInfo']['itemStruct'], dict)
author_info = try_get(
@@ -32,14 +33,20 @@ class TikTokBaseIE(InfoExtractor):
'height': height
})
+ url = ''
+ if not url:
+ url = try_get(video_info, lambda x: x['video']['playAddr'])
+ if not url:
+ url = try_get(video_info, lambda x: x['video']['downloadAddr'])
formats = []
formats.append({
- 'url': try_get(video_info, lambda x: x['video']['playAddr']),
+ 'url': url,
'ext': 'mp4',
'height': height,
'width': width
})
+ tracker = try_get(props_data, lambda x: x['initialProps']['$wid'])
return {
'comment_count': int_or_none(video_info.get('commentCount')),
'duration': try_get(video_info, lambda x: x['video']['videoMeta']['duration'], int),
@@ -63,6 +70,7 @@ class TikTokBaseIE(InfoExtractor):
'formats': formats,
'http_headers': {
'Referer': url,
+ 'Cookie': 'tt_webid=%s; tt_webid_v2=%s' % (tracker, tracker),
}
}
@@ -130,10 +138,10 @@ class TikTokIE(TikTokBaseIE):
r'id=\"__NEXT_DATA__\"\s+type=\"application\/json\"\s*[^>]+>\s*(?P<json_string_ld>[^<]+)',
webpage, 'json_string', group='json_string_ld')
json_data = self._parse_json(json_string, video_id)
- video_data = try_get(json_data, lambda x: x['props']['pageProps'], expected_type=dict)
+ props_data = try_get(json_data, lambda x: x['props'], expected_type=dict)
# Chech statusCode for success
- if video_data.get('statusCode') == 0:
- return self._extract_aweme(video_data, webpage, url)
+ if props_data.get('pageProps').get('statusCode') == 0:
+ return self._extract_aweme(props_data, webpage, url)
raise ExtractorError('Video not available', video_id=video_id)
From fb2c9277bc16bbfb22ccc61bc913d9259f3e6ba9 Mon Sep 17 00:00:00 2001
From: Unknown <blackjack4494@web.de>
Date: Mon, 26 Oct 2020 04:32:37 +0100
Subject: [PATCH 3/4] [youtube] ultimate age gate fix
---
youtube_dlc/extractor/youtube.py | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/youtube_dlc/extractor/youtube.py b/youtube_dlc/extractor/youtube.py
index b5b229ccbc..4fb49b864c 100644
--- a/youtube_dlc/extractor/youtube.py
+++ b/youtube_dlc/extractor/youtube.py
@@ -1742,6 +1742,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
add_dash_mpd_pr(pl_response)
return pl_response
+ def extract_embedded_config(embed_webpage, video_id):
+ embedded_config = self._search_regex(
+ r'setConfig\(({.*})\);',
+ embed_webpage, 'ytInitialData', default=None)
+ if embedded_config:
+ return embedded_config
+
player_response = {}
# Get video info
@@ -1755,8 +1762,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# this can be viewed without login into Youtube
url = proto + '://www.youtube.com/embed/%s' % video_id
embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
- # check if video is only playable on youtube - if so it requires auth (cookies)
- if re.search(r'player-unavailable">', embed_webpage) is not None:
+ ext = extract_embedded_config(embed_webpage, video_id)
+ # playabilityStatus = re.search(r'{\\\"status\\\":\\\"(?P<playabilityStatus>[^\"]+)\\\"', ext)
+ playable_in_embed = re.search(r'{\\\"playableInEmbed\\\":(?P<playableinEmbed>[^\,]+)', ext)
+ if not playable_in_embed:
+ self.to_screen('Could not determine whether playabale in embed for video %s' % video_id)
+ playable_in_embed = ''
+ else:
+ playable_in_embed = playable_in_embed.group('playableinEmbed')
+ # check if video is only playable on youtube in other words not playable in embed - if so it requires auth (cookies)
+ # if re.search(r'player-unavailable">', embed_webpage) is not None:
+ if playable_in_embed == 'false':
'''
# TODO apply this patch when Support for Python 2.6(!) and above drops
if ({'VISITOR_INFO1_LIVE', 'HSID', 'SSID', 'SID'} <= cookie_keys
From 8045ef7760329db0e69cce0f9af3e1e9efbe1820 Mon Sep 17 00:00:00 2001
From: Unknown <blackjack4494@web.de>
Date: Mon, 26 Oct 2020 04:34:25 +0100
Subject: [PATCH 4/4] next release: 2020.10.26
---
.github/ISSUE_TEMPLATE/1_broken_site.md | 6 +++---
.github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
.github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
.github/ISSUE_TEMPLATE/4_bug_report.md | 6 +++---
.github/ISSUE_TEMPLATE/5_feature_request.md | 4 ++--
youtube_dlc/version.py | 2 +-
6 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 92db313a44..bf42510040 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -21,7 +21,7 @@ assignees: ''
<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc:
-- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -29,7 +29,7 @@ Carefully read and work through this check list in order to prevent the most com
-->
- [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dlc version **2020.10.24**
+- [ ] I've verified that I'm running youtube-dlc version **2020.10.26**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
- [ ] I've searched the bugtracker for similar issues including closed ones
@@ -44,7 +44,7 @@ Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v
[debug] User config: []
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dlc version 2020.10.24
+ [debug] youtube-dlc version 2020.10.26
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
[debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index e70c853e1d..889005097b 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -21,7 +21,7 @@ assignees: ''
<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc:
-- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
- Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dlc does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
- Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -29,7 +29,7 @@ Carefully read and work through this check list in order to prevent the most com
-->
- [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dlcc version **2020.10.24**
+- [ ] I've verified that I'm running youtube-dlcc version **2020.10.26**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that none of provided URLs violate any copyrights
- [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index f91c70835c..e5d7143887 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -21,13 +21,13 @@ assignees: ''
<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc:
-- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
- Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
- Finally, put x into all relevant boxes (like this [x])
-->
- [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dlc version **2020.10.24**
+- [ ] I've verified that I'm running youtube-dlc version **2020.10.26**
- [ ] I've searched the bugtracker for similar site feature requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index b34dbe8777..9de52f98c9 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -21,7 +21,7 @@ assignees: ''
<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc:
-- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
- Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
- Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
- Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -30,7 +30,7 @@ Carefully read and work through this check list in order to prevent the most com
-->
- [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dlc version **2020.10.24**
+- [ ] I've verified that I'm running youtube-dlc version **2020.10.26**
- [ ] I've checked that all provided URLs are alive and playable in a browser
- [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
- [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -46,7 +46,7 @@ Add the `-v` flag to your command line you run youtube-dlc with (`youtube-dlc -v
[debug] User config: []
[debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
[debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dlc version 2020.10.24
+ [debug] youtube-dlc version 2020.10.26
[debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
[debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
[debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 8646df1608..86fac96dde 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -21,13 +21,13 @@ assignees: ''
<!--
Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dlc:
-- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dlc. Run `youtube-dlc --version` and ensure your version is 2020.10.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
- Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
- Finally, put x into all relevant boxes (like this [x])
-->
- [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dlc version **2020.10.24**
+- [ ] I've verified that I'm running youtube-dlc version **2020.10.26**
- [ ] I've searched the bugtracker for similar feature requests including closed ones
diff --git a/youtube_dlc/version.py b/youtube_dlc/version.py
index 5eb581675c..440d8e4882 100644
--- a/youtube_dlc/version.py
+++ b/youtube_dlc/version.py
@@ -1,3 +1,3 @@
from __future__ import unicode_literals
-__version__ = '2020.10.24-5'
+__version__ = '2020.10.25'