From 60351178a59123ef6917bbfa366f0a58fa812be5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?=
<diegorodriguezv@gmail.com>
Date: Sun, 1 Nov 2020 21:25:34 -0500
Subject: [PATCH 1/3] [TMZ] Fix TMZ.com extractor
---
youtube_dlc/extractor/tmz.py | 73 ++++++++++++++++++++++++++++--------
1 file changed, 58 insertions(+), 15 deletions(-)
diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py
index 419f9d92ee..dae6eab9d4 100644
--- a/youtube_dlc/extractor/tmz.py
+++ b/youtube_dlc/extractor/tmz.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
class TMZIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/?#]+)'
+ _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/.*(?P<id>[^/?#]{10,10})'
_TESTS = [{
'url': 'http://www.tmz.com/videos/0_okj015ty/',
'md5': '4d22a51ef205b6c06395d8394f72d560',
@@ -13,14 +13,30 @@ class TMZIE(InfoExtractor):
'id': '0_okj015ty',
'ext': 'mp4',
'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
- 'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie??? Or is she just showing off her amazing boobs?',
'timestamp': 1394747163,
'uploader_id': 'batchUser',
'upload_date': '20140313',
}
}, {
'url': 'http://www.tmz.com/videos/0-cegprt2p/',
- 'only_matching': True,
+ 'info_dict': {
+ 'id': '0_cegprt2p',
+ 'ext': 'mp4',
+ 'title': "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet",
+ 'timestamp': 1467831837,
+ 'uploader_id': 'batchUser',
+ 'upload_date': '20160706',
+ }
+ }, {
+ 'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/',
+ 'info_dict': {
+ 'id': '0_zcsejvcr',
+ 'ext': 'mxf',
+ 'title': "Angry Bagel Shop Guy Says He Doesn't Trust Women",
+ 'timestamp': 1562889485,
+ 'uploader_id': 'batchUser',
+ 'upload_date': '20190711',
+ }
}]
def _real_extract(self, url):
@@ -30,27 +46,54 @@ class TMZIE(InfoExtractor):
class TMZArticleIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
- _TEST = {
+ _TESTS = [{
'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
- 'md5': '3316ff838ae5bb7f642537825e1e90d2',
+ 'md5': '5429c85db8bde39a473a56ca8c4c5602',
'info_dict': {
'id': '0_6snoelag',
- 'ext': 'mov',
+ 'ext': 'mp4',
'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
- 'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake. She\'s watching me."',
'timestamp': 1429467813,
'upload_date': '20150419',
'uploader_id': 'batchUser',
}
- }
+ }, {
+ 'url': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/',
+ 'info_dict': {
+ 'id': '0_jerz7s3l',
+ 'ext': 'mp4',
+ 'title': 'Patti LaBelle -- Goes Nuclear On Stripping Fan',
+ 'timestamp': 1442683746,
+ 'upload_date': '20150919',
+ 'uploader_id': 'batchUser',
+ }
+ }, {
+ 'url': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/',
+ 'info_dict': {
+ 'id': '0_ytz87kk7',
+ 'ext': 'mp4',
+ 'title': "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This",
+ 'timestamp': 1454010989,
+ 'upload_date': '20160128',
+ 'uploader_id': 'batchUser',
+ }
+ }, {
+ 'url': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/',
+ 'info_dict': {
+ 'id': '0_isigfatu',
+ 'ext': 'mp4',
+ 'title': "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!",
+ 'timestamp': 1477500095,
+ 'upload_date': '20161026',
+ 'uploader_id': 'batchUser',
+ }
+ }]
def _real_extract(self, url):
video_id = self._match_id(url)
-
webpage = self._download_webpage(url, video_id)
- embedded_video_info = self._parse_json(self._html_search_regex(
- r'tmzVideoEmbed\(({.+?})\);', webpage, 'embedded video info'),
- video_id)
-
- return self.url_result(
- 'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])
+ params = self._html_search_regex(r'TMZ.actions.clickLink\(([\s\S]+?)\)',
+ webpage, 'embedded video info').split(',')
+ new_url = params[0].strip("'\"")
+ if new_url != url:
+ return self.url_result(new_url)
From fff50711120b1a1c0477550748768d1e5b1fb755 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?=
<diegorodriguezv@gmail.com>
Date: Sun, 8 Nov 2020 15:36:41 -0500
Subject: [PATCH 2/3] [TMZ] Add support for new page structure using JSON-LD
---
youtube_dlc/extractor/extractors.py | 5 +-
youtube_dlc/extractor/tmz.py | 188 +++++++++++++++-------------
2 files changed, 99 insertions(+), 94 deletions(-)
diff --git a/youtube_dlc/extractor/extractors.py b/youtube_dlc/extractor/extractors.py
index 666134d868..5dde2965ba 100644
--- a/youtube_dlc/extractor/extractors.py
+++ b/youtube_dlc/extractor/extractors.py
@@ -1178,10 +1178,7 @@ from .thisoldhouse import ThisOldHouseIE
from .threeqsdn import ThreeQSDNIE
from .tiktok import TikTokIE
from .tinypic import TinyPicIE
-from .tmz import (
- TMZIE,
- TMZArticleIE,
-)
+from .tmz import TMZIE
from .tnaflix import (
TNAFlixNetworkEmbedIE,
TNAFlixIE,
diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py
index dae6eab9d4..a2f1009224 100644
--- a/youtube_dlc/extractor/tmz.py
+++ b/youtube_dlc/extractor/tmz.py
@@ -5,95 +5,103 @@ from .common import InfoExtractor
class TMZIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/.*(?P<id>[^/?#]{10,10})'
- _TESTS = [{
- 'url': 'http://www.tmz.com/videos/0_okj015ty/',
- 'md5': '4d22a51ef205b6c06395d8394f72d560',
- 'info_dict': {
- 'id': '0_okj015ty',
- 'ext': 'mp4',
- 'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
- 'timestamp': 1394747163,
- 'uploader_id': 'batchUser',
- 'upload_date': '20140313',
- }
- }, {
- 'url': 'http://www.tmz.com/videos/0-cegprt2p/',
- 'info_dict': {
- 'id': '0_cegprt2p',
- 'ext': 'mp4',
- 'title': "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet",
- 'timestamp': 1467831837,
- 'uploader_id': 'batchUser',
- 'upload_date': '20160706',
- }
- }, {
- 'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/',
- 'info_dict': {
- 'id': '0_zcsejvcr',
- 'ext': 'mxf',
- 'title': "Angry Bagel Shop Guy Says He Doesn't Trust Women",
- 'timestamp': 1562889485,
- 'uploader_id': 'batchUser',
- 'upload_date': '20190711',
- }
- }]
+ _VALID_URL = r"https?://(?:www\.)?tmz\.com/.*"
+ _TESTS = [
+ {
+ "url": "http://www.tmz.com/videos/0-cegprt2p/",
+ "info_dict": {
+ "id": "http://www.tmz.com/videos/0-cegprt2p/",
+ "ext": "mp4",
+ "title": "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet",
+ "description": "Harvey talks about Director Comey’s decision not to prosecute Hillary Clinton.",
+ "timestamp": 1467831837,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20160706",
+ },
+ },
+ {
+ "url": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/",
+ "info_dict": {
+ "id": "https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/",
+ "ext": "mp4",
+ "title": "Angry Bagel Shop Guy Says He Doesn't Trust Women",
+ "description": "The enraged man who went viral for ranting about women on dating sites before getting ragdolled in a bagel shop is defending his misogyny ... he says it's women's fault in the first place.",
+ "timestamp": 1562889485,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20190711",
+ },
+ },
+ {
+ "url": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert",
+ "md5": "5429c85db8bde39a473a56ca8c4c5602",
+ "info_dict": {
+ "id": "http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert",
+ "ext": "mp4",
+ "title": "Bobby Brown Tells Crowd ... Bobbi Kristina is Awake",
+ "description": 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake. She\'s watching me."',
+ "timestamp": 1429467813,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20150419",
+ },
+ },
+ {
+ "url": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/",
+ "info_dict": {
+ "id": "http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/",
+ "ext": "mp4",
+ "title": "Patti LaBelle -- Goes Nuclear On Stripping Fan",
+ "description": "Patti LaBelle made it known loud and clear last night ... NO "
+ "ONE gets on her stage and strips down.",
+ "timestamp": 1442683746,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20150919",
+ },
+ },
+ {
+ "url": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/",
+ "info_dict": {
+ "id": "http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/",
+ "ext": "mp4",
+ "title": "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This",
+ "description": "Two pretty parts of this video with NBA Commish Adam Silver.",
+ "timestamp": 1454010989,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20160128",
+ },
+ },
+ {
+ "url": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/",
+ "info_dict": {
+ "id": "http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/",
+ "ext": "mp4",
+ "title": "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!",
+ "description": "James Otis is the the guy who took a pickaxe to Donald Trump's star on the Walk of Fame, and he tells TMZ .. he's ready and willing to go to jail for the crime.",
+ "timestamp": 1477500095,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20161026",
+ },
+ },
+ {
+ "url": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/",
+ "info_dict": {
+ "id": "https://www.tmz.com/videos/2020-10-31-103120-beverly-hills-protest-4878209/",
+ "ext": "mp4",
+ "title": "Cops Use Billy Clubs Against Pro-Trump and Anti-Fascist "
+ "Demonstrators",
+ "description": "Beverly Hills may be an omen of what's coming next week, "
+ "because things got crazy on the streets and cops started "
+ "swinging their billy clubs at both Anti-Fascist and Pro-Trump "
+ "demonstrators.",
+ "timestamp": 1604182772,
+ "uploader": "{'@type': 'Person', 'name': 'TMZ Staff'}",
+ "upload_date": "20201031",
+ },
+ },
+ ]
def _real_extract(self, url):
- video_id = self._match_id(url).replace('-', '_')
- return self.url_result('kaltura:591531:%s' % video_id, 'Kaltura', video_id)
-
-
-class TMZArticleIE(InfoExtractor):
- _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
- _TESTS = [{
- 'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
- 'md5': '5429c85db8bde39a473a56ca8c4c5602',
- 'info_dict': {
- 'id': '0_6snoelag',
- 'ext': 'mp4',
- 'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
- 'timestamp': 1429467813,
- 'upload_date': '20150419',
- 'uploader_id': 'batchUser',
- }
- }, {
- 'url': 'http://www.tmz.com/2015/09/19/patti-labelle-concert-fan-stripping-kicked-out-nicki-minaj/',
- 'info_dict': {
- 'id': '0_jerz7s3l',
- 'ext': 'mp4',
- 'title': 'Patti LaBelle -- Goes Nuclear On Stripping Fan',
- 'timestamp': 1442683746,
- 'upload_date': '20150919',
- 'uploader_id': 'batchUser',
- }
- }, {
- 'url': 'http://www.tmz.com/2016/01/28/adam-silver-sting-drake-blake-griffin/',
- 'info_dict': {
- 'id': '0_ytz87kk7',
- 'ext': 'mp4',
- 'title': "NBA's Adam Silver -- Blake Griffin's a Great Guy ... He'll Learn from This",
- 'timestamp': 1454010989,
- 'upload_date': '20160128',
- 'uploader_id': 'batchUser',
- }
- }, {
- 'url': 'http://www.tmz.com/2016/10/27/donald-trump-star-vandal-arrested-james-otis/',
- 'info_dict': {
- 'id': '0_isigfatu',
- 'ext': 'mp4',
- 'title': "Trump Star Vandal -- I'm Not Afraid of Donald or the Cops!",
- 'timestamp': 1477500095,
- 'upload_date': '20161026',
- 'uploader_id': 'batchUser',
- }
- }]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- webpage = self._download_webpage(url, video_id)
- params = self._html_search_regex(r'TMZ.actions.clickLink\(([\s\S]+?)\)',
- webpage, 'embedded video info').split(',')
- new_url = params[0].strip("'\"")
- if new_url != url:
- return self.url_result(new_url)
+ webpage = self._download_webpage(url, url)
+ jsonld = self._search_json_ld(webpage, url)
+ if id not in jsonld:
+ jsonld["id"] = url
+ return jsonld
From d71eb83b057d4933c3a0c655951ea4ad7a36c132 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Fernando=20Rodr=C3=ADguez=20Var=C3=B3n?=
<diegorodriguezv@gmail.com>
Date: Thu, 19 Nov 2020 23:51:43 -0500
Subject: [PATCH 3/3] Extract embedded youtube and twitter videos
---
youtube_dlc/extractor/tmz.py | 50 ++++++++++++++++++++++++++++++++++++
1 file changed, 50 insertions(+)
diff --git a/youtube_dlc/extractor/tmz.py b/youtube_dlc/extractor/tmz.py
index a2f1009224..aee2273b82 100644
--- a/youtube_dlc/extractor/tmz.py
+++ b/youtube_dlc/extractor/tmz.py
@@ -1,7 +1,13 @@
# coding: utf-8
from __future__ import unicode_literals
+import re
+
from .common import InfoExtractor
+from ..utils import (
+ ExtractorError,
+ get_element_by_attribute,
+)
class TMZIE(InfoExtractor):
@@ -97,11 +103,55 @@ class TMZIE(InfoExtractor):
"upload_date": "20201031",
},
},
+ {
+ "url": "https://www.tmz.com/2020/11/05/gervonta-davis-car-crash-hit-and-run-police/",
+ "info_dict": {
+ "id": "Dddb6IGe-ws",
+ "ext": "mp4",
+ "title": "SICK LAMBO GERVONTA DAVIS IN HIS NEW RIDE RIGHT AFTER KO AFTER LEO EsNews Boxing",
+ "uploader": "ESNEWS",
+ "description": "md5:49675bc58883ccf80474b8aa701e1064",
+ "upload_date": "20201101",
+ "uploader_id": "ESNEWS",
+ },
+ },
+ {
+ "url": "https://www.tmz.com/2020/11/19/conor-mcgregor-dustin-poirier-contract-fight-ufc-257-fight-island/",
+ "info_dict": {
+ "id": "1329450007125225473",
+ "ext": "mp4",
+ "title": "TheMacLife - BREAKING: Conor McGregor (@thenotoriousmma) has signed his bout agreement for his rematch with Dustin Poirier for January 23.",
+ "uploader": "TheMacLife",
+ "description": "md5:56e6009bbc3d12498e10d08a8e1f1c69",
+ "upload_date": "20201119",
+ "uploader_id": "Maclifeofficial",
+ "timestamp": 1605800556,
+ },
+ },
]
def _real_extract(self, url):
webpage = self._download_webpage(url, url)
jsonld = self._search_json_ld(webpage, url)
+ if not jsonld or "url" not in jsonld:
+ # try to extract from YouTube Player API
+ # see https://developers.google.com/youtube/iframe_api_reference#Video_Queueing_Functions
+ match_obj = re.search(r'\.cueVideoById\(\s*(?P<quote>[\'"])(?P<id>.*?)(?P=quote)', webpage)
+ if match_obj:
+ res = self.url_result(match_obj.group("id"))
+ return res
+ # try to extract from twitter
+ blockquote_el = get_element_by_attribute("class", "twitter-tweet", webpage)
+ if blockquote_el:
+ matches = re.findall(
+ r'<a[^>]+href=\s*(?P<quote>[\'"])(?P<link>.*?)(?P=quote)',
+ blockquote_el)
+ if matches:
+ for _, match in matches:
+ if "/status/" in match:
+ res = self.url_result(match)
+ return res
+ raise ExtractorError("No video found!")
if id not in jsonld:
jsonld["id"] = url
return jsonld