[heise] Improve extraction (closes #15496, closes #15784, closes #15026)

This commit is contained in:
Sergey M․ 2018-03-15 22:37:41 +07:00
parent 27b1c73f14
commit 8e70c1bfac
No known key found for this signature in database
GPG Key ID: 2C393E0F18A9236D
1 changed files with 54 additions and 22 deletions

View File

@ -7,6 +7,7 @@ from .youtube import YoutubeIE
from ..utils import ( from ..utils import (
determine_ext, determine_ext,
int_or_none, int_or_none,
NO_DEFAULT,
parse_iso8601, parse_iso8601,
smuggle_url, smuggle_url,
xpath_text, xpath_text,
@ -16,18 +17,19 @@ from ..utils import (
class HeiseIE(InfoExtractor): class HeiseIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?heise\.de/(?:[^/]+/)+[^/]+-(?P<id>[0-9]+)\.html' _VALID_URL = r'https?://(?:www\.)?heise\.de/(?:[^/]+/)+[^/]+-(?P<id>[0-9]+)\.html'
_TESTS = [{ _TESTS = [{
# kaltura embed
'url': 'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html', 'url': 'http://www.heise.de/video/artikel/Podcast-c-t-uplink-3-3-Owncloud-Tastaturen-Peilsender-Smartphone-2404147.html',
'md5': 'ffed432483e922e88545ad9f2f15d30e',
'info_dict': { 'info_dict': {
'id': '2404147', 'id': '1_kkrq94sm',
'ext': 'mp4', 'ext': 'mp4',
'title': "Podcast: c't uplink 3.3 Owncloud / Tastaturen / Peilsender Smartphone", 'title': "Podcast: c't uplink 3.3 Owncloud / Tastaturen / Peilsender Smartphone",
'format_id': 'mp4_720p', 'timestamp': 1512734959,
'timestamp': 1411812600, 'upload_date': '20171208',
'upload_date': '20140927',
'description': 'md5:c934cbfb326c669c2bcabcbe3d3fcd20', 'description': 'md5:c934cbfb326c669c2bcabcbe3d3fcd20',
'thumbnail': r're:^https?://.*/gallery/$', },
} 'params': {
'skip_download': True,
},
}, { }, {
# YouTube embed # YouTube embed
'url': 'http://www.heise.de/newsticker/meldung/Netflix-In-20-Jahren-vom-Videoverleih-zum-TV-Revolutionaer-3814130.html', 'url': 'http://www.heise.de/newsticker/meldung/Netflix-In-20-Jahren-vom-Videoverleih-zum-TV-Revolutionaer-3814130.html',
@ -46,13 +48,26 @@ class HeiseIE(InfoExtractor):
}, },
}, { }, {
'url': 'https://www.heise.de/video/artikel/nachgehakt-Wie-sichert-das-c-t-Tool-Restric-tor-Windows-10-ab-3700244.html', 'url': 'https://www.heise.de/video/artikel/nachgehakt-Wie-sichert-das-c-t-Tool-Restric-tor-Windows-10-ab-3700244.html',
'md5': '4b58058b46625bdbd841fc2804df95fc',
'info_dict': { 'info_dict': {
'id': '1_ntrmio2s', 'id': '1_ntrmio2s',
'ext': 'mp4',
'title': "nachgehakt: Wie sichert das c't-Tool Restric'tor Windows 10 ab?",
'description': 'md5:47e8ffb6c46d85c92c310a512d6db271',
'timestamp': 1512470717, 'timestamp': 1512470717,
'upload_date': '20171205', 'upload_date': '20171205',
},
'params': {
'skip_download': True,
},
}, {
'url': 'https://www.heise.de/ct/artikel/c-t-uplink-20-8-Staubsaugerroboter-Xiaomi-Vacuum-2-AR-Brille-Meta-2-und-Android-rooten-3959893.html',
'info_dict': {
'id': '1_59mk80sf',
'ext': 'mp4', 'ext': 'mp4',
'title': 'ct10 nachgehakt hos restrictor', 'title': "c't uplink 20.8: Staubsaugerroboter Xiaomi Vacuum 2, AR-Brille Meta 2 und Android rooten",
'description': 'md5:f50fe044d3371ec73a8f79fcebd74afc',
'timestamp': 1517567237,
'upload_date': '20180202',
}, },
'params': { 'params': {
'skip_download': True, 'skip_download': True,
@ -72,19 +87,40 @@ class HeiseIE(InfoExtractor):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) webpage = self._download_webpage(url, video_id)
title = self._html_search_meta('fulltitle', webpage, default=None) def extract_title(default=NO_DEFAULT):
if not title or title == "c't": title = self._html_search_meta(
title = self._search_regex( ('fulltitle', 'title'), webpage, default=None)
r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"', if not title or title == "c't":
webpage, 'title') title = self._search_regex(
r'<div[^>]+class="videoplayerjw"[^>]+data-title="([^"]+)"',
webpage, 'title', default=None)
if not title:
title = self._html_search_regex(
r'<h1[^>]+\bclass=["\']article_page_title[^>]+>(.+?)<',
webpage, 'title', default=default)
return title
yt_urls = YoutubeIE._extract_urls(webpage) title = extract_title(default=None)
if yt_urls: description = self._og_search_description(
return self.playlist_from_matches(yt_urls, video_id, title, ie=YoutubeIE.ie_key()) webpage, default=None) or self._html_search_meta(
'description', webpage)
kaltura_url = KalturaIE._extract_url(webpage) kaltura_url = KalturaIE._extract_url(webpage)
if kaltura_url: if kaltura_url:
return self.url_result(smuggle_url(kaltura_url, {'source_url': url}), KalturaIE.ie_key()) return {
'_type': 'url_transparent',
'url': smuggle_url(kaltura_url, {'source_url': url}),
'ie_key': KalturaIE.ie_key(),
'title': title,
'description': description,
}
yt_urls = YoutubeIE._extract_urls(webpage)
if yt_urls:
return self.playlist_from_matches(
yt_urls, video_id, title, ie=YoutubeIE.ie_key())
title = extract_title()
container_id = self._search_regex( container_id = self._search_regex(
r'<div class="videoplayerjw"[^>]+data-container="([0-9]+)"', r'<div class="videoplayerjw"[^>]+data-container="([0-9]+)"',
@ -115,10 +151,6 @@ class HeiseIE(InfoExtractor):
}) })
self._sort_formats(formats) self._sort_formats(formats)
description = self._og_search_description(
webpage, default=None) or self._html_search_meta(
'description', webpage)
return { return {
'id': video_id, 'id': video_id,
'title': title, 'title': title,