From e89d7e30294d3c5a36e6af5dd730ed543934d40e Mon Sep 17 00:00:00 2001 From: Roland Hieber Date: Sun, 19 Jul 2015 03:34:22 +0200 Subject: [PATCH 1/4] [tagesschau] add support for more video types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I found that currently only tagesschau videos are played. There are some more shows hosted on tagesschau.de (see [0] for example) which are easily playable by adjusting the regex. So this patch adds support for: * tagesthemen * tagesschau vor 20 Jahren * tagesschau (mit Gebärdensprache) * nachtmagazin Note that some videos don't provide a description, so in order for the tests to succeed, an ExtractorError needs to get caught. [0]: http://www.tagesschau.de/multimedia/video/videoarchiv2~_date-20150714.html --- youtube_dl/extractor/tagesschau.py | 55 +++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py index bfe07b024..2a2aafca0 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dl/extractor/tagesschau.py @@ -4,11 +4,11 @@ import re from .common import InfoExtractor -from ..utils import parse_filesize +from ..utils import parse_filesize, ExtractorError class TagesschauIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/ts|video/video)(?P-?[0-9]+)\.html' + _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P-?[0-9]+)\.html' _TESTS = [{ 'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html', @@ -30,6 +30,46 @@ class TagesschauIE(InfoExtractor): 'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr', 'thumbnail': 're:^http:.*\.jpg$', } + }, { + 'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html', + 'md5': '90757268b49ef56deae90c7b48928d58', + 'info_dict': { + 'id': '3771', + 'ext': 'mp4', + 'description': '', + 'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr', + 'thumbnail': 're:^http:.*\.jpg$', + } + }, { + 'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html', + 'md5': '6e3ebdc75e8d67da966a8d06721eda71', + 'info_dict': { + 'id': '3827', + 'ext': 'mp4', + 'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66', + 'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr', + 'thumbnail': 're:^http:.*\.jpg$', + } + }, { + 'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html', + 'md5': '8a8875a568f0a5ae5ceef93c501a225f', + 'info_dict': { + 'id': '3475', + 'ext': 'mp4', + 'description': 'md5:ed149f5649cda3dac86813a9d777e131', + 'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr', + 'thumbnail': 're:^http:.*\.jpg$', + } + }, { + 'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html', + 'md5': 'be4d6f0421f2acd8abe25ea29f6f015b', + 'info_dict': { + 'id': '959', + 'ext': 'mp4', + 'description': '', + 'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr', + 'thumbnail': 're:^http:.*\.jpg$', + } }] _FORMATS = { @@ -102,9 +142,14 @@ def _real_extract(self, url): thumbnail_fn = self._search_regex( r'(?s)Sendungsbild(.*?)

', - webpage, 'description', fatal=False) + # there are some videos without description + description = "" + try: + description = self._html_search_regex( + r'(?s)

(.*?)

', + webpage, 'description', fatal=False) + except ExtractorError: + pass title = self._html_search_regex( r'(.*?)', webpage, 'title') From 3c6ae8b59ed2517891ff19507e4cef2322ef9eb4 Mon Sep 17 00:00:00 2001 From: Roland Hieber Date: Sun, 19 Jul 2015 04:27:58 +0200 Subject: [PATCH 2/4] [tagesschau] add support for Bericht aus Berlin videos --- youtube_dl/extractor/tagesschau.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py index 2a2aafca0..682f8df8f 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dl/extractor/tagesschau.py @@ -8,7 +8,7 @@ class TagesschauIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm)|video/video|tsvorzwanzig)(?P-?[0-9]+)\.html' + _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html' _TESTS = [{ 'url': 'http://www.tagesschau.de/multimedia/video/video1399128.html', @@ -70,6 +70,16 @@ class TagesschauIE(InfoExtractor): 'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr', 'thumbnail': 're:^http:.*\.jpg$', } + }, { + 'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html', + 'md5': '42e3757018d9908581481a80cc1806da', + 'info_dict': { + 'id': '3299', + 'ext': 'mp4', + 'description': '', + 'title': 'Nach dem Referendum: Schaltgespräch nach Athen', + 'thumbnail': 're:^http:.*\.jpg$', + } }] _FORMATS = { From 726adc43ec184b6d7e05cdbc197aa02c53b32347 Mon Sep 17 00:00:00 2001 From: Roland Hieber Date: Sun, 19 Jul 2015 05:09:29 +0200 Subject: [PATCH 3/4] [tagesschau] set description=None for empty descriptions --- youtube_dl/extractor/tagesschau.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py index 682f8df8f..4a755c657 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dl/extractor/tagesschau.py @@ -36,7 +36,7 @@ class TagesschauIE(InfoExtractor): 'info_dict': { 'id': '3771', 'ext': 'mp4', - 'description': '', + 'description': None, 'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr', 'thumbnail': 're:^http:.*\.jpg$', } @@ -66,7 +66,7 @@ class TagesschauIE(InfoExtractor): 'info_dict': { 'id': '959', 'ext': 'mp4', - 'description': '', + 'description': None, 'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr', 'thumbnail': 're:^http:.*\.jpg$', } @@ -76,7 +76,7 @@ class TagesschauIE(InfoExtractor): 'info_dict': { 'id': '3299', 'ext': 'mp4', - 'description': '', + 'description': None, 'title': 'Nach dem Referendum: Schaltgespräch nach Athen', 'thumbnail': 're:^http:.*\.jpg$', } @@ -154,12 +154,9 @@ def _real_extract(self, url): webpage, 'thumbnail', fatal=False) # there are some videos without description description = "" - try: - description = self._html_search_regex( - r'(?s)

(.*?)

', - webpage, 'description', fatal=False) - except ExtractorError: - pass + description = self._html_search_regex( + r'(?s)

(.*?)

', + webpage, 'description', fatal=False, default=None) title = self._html_search_regex( r'(.*?)', webpage, 'title') From 948199deac4d9ec6ba6bc4359ad43db7951a1c53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Thu, 23 Jul 2015 00:30:48 +0600 Subject: [PATCH 4/4] [tagesschau] Relax _VALID_URL and simplify --- youtube_dl/extractor/tagesschau.py | 62 ++++++++---------------------- 1 file changed, 17 insertions(+), 45 deletions(-) diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py index 636607db5..b84892364 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dl/extractor/tagesschau.py @@ -8,7 +8,7 @@ class TagesschauIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:sendung/(ts|tsg|tt|nm|bab/bab)|video/video|tsvorzwanzig)(?P-?[0-9]+)(?:~[-_a-zA-Z0-9]*)?\.html' + _VALID_URL = r'https?://(?:www\.)?tagesschau\.de/multimedia/(?:[^/]+/)*?[^/#?]+?(?P-?[0-9]+)(?:~_[^/#?]+?)?\.html' _TESTS = [{ 'url': 'http://www.tagesschau.de/multimedia/video/video-102143.html', @@ -29,57 +29,31 @@ class TagesschauIE(InfoExtractor): 'description': 'md5:695c01bfd98b7e313c501386327aea59', 'title': 'Sendung: tagesschau \t04.12.2014 20:00 Uhr', 'thumbnail': 're:^http:.*\.jpg$', - } + }, }, { 'url': 'http://www.tagesschau.de/multimedia/sendung/tsg-3771.html', - 'md5': '90757268b49ef56deae90c7b48928d58', - 'info_dict': { - 'id': '3771', - 'ext': 'mp4', - 'description': None, - 'title': 'Sendung: tagesschau (mit Gebärdensprache) \t14.07.2015 20:00 Uhr', - 'thumbnail': 're:^http:.*\.jpg$', - } + 'only_matching': True, }, { 'url': 'http://www.tagesschau.de/multimedia/sendung/tt-3827.html', - 'md5': '6e3ebdc75e8d67da966a8d06721eda71', - 'info_dict': { - 'id': '3827', - 'ext': 'mp4', - 'description': 'md5:d511d0e278b0ad341a95ad9ab992ce66', - 'title': 'Sendung: tagesthemen \t14.07.2015 22:15 Uhr', - 'thumbnail': 're:^http:.*\.jpg$', - } + 'only_matching': True, }, { 'url': 'http://www.tagesschau.de/multimedia/sendung/nm-3475.html', - 'md5': '8a8875a568f0a5ae5ceef93c501a225f', - 'info_dict': { - 'id': '3475', - 'ext': 'mp4', - 'description': 'md5:ed149f5649cda3dac86813a9d777e131', - 'title': 'Sendung: nachtmagazin \t15.07.2015 00:15 Uhr', - 'thumbnail': 're:^http:.*\.jpg$', - } + 'only_matching': True, + }, { + 'url': 'http://www.tagesschau.de/multimedia/sendung/weltspiegel-3167.html', + 'only_matching': True, }, { 'url': 'http://www.tagesschau.de/multimedia/tsvorzwanzig-959.html', - 'md5': 'be4d6f0421f2acd8abe25ea29f6f015b', - 'info_dict': { - 'id': '959', - 'ext': 'mp4', - 'description': None, - 'title': 'Sendung: tagesschau vor 20 Jahren \t14.07.2015 22:45 Uhr', - 'thumbnail': 're:^http:.*\.jpg$', - } + 'only_matching': True, }, { 'url': 'http://www.tagesschau.de/multimedia/sendung/bab/bab-3299~_bab-sendung-209.html', - 'md5': '42e3757018d9908581481a80cc1806da', - 'info_dict': { - 'id': '3299', - 'ext': 'mp4', - 'description': None, - 'title': 'Nach dem Referendum: Schaltgespräch nach Athen', - 'thumbnail': 're:^http:.*\.jpg$', - } + 'only_matching': True, + }, { + 'url': 'http://www.tagesschau.de/multimedia/video/video-102303~_bab-sendung-211.html', + 'only_matching': True, + }, { + 'url': 'http://www.tagesschau.de/multimedia/politikimradio/audio-18407.html', + 'only_matching': True, }] _FORMATS = { @@ -152,11 +126,9 @@ def _real_extract(self, url): thumbnail_fn = self._search_regex( r'(?s)Sendungsbild(.*?)

', - webpage, 'description', fatal=False, default=None) + webpage, 'description', default=None) title = self._html_search_regex( r'(.*?)', webpage, 'title')