Merge pull request #6242 from dstftw/f4m-improvements

[f4m] Improvements

youtube_dl/extractor/common.py

@@ -27,7 +27,9 @@
     bug_reports_message,
     clean_html,
     compiled_regex_type,
+    determine_ext,
     ExtractorError,
+    fix_xml_ampersands,
     float_or_none,
     int_or_none,
     RegexNotFoundError,
@@ -837,7 +839,10 @@ def _sleep(self, timeout, video_id, msg_template=None):
     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None):
         manifest = self._download_xml(
             manifest_url, video_id, 'Downloading f4m manifest',
-            'Unable to download f4m manifest')
+            'Unable to download f4m manifest',
+            # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
+            # (see https://github.com/rg3/youtube-dl/issues/6215#issuecomment-121704244)
+            transform_source=lambda s: fix_xml_ampersands(s).strip())
 
         formats = []
         manifest_version = '1.0'
@@ -847,8 +852,19 @@ def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=N
             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
         for i, media_el in enumerate(media_nodes):
             if manifest_version == '2.0':
-                manifest_url = ('/'.join(manifest_url.split('/')[:-1]) + '/' +
+                media_url = media_el.attrib.get('href') or media_el.attrib.get('url')
-                                (media_el.attrib.get('href') or media_el.attrib.get('url')))
+                if not media_url:
+                    continue
+                manifest_url = (
+                    media_url if media_url.startswith('http://') or media_url.startswith('https://')
+                    else ('/'.join(manifest_url.split('/')[:-1]) + '/' + media_url))
+                # If media_url is itself a f4m manifest do the recursive extraction
+                # since bitrates in parent manifest (this one) and media_url manifest
+                # may differ leading to inability to resolve the format by requested
+                # bitrate in f4m downloader
+                if determine_ext(manifest_url) == 'f4m':
+                    formats.extend(self._extract_f4m_formats(manifest_url, video_id, preference, f4m_id))
+                    continue
             tbr = int_or_none(media_el.attrib.get('bitrate'))
             formats.append({
                 'format_id': '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)])),

youtube_dl/extractor/prosiebensat1.py

@@ -9,7 +9,7 @@
     compat_urllib_parse,
 )
 from ..utils import (
-    fix_xml_ampersands,
+    determine_ext,
     int_or_none,
     unified_strdate,
 )
@@ -22,6 +22,11 @@ class ProSiebenSat1IE(InfoExtractor):
 
     _TESTS = [
         {
+            # Tests changes introduced in https://github.com/rg3/youtube-dl/pull/6242
+            # in response to fixing https://github.com/rg3/youtube-dl/issues/6215:
+            # - malformed f4m manifest support
+            # - proper handling of URLs starting with `https?://` in 2.0 manifests
+            # - recursive child f4m manifests extraction
             'url': 'http://www.prosieben.de/tv/circus-halligalli/videos/218-staffel-2-episode-18-jahresrueckblick-ganze-folge',
             'info_dict': {
                 'id': '2104602',
@@ -295,15 +300,8 @@ def fix_bitrate(bitrate):
                     'ext': 'mp4',
                     'format_id': '%s_%s' % (source['cdn'], source['bitrate']),
                 })
-            elif 'f4mgenerator' in source_url:
+            elif 'f4mgenerator' in source_url or determine_ext(source_url) == 'f4m':
-                manifest = self._download_xml(
+                formats.extend(self._extract_f4m_formats(source_url, clip_id))
-                    source_url, clip_id, 'Downloading generated f4m manifest',
-                    transform_source=lambda s: fix_xml_ampersands(s).strip())
-                for media in manifest.findall('./{http://ns.adobe.com/f4m/2.0}media'):
-                    manifest_url = media.get('href')
-                    if manifest_url:
-                        formats.extend(self._extract_f4m_formats(
-                            manifest_url, clip_id, f4m_id='hds'))
             else:
                 formats.append({
                     'url': source_url,