From 0de09c5b9ed619d4a93d7c451c6ddff0381de808 Mon Sep 17 00:00:00 2001 From: sepro <4618135+seproDev@users.noreply.github.com> Date: Sat, 24 Feb 2024 17:08:47 +0100 Subject: [PATCH] [ie/nebula] Support podcasts (#9140) Closes #8838 Authored by: seproDev, c-basalt Co-authored-by: c-basalt <117849907+c-basalt@users.noreply.github.com> --- yt_dlp/extractor/nebula.py | 105 +++++++++++++++++++++++++++++++++---- 1 file changed, 95 insertions(+), 10 deletions(-) diff --git a/yt_dlp/extractor/nebula.py b/yt_dlp/extractor/nebula.py index 136b0e10a..cb8f6a67d 100644 --- a/yt_dlp/extractor/nebula.py +++ b/yt_dlp/extractor/nebula.py @@ -1,6 +1,7 @@ import itertools import json +from .art19 import Art19IE from .common import InfoExtractor from ..networking.exceptions import HTTPError from ..utils import ( @@ -112,7 +113,8 @@ class NebulaBaseIE(InfoExtractor): class NebulaIE(NebulaBaseIE): - _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P[-\w]+)' + IE_NAME = 'nebula:video' + _VALID_URL = rf'{_BASE_URL_RE}/videos/(?P[\w-]+)' _TESTS = [{ 'url': 'https://nebula.tv/videos/that-time-disney-remade-beauty-and-the-beast', 'info_dict': { @@ -236,8 +238,8 @@ class NebulaIE(NebulaBaseIE): class NebulaClassIE(NebulaBaseIE): - IE_NAME = 'nebula:class' - _VALID_URL = rf'{_BASE_URL_RE}/(?P[-\w]+)/(?P\d+)' + IE_NAME = 'nebula:media' + _VALID_URL = rf'{_BASE_URL_RE}/(?!(?:myshows|library|videos)/)(?P[\w-]+)/(?P[\w-]+)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/copyright-for-fun-and-profit/14', 'info_dict': { @@ -253,6 +255,46 @@ class NebulaClassIE(NebulaBaseIE): 'title': 'Photos, Sculpture, and Video', }, 'params': {'skip_download': 'm3u8'}, + }, { + 'url': 'https://nebula.tv/extremitiespodcast/pyramiden-the-high-arctic-soviet-ghost-town', + 'info_dict': { + 'ext': 'mp3', + 'id': '018f65f0-0033-4021-8f87-2d132beb19aa', + 'description': 'md5:05d2b23ab780c955e2511a2b9127acff', + 'series_id': '335e8159-d663-491a-888f-1732285706ac', + 'modified_timestamp': 1599091504, + 'episode_id': '018f65f0-0033-4021-8f87-2d132beb19aa', + 'series': 'Extremities', + 'modified_date': '20200903', + 'upload_date': '20200902', + 'title': 'Pyramiden: The High-Arctic Soviet Ghost Town', + 'release_timestamp': 1571237958, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$', + 'duration': 1546.05714, + 'timestamp': 1599085608, + 'release_date': '20191016', + }, + }, { + 'url': 'https://nebula.tv/thelayover/the-layover-episode-1', + 'info_dict': { + 'ext': 'mp3', + 'id': '9d74a762-00bb-45a8-9e8d-9ed47c04a1d0', + 'episode_number': 1, + 'thumbnail': r're:^https?://content\.production\.cdn\.art19\.com.*\.jpeg$', + 'release_date': '20230304', + 'modified_date': '20230403', + 'series': 'The Layover', + 'episode_id': '9d74a762-00bb-45a8-9e8d-9ed47c04a1d0', + 'modified_timestamp': 1680554566, + 'duration': 3130.46401, + 'release_timestamp': 1677943800, + 'title': 'The Layover — Episode 1', + 'series_id': '874303a5-4900-4626-a4b6-2aacac34466a', + 'upload_date': '20230303', + 'episode': 'Episode 1', + 'timestamp': 1677883672, + 'description': 'md5:002cca89258e3bc7c268d5b8c24ba482', + }, }] def _real_extract(self, url): @@ -268,16 +310,38 @@ class NebulaClassIE(NebulaBaseIE): metadata = self._call_api( f'https://content.api.nebula.app/content/{slug}/{episode}/?include=lessons', - slug, note='Fetching video metadata') - return { - **self._extract_video_metadata(metadata), - **self._extract_formats(metadata['id'], slug), - } + slug, note='Fetching class/podcast metadata') + content_type = metadata.get('type') + if content_type == 'lesson': + return { + **self._extract_video_metadata(metadata), + **self._extract_formats(metadata['id'], slug), + } + elif content_type == 'podcast_episode': + episode_url = metadata['episode_url'] + if not episode_url and metadata.get('premium'): + self.raise_login_required() + + if Art19IE.suitable(episode_url): + return self.url_result(episode_url, Art19IE) + return traverse_obj(metadata, { + 'id': ('id', {str}), + 'url': ('episode_url', {url_or_none}), + 'title': ('title', {str}), + 'description': ('description', {str}), + 'timestamp': ('published_at', {parse_iso8601}), + 'duration': ('duration', {int_or_none}), + 'channel_id': ('channel_id', {str}), + 'chnanel': ('channel_title', {str}), + 'thumbnail': ('assets', 'regular', {url_or_none}), + }) + + raise ExtractorError(f'Unexpected content type {content_type!r}') class NebulaSubscriptionsIE(NebulaBaseIE): IE_NAME = 'nebula:subscriptions' - _VALID_URL = rf'{_BASE_URL_RE}/(?Pmyshows|library/latest-videos)' + _VALID_URL = rf'{_BASE_URL_RE}/(?Pmyshows|library/latest-videos)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/myshows', 'playlist_mincount': 1, @@ -310,7 +374,7 @@ class NebulaSubscriptionsIE(NebulaBaseIE): class NebulaChannelIE(NebulaBaseIE): IE_NAME = 'nebula:channel' - _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|library|videos/)(?P[-\w]+)/?(?:$|[?#])' + _VALID_URL = rf'{_BASE_URL_RE}/(?!myshows|library|videos)(?P[\w-]+)/?(?:$|[?#])' _TESTS = [{ 'url': 'https://nebula.tv/tom-scott-presents-money', 'info_dict': { @@ -343,6 +407,14 @@ class NebulaChannelIE(NebulaBaseIE): 'description': 'md5:6690248223eed044a9f11cd5a24f9742', }, 'playlist_count': 23, + }, { + 'url': 'https://nebula.tv/trussissuespodcast', + 'info_dict': { + 'id': 'trussissuespodcast', + 'title': 'The TLDR News Podcast', + 'description': 'md5:a08c4483bc0b705881d3e0199e721385', + }, + 'playlist_mincount': 80, }] def _generate_playlist_entries(self, collection_id, collection_slug): @@ -365,6 +437,17 @@ class NebulaChannelIE(NebulaBaseIE): lesson.get('share_url') or f'https://nebula.tv/{metadata["class_slug"]}/{metadata["slug"]}', {'id': lesson['id']}), NebulaClassIE, url_transparent=True, **metadata) + def _generate_podcast_entries(self, collection_id, collection_slug): + next_url = f'https://content.api.nebula.app/podcast_channels/{collection_id}/podcast_episodes/?ordering=-published_at&premium=true' + for page_num in itertools.count(1): + episodes = self._call_api(next_url, collection_slug, note=f'Retrieving podcast page {page_num}') + + for episode in traverse_obj(episodes, ('results', lambda _, v: url_or_none(v['share_url']))): + yield self.url_result(episode['share_url'], NebulaClassIE) + next_url = episodes.get('next') + if not next_url: + break + def _real_extract(self, url): collection_slug = self._match_id(url) channel = self._call_api( @@ -373,6 +456,8 @@ class NebulaChannelIE(NebulaBaseIE): if channel.get('type') == 'class': entries = self._generate_class_entries(channel) + elif channel.get('type') == 'podcast_channel': + entries = self._generate_podcast_entries(channel['id'], collection_slug) else: entries = self._generate_playlist_entries(channel['id'], collection_slug)