From 245524e6a3782efccb27f35d1f75bf9b63fe36c6 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Wed, 21 Jul 2021 23:32:21 +0530 Subject: [PATCH] Release 2021.07.21 and fix some typos Closes #538 --- .gitignore | 1 + CONTRIBUTORS | 5 +++ Changelog.md | 63 +++++++++++++++++++++++++++++++++++++ README.md | 11 ++++--- supportedsites.md | 2 ++ yt_dlp/YoutubeDL.py | 2 +- yt_dlp/extractor/youtube.py | 1 + 7 files changed, 79 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index b6431b766..711bffaba 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ cookies.txt *.info.json *.live_chat.json *.jpg +*.jpeg *.png *.webp *.annotations.xml diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 50f0fe739..f0d00068c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -58,3 +58,8 @@ krichbanana ohmybahgosh nyuszika7h blackjack4494 +pyx +TpmKranz +mzbaulhaque +zackmark29 +mbway diff --git a/Changelog.md b/Changelog.md index d8e818b65..e3c5eb73c 100644 --- a/Changelog.md +++ b/Changelog.md @@ -18,6 +18,69 @@ # Instuctions for creating release --> +### 2021.07.21 + +* **Add option `--cookies-from-browser`** to load cookies from a browser by [mbway](https://github.com/mbway) + * Usage: `--cookies-from-browser BROWSER[:PROFILE_NAME_OR_PATH]` + * Also added `--no-cookies-from-browser` + * To decrypt chromium cookies, `keyring` is needed for UNIX and `pycryptodome` for Windows +* Add option `--exec-before-download` +* Add field `live_status` +* [FFmpegMetadata] Add language of each stream and some refactoring +* [douyin] Add extractor by [pukkandan](https://github.com/pukkandan), [pyx](https://github.com/pyx) +* [pornflip] Add extractor by [mzbaulhaque](https://github.com/mzbaulhaque) +* **[youtube] Extract data from multiple clients** by [pukkandan](https://github.com/pukkandan), [colethedj](https://github.com/colethedj) + * `player_client` now accepts multiple clients + * Default `player_client` = `android,web` + * This uses twice as many requests, but avoids throttling for most videos while also not losing any formats + * Music clients can be specifically requested and is enabled by default if `music.youtube.com` + * Added `player_client=ios` (Known issue: formats from ios are not sorted correctly) + * Add age-gate bypass for android and ios clients +* [youtube] Extract more thumbnails + * The thumbnail URLs are hard-coded and their actual existence is tested lazily + * Added option `--no-check-formats` to not test them +* [youtube] Misc fixes + * Improve extraction of livestream metadata by [pukkandan](https://github.com/pukkandan), [krichbanana](https://github.com/krichbanana) + * Hide live dash formats since they can't be downloaded anyway + * Fix authentication when using multiple accounts by [colethedj](https://github.com/colethedj) + * Fix controversial videos when requested via API by [colethedj](https://github.com/colethedj) + * Fix session index extraction and headers for non-web player clients by [colethedj](https://github.com/colethedj) + * Make `--extractor-retries` work for more errors + * Fix sorting of 3gp format + * Sanity check `chapters` (and refactor related code) + * Make `parse_time_text` and `_extract_chapters` non-fatal + * Misc cleanup and bug fixes by [colethedj](https://github.com/colethedj) +* [youtube:tab] Fix channels tab +* [youtube:tab] Extract playlist availability by [colethedj](https://github.com/colethedj) +* **[youtube:comments] Move comment extraction to new API** by [colethedj](https://github.com/colethedj) +* [youtube:comments] Fix `is_favorited`, improve `like_count` parsing by [colethedj](https://github.com/colethedj) +* [BravoTV] Improve metadata extraction by [kevinoconnor7](https://github.com/kevinoconnor7) +* [crunchyroll:playlist] Force http +* [yahoo:gyao:player] Relax `_VALID_URL` by [nao20010128nao](https://github.com/nao20010128nao) +* [nebula] Authentication via tokens from cookie jar by [hheimbuerger](https://github.com/hheimbuerger), [TpmKranz](https://github.com/TpmKranz) +* [RTP] Fix extraction and add subtitles by [fstirlitz](https://github.com/fstirlitz) +* [viki] Rewrite extractors and add extractor-arg `video_types` to `vikichannel` by [zackmark29](https://github.com/zackmark29), [pukkandan](https://github.com/pukkandan) +* [vlive] Extract thumbnail directly in addition to the one from Naver +* [generic] Extract previously missed subtitles by [fstirlitz](https://github.com/fstirlitz) +* [generic] Extract everything in the SMIL manifest and detect discarded subtitles by [fstirlitz](https://github.com/fstirlitz) +* [embedthumbnail] Fix `_get_thumbnail_resolution` +* [metadatafromfield] Do not detect numbers as field names +* Fix selectors `all`, `mergeall` and add tests +* Errors in playlist extraction should obey `--ignore-errors` +* Fix bug where `original_url` was not propagated when `_type`=`url` +* Revert "Merge webm formats into mkv if thumbnails are to be embedded (#173)" + * This was wrongly checking for `write_thumbnail` +* Improve `extractor_args` parsing +* Rename `NOTE` in `-F` to `MORE INFO` since it's often confused to be the same as `format_note` +* Add `only_once` param for `write_debug` and `report_warning` +* [extractor] Allow extracting multiple groups in `_search_regex` by [fstirlitz](https://github.com/fstirlitz) +* [utils] Improve `traverse_obj` +* [utils] Add `variadic` +* [utils] Improve `js_to_json` comment regex by [fstirlitz](https://github.com/fstirlitz) +* [webtt] Fix timestamps +* [compat] Remove unnecessary code +* [doc] fix default of multistreams + ### 2021.07.07 diff --git a/README.md b/README.md index f3a0a5a60..d879dc4c4 100644 --- a/README.md +++ b/README.md @@ -75,19 +75,22 @@ # NEW FEATURES * All Feeds (`:ytfav`, `:ytwatchlater`, `:ytsubs`, `:ythistory`, `:ytrec`) supports downloading multiple pages of content * Search (`ytsearch:`, `ytsearchdate:`), search URLs and in-channel search works * Mixes supports downloading multiple pages of content + * Partial workarounds for age-gate and throttling issues * Redirect channel's home URL automatically to `/video` to preserve the old behaviour * `255kbps` audio is extracted from youtube music if premium cookies are given * Youtube music Albums, channels etc can be downloaded +* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser BROWSER[:PROFILE]` + * **Split video by chapters**: Videos can be split into multiple files based on chapters using `--split-chapters` * **Multi-threaded fragment downloads**: Download multiple fragments of m3u8/mpd videos in parallel. Use `--concurrent-fragments` (`-N`) option to set the number of threads used * **Aria2c with HLS/DASH**: You can use `aria2c` as the external downloader for DASH(mpd) and HLS(m3u8) formats -* **New extractors**: AnimeLab, Philo MSO, Spectrum MSO, Rcs, Gedi, bitwave.tv, mildom, audius, zee5, mtv.it, wimtv, pluto.tv, niconico users, discoveryplus.in, mediathek, NFHSNetwork, nebula, ukcolumn, whowatch, MxplayerShow, parlview (au), YoutubeWebArchive, fancode, Saitosan, ShemarooMe, telemundo, VootSeries, SonyLIVSeries, HotstarSeries, VidioPremier, VidioLive, RCTIPlus, TBS Live +* **New extractors**: AnimeLab, Philo MSO, Spectrum MSO, Rcs, Gedi, bitwave.tv, mildom, audius, zee5, mtv.it, wimtv, pluto.tv, niconico users, discoveryplus.in, mediathek, NFHSNetwork, nebula, ukcolumn, whowatch, MxplayerShow, parlview (au), YoutubeWebArchive, fancode, Saitosan, ShemarooMe, telemundo, VootSeries, SonyLIVSeries, HotstarSeries, VidioPremier, VidioLive, RCTIPlus, TBS Live, douyin, pornflip -* **Fixed extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, akamai, ina, rumble, tennistv, amcnetworks, la7 podcasts, linuxacadamy, nitter, twitcasting, viu, crackle, curiositystream, mediasite, rmcdecouverte, sonyliv, tubi, tenplay, patreon, videa, yahoo +* **Fixed extractors**: archive.org, roosterteeth.com, skyit, instagram, itv, SouthparkDe, spreaker, Vlive, akamai, ina, rumble, tennistv, amcnetworks, la7 podcasts, linuxacadamy, nitter, twitcasting, viu, crackle, curiositystream, mediasite, rmcdecouverte, sonyliv, tubi, tenplay, patreon, videa, yahoo, BravoTV, crunchyroll playlist, RTP, viki * **Subtitle extraction from manifests**: Subtitles can be extracted from streaming media manifests. See [commit/be6202f](https://github.com/yt-dlp/yt-dlp/commit/be6202f12b97858b9d716e608394b51065d0419f) for details @@ -105,8 +108,6 @@ # NEW FEATURES * **Self-updater**: The releases can be updated using `yt-dlp -U` -* **Cookies from browser**: Cookies can be automatically extracted from all major web browsers using `--cookies-from-browser ` - See [changelog](Changelog.md) or [commits](https://github.com/yt-dlp/yt-dlp/commits) for the full list of changes @@ -841,7 +842,7 @@ ## Extractor Options: --extractor-args KEY:ARGS Pass these arguments to the extractor. See "EXTRACTOR ARGUMENTS" for details. You can use this option multiple times to give - different arguments to different extractors + arguments for different extractors # CONFIGURATION diff --git a/supportedsites.md b/supportedsites.md index 68d7ec5c3..3e386bcfa 100644 --- a/supportedsites.md +++ b/supportedsites.md @@ -259,6 +259,7 @@ # Supported sites - **dlive:vod** - **DoodStream** - **Dotsub** + - **Douyin** - **DouyuShow** - **DouyuTV**: 斗鱼 - **DPlay** @@ -769,6 +770,7 @@ # Supported sites - **PopcornTV** - **PornCom** - **PornerBros** + - **PornFlip** - **PornHd** - **PornHub**: PornHub and Thumbzilla - **PornHubPagedVideoList** diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py index 494c0d33b..2a7c05374 100644 --- a/yt_dlp/YoutubeDL.py +++ b/yt_dlp/YoutubeDL.py @@ -255,7 +255,7 @@ class YoutubeDL(object): writedesktoplink: Write a Linux internet shortcut file (.desktop) writesubtitles: Write the video subtitles to a file writeautomaticsub: Write the automatically generated subtitles to a file - allsubtitles: Deprecated - Use subtitlelangs = ['all'] + allsubtitles: Deprecated - Use subtitleslangs = ['all'] Downloads all the subtitles of the video (requires writesubtitles or writeautomaticsub) listsubtitles: Lists all available subtitles for the video diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube.py index 49cb8a233..7a1d39ac8 100644 --- a/yt_dlp/extractor/youtube.py +++ b/yt_dlp/extractor/youtube.py @@ -2769,6 +2769,7 @@ def feed_entry(name): # See: https://github.com/ytdl-org/youtube-dl/issues/29049, https://github.com/yt-dlp/yt-dlp/issues/340 # List of possible thumbnails - Ref: hq_thumbnail_names = ['maxresdefault', 'hq720', 'sddefault', 'sd1', 'sd2', 'sd3'] + # TODO: Test them also? - For some videos, even these don't exist guaranteed_thumbnail_names = [ 'hqdefault', 'hq1', 'hq2', 'hq3', '0', 'mqdefault', 'mq1', 'mq2', 'mq3',