MarcoBuster/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-08 22:17:07 +01:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into networking/websockets-http-proxy

Browse Source
This commit is contained in:
coletdjnz 2024-06-03 11:41:09 +12:00
commit 31c7479bef
No known key found for this signature in database
GPG Key ID: 91984263BB39894A
308 changed files with 4289 additions and 2146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -28,7 +28,6 @@ # PLEASE FOLLOW THE GUIDE BELOW
### Before submitting a *pull request* make sure you have:
- [ ] At least skimmed through [contributing guidelines](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions) including [yt-dlp coding conventions](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#yt-dlp-coding-conventions)
- [ ] [Searched](https://github.com/yt-dlp/yt-dlp/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests
- [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) and [ran relevant tests](https://github.com/yt-dlp/yt-dlp/blob/master/CONTRIBUTING.md#developer-instructions)
### In order to be accepted and merged into yt-dlp each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check all of the following options that apply:
- [ ] I am the original author of this code and I am willing to release it under [Unlicense](http://unlicense.org/)

42
.github/workflows/build.yml vendored
View File

@ -237,7 +237,7 @@ jobs:
macos:
needs: process
if: inputs.macos
runs-on: macos-11
runs-on: macos-12
steps:
- uses: actions/checkout@v4
@ -260,11 +260,23 @@ jobs:
--pre -d curl_cffi_whls \
-r requirements.txt
done
( # Overwrite x86_64-only libs with fat/universal2 libs or else Pyinstaller will do the opposite
# See https://github.com/yt-dlp/yt-dlp/pull/10069
cd curl_cffi_whls
mkdir -p curl_cffi/.dylibs
python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
for dylib in lib{ssl,crypto}.3.dylib; do
cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
for wheel in curl_cffi*macos*x86_64.whl; do
zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
done
done
)
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
cd curl_cffi_universal2
for wheel in *cffi*.whl; do mv -n -- "${wheel}" "${wheel/x86_64/universal2}"; done
python3 -m pip install -U --user *cffi*.whl
for wheel in ./*cffi*.whl; do mv -n -- "${wheel}" "${wheel/x86_64/universal2}"; done
python3 -m pip install -U --user ./*cffi*.whl
- name: Prepare
run: |
@ -311,7 +323,7 @@ jobs:
# Hack to get the latest patch version. Uncomment if needed
#brew install python@3.10
#export PYTHON_VERSION=$( $(brew --prefix)/opt/python@3.10/bin/python3 --version | cut -d ' ' -f 2 )
curl https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg -o "python.pkg"
curl "https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-macos11.pkg" -o "python.pkg"
sudo installer -pkg python.pkg -target /
python3 --version
- name: Install Requirements
@ -360,8 +372,8 @@ jobs:
- name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python devscripts/install_deps.py -o --include build
python devscripts/install_deps.py --include py2exe --include curl-cffi
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
python devscripts/install_deps.py --include curl-cffi
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-6.7.0-py3-none-any.whl"
- name: Prepare
run: |
@ -369,12 +381,20 @@ jobs:
python devscripts/make_lazy_extractors.py
- name: Build
run: |
python -m bundle.py2exe
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
python -m bundle.pyinstaller
python -m bundle.pyinstaller --onedir
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_real.exe
Compress-Archive -Path ./dist/yt-dlp/* -DestinationPath ./dist/yt-dlp_win.zip
- name: Install Requirements (py2exe)
run: |
python devscripts/install_deps.py --include py2exe
- name: Build (py2exe)
run: |
python -m bundle.py2exe
Move-Item ./dist/yt-dlp.exe ./dist/yt-dlp_min.exe
Move-Item ./dist/yt-dlp_real.exe ./dist/yt-dlp.exe
- name: Verify --update-to
if: vars.UPDATE_TO_VERIFICATION
run: |
@ -413,7 +433,7 @@ jobs:
run: |
python devscripts/install_deps.py -o --include build
python devscripts/install_deps.py
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-6.7.0-py3-none-any.whl"
- name: Prepare
run: |
@ -467,8 +487,8 @@ jobs:
run: |
cd ./artifact/
# make sure SHA sums are also printed to stdout
sha256sum * | tee ../SHA2-256SUMS
sha512sum * | tee ../SHA2-512SUMS
sha256sum -- * | tee ../SHA2-256SUMS
sha512sum -- * | tee ../SHA2-512SUMS
- name: Make Update spec
run: |

2
.github/workflows/core.yml vendored
View File

@ -53,7 +53,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
run: python3 ./devscripts/install_deps.py --include dev --include curl-cffi
run: python3 ./devscripts/install_deps.py --include test --include curl-cffi
- name: Run tests
continue-on-error: False
run: |

16
.github/workflows/quick-test.yml vendored
View File

@ -15,13 +15,13 @@ jobs:
with:
python-version: '3.8'
- name: Install test requirements
run: python3 ./devscripts/install_deps.py --include dev
run: python3 ./devscripts/install_deps.py --include test
- name: Run tests
run: |
python3 -m yt_dlp -v || true
python3 ./devscripts/run_tests.py core
flake8:
name: Linter
check:
name: Code check
if: "!contains(github.event.head_commit.message, 'ci skip all')"
runs-on: ubuntu-latest
steps:
@ -29,9 +29,11 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: '3.8'
- name: Install flake8
run: python3 ./devscripts/install_deps.py -o --include dev
- name: Install dev dependencies
run: python3 ./devscripts/install_deps.py -o --include static-analysis
- name: Make lazy extractors
run: python3 ./devscripts/make_lazy_extractors.py
- name: Run flake8
run: flake8 .
- name: Run ruff
run: ruff check --output-format github .
- name: Run autopep8
run: autopep8 --diff .

2
.gitignore vendored
View File

@ -67,7 +67,7 @@ cookies
# Python
*.pyc
*.pyo
.pytest_cache
.*_cache
wine-py2exe/
py2exe.log
build/

14
.pre-commit-config.yaml Normal file
View File

@ -0,0 +1,14 @@
repos:
- repo: local
hooks:
- id: linter
name: Apply linter fixes
entry: ruff check --fix .
language: system
types: [python]
require_serial: true
- id: format
name: Apply formatting fixes
entry: autopep8 --in-place .
language: system
types: [python]

9
.pre-commit-hatch.yaml Normal file
View File

@ -0,0 +1,9 @@
repos:
- repo: local
hooks:
- id: fix
name: Apply code fixes
entry: hatch fmt
language: system
types: [python]
require_serial: true

77
CONTRIBUTING.md
View File

@ -134,18 +134,53 @@ ### Is the website primarily used for piracy?
# DEVELOPER INSTRUCTIONS
Most users do not need to build yt-dlp and can [download the builds](https://github.com/yt-dlp/yt-dlp/releases) or get them via [the other installation methods](README.md#installation).
Most users do not need to build yt-dlp and can [download the builds](https://github.com/yt-dlp/yt-dlp/releases), get them via [the other installation methods](README.md#installation) or directly run it using `python -m yt_dlp`.
To run yt-dlp as a developer, you don't need to build anything either. Simply execute
`yt-dlp` uses [`hatch`](<https://hatch.pypa.io>) as a project management tool.
You can easily install it using [`pipx`](<https://pipx.pypa.io>) via `pipx install hatch`, or else via `pip` or your package manager of choice. Make sure you are using at least version `1.10.0`, otherwise some functionality might not work as expected.
python3 -m yt_dlp
If you plan on contributing to `yt-dlp`, best practice is to start by running the following command:
To run all the available core tests, use:
```shell
$ hatch run setup
```
python3 devscripts/run_tests.py
The above command will install a `pre-commit` hook so that required checks/fixes (linting, formatting) will run automatically before each commit. If any code needs to be linted or formatted, then the commit will be blocked and the necessary changes will be made; you should review all edits and re-commit the fixed version.
After this you can use `hatch shell` to enable a virtual environment that has `yt-dlp` and its development dependencies installed.
In addition, the following script commands can be used to run simple tasks such as linting or testing (without having to run `hatch shell` first):
* `hatch fmt`: Automatically fix linter violations and apply required code formatting changes
* See `hatch fmt --help` for more info
* `hatch test`: Run extractor or core tests
* See `hatch test --help` for more info
See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
While it is strongly recommended to use `hatch` for yt-dlp development, if you are unable to do so, alternatively you can manually create a virtual environment and use the following commands:
```shell
# To only install development dependencies:
$ python -m devscripts.install_deps --include dev
# Or, for an editable install plus dev dependencies:
$ python -m pip install -e ".[default,dev]"
# To setup the pre-commit hook:
$ pre-commit install
# To be used in place of `hatch test`:
$ python -m devscripts.run_tests
# To be used in place of `hatch fmt`:
$ ruff check --fix .
$ autopep8 --in-place .
# To only check code instead of applying fixes:
$ ruff check .
$ autopep8 --diff .
```
If you want to create a build of yt-dlp yourself, you can follow the instructions [here](README.md#compile).
@ -165,12 +200,16 @@ ## Adding support for a new site
1. [Fork this repository](https://github.com/yt-dlp/yt-dlp/fork)
1. Check out the source code with:
git clone git@github.com:YOUR_GITHUB_USERNAME/yt-dlp.git
```shell
$ git clone git@github.com:YOUR_GITHUB_USERNAME/yt-dlp.git
```
1. Start a new git branch with
cd yt-dlp
git checkout -b yourextractor
```shell
$ cd yt-dlp
$ git checkout -b yourextractor
```
1. Start with this simple template and save it to `yt_dlp/extractor/yourextractor.py`:
@ -217,21 +256,27 @@ ## Adding support for a new site
# TODO more properties (see yt_dlp/extractor/common.py)
}
```
1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`.
1. Run `python3 devscripts/run_tests.py YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
1. Add an import in [`yt_dlp/extractor/_extractors.py`](yt_dlp/extractor/_extractors.py). Note that the class name must end with `IE`. Also note that when adding a parenthesized import group, the last import in the group must have a trailing comma in order for this formatting to be respected by our code formatter.
1. Run `hatch test YourExtractor`. This *may fail* at first, but you can continually re-run it until you're done. Upon failure, it will output the missing fields and/or correct values which you can copy. If you decide to add more than one test, the tests will then be named `YourExtractor`, `YourExtractor_1`, `YourExtractor_2`, etc. Note that tests with an `only_matching` key in the test's dict are not included in the count. You can also run all the tests in one go with `YourExtractor_all`
1. Make sure you have at least one test for your extractor. Even if all videos covered by the extractor are expected to be inaccessible for automated testing, tests should still be added with a `skip` parameter indicating why the particular test is disabled from running.
1. Have a look at [`yt_dlp/extractor/common.py`](yt_dlp/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](yt_dlp/extractor/common.py#L119-L440). Add tests and code for as many as you want.
1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
1. Make sure your code follows [yt-dlp coding conventions](#yt-dlp-coding-conventions), passes [ruff](https://docs.astral.sh/ruff/tutorial/#getting-started) code checks and is properly formatted:
$ flake8 yt_dlp/extractor/yourextractor.py
```shell
$ hatch fmt --check
```
You can use `hatch fmt` to automatically fix problems.
1. Make sure your code works under all [Python](https://www.python.org/) versions supported by yt-dlp, namely CPython and PyPy for Python 3.8 and above. Backward compatibility is not required for even older versions of Python.
1. When the tests pass, [add](https://git-scm.com/docs/git-add) the new files, [commit](https://git-scm.com/docs/git-commit) them and [push](https://git-scm.com/docs/git-push) the result, like this:
$ git add yt_dlp/extractor/_extractors.py
$ git add yt_dlp/extractor/yourextractor.py
$ git commit -m '[yourextractor] Add extractor'
$ git push origin yourextractor
```shell
$ git add yt_dlp/extractor/_extractors.py
$ git add yt_dlp/extractor/yourextractor.py
$ git commit -m '[yourextractor] Add extractor'
$ git push origin yourextractor
```
1. Finally, [create a pull request](https://help.github.com/articles/creating-a-pull-request). We'll then review and merge it.

21
CONTRIBUTORS
View File

@ -610,3 +610,24 @@ Offert4324
sta1us
Tomoka1
trwstin
alexhuot1
clienthax
DaPotato69
emqi
hugohaa
imanoreotwe
JakeFinley96
lostfictions
minamotorin
ocococococ
Podiumnoche
RasmusAntons
roeniss
shoxie007
Szpachlarz
The-MAGI
TuxCoder
voidful
vtexier
WyohKnott
trueauracoral

121
Changelog.md
View File

@ -4,6 +4,127 @@ # Changelog
# To create a release, dispatch the https://github.com/yt-dlp/yt-dlp/actions/workflows/release.yml workflow on master
-->
### 2024.05.27
#### Extractor changes
- [Fix parsing of base URL in SMIL manifest](https://github.com/yt-dlp/yt-dlp/commit/26603d0b34898818992bee4598e0607c07059511) ([#9225](https://github.com/yt-dlp/yt-dlp/issues/9225)) by [seproDev](https://github.com/seproDev)
- **peertube**: [Support livestreams](https://github.com/yt-dlp/yt-dlp/commit/12b248ce60be1aa1362edd839d915bba70dbee4b) ([#10044](https://github.com/yt-dlp/yt-dlp/issues/10044)) by [bashonly](https://github.com/bashonly), [trueauracoral](https://github.com/trueauracoral)
- **piksel**: [Update domain](https://github.com/yt-dlp/yt-dlp/commit/ae2194e1dd4a99d32eb3cab7c48a0ff03101ef3b) ([#9223](https://github.com/yt-dlp/yt-dlp/issues/9223)) by [seproDev](https://github.com/seproDev)
- **tiktok**: user: [Fix extraction loop](https://github.com/yt-dlp/yt-dlp/commit/c53c2e40fde8f2e15c7c62f8ca1a5d9e90ddc079) ([#10035](https://github.com/yt-dlp/yt-dlp/issues/10035)) by [bashonly](https://github.com/bashonly)
#### Misc. changes
- **cleanup**: Miscellaneous: [5e3e19c](https://github.com/yt-dlp/yt-dlp/commit/5e3e19c93c52830da98d9d1ed84ea7a559efefbd) by [bashonly](https://github.com/bashonly)
### 2024.05.26
#### Core changes
- [Better warning when requested subs format not found](https://github.com/yt-dlp/yt-dlp/commit/7e4259dff0b681a3f0e8a930799ce0394328c86e) ([#9873](https://github.com/yt-dlp/yt-dlp/issues/9873)) by [DaPotato69](https://github.com/DaPotato69)
- [Merged with youtube-dl a08f2b7](https://github.com/yt-dlp/yt-dlp/commit/a4da9db87b6486b270c15dfa07ab5bfedc83f6bd) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
- [Warn if lack of ffmpeg alters format selection](https://github.com/yt-dlp/yt-dlp/commit/96da9525043f78aca4544d01761b13b2140e9ae6) ([#9805](https://github.com/yt-dlp/yt-dlp/issues/9805)) by [pukkandan](https://github.com/pukkandan), [seproDev](https://github.com/seproDev)
- **cookies**
- [Add `--cookies-from-browser` support for Whale](https://github.com/yt-dlp/yt-dlp/commit/dd9ad97b1fbdd36c086b8ba82328a4d954f78f8e) ([#9649](https://github.com/yt-dlp/yt-dlp/issues/9649)) by [roeniss](https://github.com/roeniss)
- [Get chrome session cookies with `--cookies-from-browser`](https://github.com/yt-dlp/yt-dlp/commit/f1f158976e38d38a260762accafe7bbe6d451151) ([#9747](https://github.com/yt-dlp/yt-dlp/issues/9747)) by [StefanLobbenmeier](https://github.com/StefanLobbenmeier)
- **windows**: [Improve shell quoting and tests](https://github.com/yt-dlp/yt-dlp/commit/64766459e37451b665c1464073c28361fbcf1c25) ([#9802](https://github.com/yt-dlp/yt-dlp/issues/9802)) by [Grub4K](https://github.com/Grub4K) (With fixes in [7e26bd5](https://github.com/yt-dlp/yt-dlp/commit/7e26bd53f9c5893518fde81dfd0079ec08dd841e))
#### Extractor changes
- [Add POST data hash to `--write-pages` filenames](https://github.com/yt-dlp/yt-dlp/commit/61b17437dc14a1c7e90ff48a6198df77828c6df4) ([#9879](https://github.com/yt-dlp/yt-dlp/issues/9879)) by [minamotorin](https://github.com/minamotorin) (With fixes in [c999bac](https://github.com/yt-dlp/yt-dlp/commit/c999bac02c5a4f755b2a82488a975e91c988ffd8) by [bashonly](https://github.com/bashonly))
- [Make `_search_nextjs_data` non fatal](https://github.com/yt-dlp/yt-dlp/commit/3ee1194288981c4f2c4abd8315326de0c424d2ce) ([#8937](https://github.com/yt-dlp/yt-dlp/issues/8937)) by [Grub4K](https://github.com/Grub4K)
- **afreecatv**: live: [Add `cdn` extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/315b3544296bb83012e20ee3af9d3cbf5600dd1c) ([#9666](https://github.com/yt-dlp/yt-dlp/issues/9666)) by [bashonly](https://github.com/bashonly)
- **alura**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/fc2879ecb05aaad36869609d154e4321362c1f63) ([#9658](https://github.com/yt-dlp/yt-dlp/issues/9658)) by [hugohaa](https://github.com/hugohaa)
- **artetv**: [Label forced subtitles](https://github.com/yt-dlp/yt-dlp/commit/7b5674949fd03a33b47b67b31d56a5adf1c48c91) ([#9945](https://github.com/yt-dlp/yt-dlp/issues/9945)) by [vtexier](https://github.com/vtexier)
- **bbc**: [Fix and extend extraction](https://github.com/yt-dlp/yt-dlp/commit/7975ddf245d22af034d5b983eeb1c5ec6c2ce053) ([#9705](https://github.com/yt-dlp/yt-dlp/issues/9705)) by [dirkf](https://github.com/dirkf), [kylegustavo](https://github.com/kylegustavo), [pukkandan](https://github.com/pukkandan)
- **bilibili**: [Fix `--geo-verification-proxy` support](https://github.com/yt-dlp/yt-dlp/commit/2338827072dacab0f15348b70aec8685feefc8d1) ([#9817](https://github.com/yt-dlp/yt-dlp/issues/9817)) by [fireattack](https://github.com/fireattack)
- **bilibilispacevideo**
- [Better error message](https://github.com/yt-dlp/yt-dlp/commit/06d52c87314e0bbc16c43c405090843885577b88) ([#9839](https://github.com/yt-dlp/yt-dlp/issues/9839)) by [fireattack](https://github.com/fireattack)
- [Fix extraction](https://github.com/yt-dlp/yt-dlp/commit/4cc99d7b6cce8b39506ead01407445d576b63ee4) ([#9905](https://github.com/yt-dlp/yt-dlp/issues/9905)) by [c-basalt](https://github.com/c-basalt)
- **boosty**: [Add cookies support](https://github.com/yt-dlp/yt-dlp/commit/145dc6f6563e80d2da1b3e9aea2ffa795b71622c) ([#9522](https://github.com/yt-dlp/yt-dlp/issues/9522)) by [RasmusAntons](https://github.com/RasmusAntons)
- **brilliantpala**: [Fix login](https://github.com/yt-dlp/yt-dlp/commit/eead3bbc01f6529862bdad1f0b2adeabda4f006e) ([#9788](https://github.com/yt-dlp/yt-dlp/issues/9788)) by [pzhlkj6612](https://github.com/pzhlkj6612)
- **canalalpha**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/00a9f2e1f7fa69499221f2e8dd73a08efeef79bc) ([#9675](https://github.com/yt-dlp/yt-dlp/issues/9675)) by [kclauhk](https://github.com/kclauhk)
- **cbc.ca**: player: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/c8bf48f3a8fa29587e7c73ef5a7710385a5ea725) ([#9866](https://github.com/yt-dlp/yt-dlp/issues/9866)) by [carusocr](https://github.com/carusocr)
- **cda**: [Fix age-gated web extraction](https://github.com/yt-dlp/yt-dlp/commit/6d8a53d870ff6795f509085bfbf3981417999038) ([#9939](https://github.com/yt-dlp/yt-dlp/issues/9939)) by [dirkf](https://github.com/dirkf), [emqi](https://github.com/emqi), [Podiumnoche](https://github.com/Podiumnoche), [Szpachlarz](https://github.com/Szpachlarz)
- **commonmistakes**: [Raise error on blob URLs](https://github.com/yt-dlp/yt-dlp/commit/98d71d8c5e5dab08b561ee6f137e968d2a004262) ([#9897](https://github.com/yt-dlp/yt-dlp/issues/9897)) by [seproDev](https://github.com/seproDev)
- **crunchyroll**
- [Always make metadata available](https://github.com/yt-dlp/yt-dlp/commit/cb2fb4a643949322adba561ca73bcba3221ec0c5) ([#9772](https://github.com/yt-dlp/yt-dlp/issues/9772)) by [bashonly](https://github.com/bashonly)
- [Fix auth and remove cookies support](https://github.com/yt-dlp/yt-dlp/commit/ff38a011d57b763f3a69bebd25a5dc9044a717ce) ([#9749](https://github.com/yt-dlp/yt-dlp/issues/9749)) by [bashonly](https://github.com/bashonly)
- [Fix stream extraction](https://github.com/yt-dlp/yt-dlp/commit/f2816634e3be88fe158b342ee33918de3c272a54) ([#10005](https://github.com/yt-dlp/yt-dlp/issues/10005)) by [bashonly](https://github.com/bashonly)
- [Support browser impersonation](https://github.com/yt-dlp/yt-dlp/commit/5904853ae5788509fdc4892cb7ecdfa9ae7f78e6) ([#9857](https://github.com/yt-dlp/yt-dlp/issues/9857)) by [bashonly](https://github.com/bashonly)
- **dangalplay**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/0d067e77c3f5527946fb0c22ee1c7011994cba40) ([#10021](https://github.com/yt-dlp/yt-dlp/issues/10021)) by [bashonly](https://github.com/bashonly)
- **discoveryplus**: [Fix dmax.de and related extractors](https://github.com/yt-dlp/yt-dlp/commit/90d2da311bbb5dc06f385ee428c7e4590936e995) ([#10020](https://github.com/yt-dlp/yt-dlp/issues/10020)) by [bashonly](https://github.com/bashonly)
- **eplus**: [Handle URLs without videos](https://github.com/yt-dlp/yt-dlp/commit/351dc0bc334c4e1b5f00c152818c3ec0ed71f788) ([#9855](https://github.com/yt-dlp/yt-dlp/issues/9855)) by [pzhlkj6612](https://github.com/pzhlkj6612)
- **europarlwebstream**: [Support new URL format](https://github.com/yt-dlp/yt-dlp/commit/800a43983e5fb719526ce4cb3956216085c63268) ([#9647](https://github.com/yt-dlp/yt-dlp/issues/9647)) by [seproDev](https://github.com/seproDev), [voidful](https://github.com/voidful)
- **facebook**: [Fix DASH formats extraction](https://github.com/yt-dlp/yt-dlp/commit/e3b42d8b1b8bcfff7ba146c19fc3f6f6ba843cea) ([#9734](https://github.com/yt-dlp/yt-dlp/issues/9734)) by [bashonly](https://github.com/bashonly)
- **godresource**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/65e709d23530959075816e966c42179ad46e8e3b) ([#9629](https://github.com/yt-dlp/yt-dlp/issues/9629)) by [HobbyistDev](https://github.com/HobbyistDev)
- **googledrive**: [Fix formats extraction](https://github.com/yt-dlp/yt-dlp/commit/85ec2a337ac325cf6427cbafd56f0a034c1a5218) ([#9908](https://github.com/yt-dlp/yt-dlp/issues/9908)) by [WyohKnott](https://github.com/WyohKnott)
- **hearthisat**: [Improve `_VALID_URL`](https://github.com/yt-dlp/yt-dlp/commit/5bbfdb7c999b22f1aeca0c3489c167d6eb73013b) ([#9949](https://github.com/yt-dlp/yt-dlp/issues/9949)) by [bohwaz](https://github.com/bohwaz), [seproDev](https://github.com/seproDev)
- **hytale**: [Use `CloudflareStreamIE` explicitly](https://github.com/yt-dlp/yt-dlp/commit/31b417e1d1ccc67d5c027bf8878f483dc34cb118) ([#9672](https://github.com/yt-dlp/yt-dlp/issues/9672)) by [llamasblade](https://github.com/llamasblade)
- **instagram**: [Support `/reels/` URLs](https://github.com/yt-dlp/yt-dlp/commit/06cb0638392b607b47d3c2ac48eb2ebecb0f060d) ([#9539](https://github.com/yt-dlp/yt-dlp/issues/9539)) by [amir16yp](https://github.com/amir16yp)
- **jiocinema**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/1463945ae5fb05986a0bd1aa02e41d1a08d93a02) ([#10026](https://github.com/yt-dlp/yt-dlp/issues/10026)) by [bashonly](https://github.com/bashonly)
- **jiosaavn**: [Extract via API and fix playlists](https://github.com/yt-dlp/yt-dlp/commit/0c21c53885cf03f4040467ae8c44d7ff51016116) ([#9656](https://github.com/yt-dlp/yt-dlp/issues/9656)) by [bashonly](https://github.com/bashonly)
- **lci**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/5a2eebc76770fca91ffabeff658d560f716fec80) ([#10025](https://github.com/yt-dlp/yt-dlp/issues/10025)) by [ocococococ](https://github.com/ocococococ)
- **mixch**: [Extract comments](https://github.com/yt-dlp/yt-dlp/commit/b38018b781b062d5169d104ab430489aef8e7f1e) ([#9860](https://github.com/yt-dlp/yt-dlp/issues/9860)) by [pzhlkj6612](https://github.com/pzhlkj6612)
- **moviepilot**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/296df0da1d38a44d34c99b60a18066c301774537) ([#9366](https://github.com/yt-dlp/yt-dlp/issues/9366)) by [panatexxa](https://github.com/panatexxa)
- **netease**: program: [Improve `--no-playlist` message](https://github.com/yt-dlp/yt-dlp/commit/73f12119b52d98281804b0c072b2ed6aa841ec88) ([#9488](https://github.com/yt-dlp/yt-dlp/issues/9488)) by [pzhlkj6612](https://github.com/pzhlkj6612)
- **nfb**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/0a1a8e3005f66c44bf67633dccd4df19c3fccd1a) ([#9650](https://github.com/yt-dlp/yt-dlp/issues/9650)) by [rrgomes](https://github.com/rrgomes)
- **ntslive**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/be7db1a5a8c483726c511c30ea4689cbb8b27962) ([#9641](https://github.com/yt-dlp/yt-dlp/issues/9641)) by [lostfictions](https://github.com/lostfictions)
- **orf**: on: [Improve extraction](https://github.com/yt-dlp/yt-dlp/commit/0dd53faeca2ba0ce138e4092d07b5f2dbf2422f9) ([#9677](https://github.com/yt-dlp/yt-dlp/issues/9677)) by [TuxCoder](https://github.com/TuxCoder)
- **orftvthek**: [Remove extractor](https://github.com/yt-dlp/yt-dlp/commit/3779f2a307ba3ef1d28e107cdd71b221dfb4eb36) ([#10011](https://github.com/yt-dlp/yt-dlp/issues/10011)) by [seproDev](https://github.com/seproDev)
- **patreon**
- [Extract multiple embeds](https://github.com/yt-dlp/yt-dlp/commit/036e0d92c6052465673d459678322ea03e61483d) ([#9850](https://github.com/yt-dlp/yt-dlp/issues/9850)) by [bashonly](https://github.com/bashonly)
- [Fix Vimeo embed extraction](https://github.com/yt-dlp/yt-dlp/commit/c9ce57d9bf51541da2381d99bc096a9d0ddf1f27) ([#9712](https://github.com/yt-dlp/yt-dlp/issues/9712)) by [bashonly](https://github.com/bashonly)
- **piapro**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/3ba8de62d61d782256f5c1e9939a0762039657de) ([#9311](https://github.com/yt-dlp/yt-dlp/issues/9311)) by [FinnRG](https://github.com/FinnRG), [seproDev](https://github.com/seproDev)
- **pornhub**: [Fix login by email address](https://github.com/yt-dlp/yt-dlp/commit/518c1afc1592cae3e4eb39dc646b5bc059333112) ([#9914](https://github.com/yt-dlp/yt-dlp/issues/9914)) by [feederbox826](https://github.com/feederbox826)
- **qub**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6b54cccdcb892bca3e55993480d8b86f1c7e6da6) ([#7019](https://github.com/yt-dlp/yt-dlp/issues/7019)) by [alexhuot1](https://github.com/alexhuot1), [dirkf](https://github.com/dirkf)
- **reddit**: [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/82f4f4444e26daf35b7302c406fe2312f78f619e) ([#10006](https://github.com/yt-dlp/yt-dlp/issues/10006)) by [kclauhk](https://github.com/kclauhk)
- **soundcloud**
- [Add `formats` extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/beaf832c7a9d57833f365ce18f6115b88071b296) ([#10004](https://github.com/yt-dlp/yt-dlp/issues/10004)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
- [Extract `genres`](https://github.com/yt-dlp/yt-dlp/commit/231c2eacc41b06b65c63edf94c0d04768a5da607) ([#9821](https://github.com/yt-dlp/yt-dlp/issues/9821)) by [bashonly](https://github.com/bashonly)
- **taptap**: [Add extractors](https://github.com/yt-dlp/yt-dlp/commit/63b569bc5e7d461753637a20ad84a575adee4c0a) ([#9776](https://github.com/yt-dlp/yt-dlp/issues/9776)) by [c-basalt](https://github.com/c-basalt)
- **tele5**: [Overhaul extractor](https://github.com/yt-dlp/yt-dlp/commit/c92e4e625e9e6bbbbf8e3b20c3e7ebe57c16072d) ([#10024](https://github.com/yt-dlp/yt-dlp/issues/10024)) by [bashonly](https://github.com/bashonly)
- **theatercomplextown**: [Fix extractors](https://github.com/yt-dlp/yt-dlp/commit/8056a3026ed6ec6a6d0ed56fdd7ebcd16e928341) ([#9754](https://github.com/yt-dlp/yt-dlp/issues/9754)) by [bashonly](https://github.com/bashonly)
- **tiktok**
- [Add `device_id` extractor-arg](https://github.com/yt-dlp/yt-dlp/commit/3584b8390bd21c0393a3079eeee71aed56a1c1d8) ([#9951](https://github.com/yt-dlp/yt-dlp/issues/9951)) by [bashonly](https://github.com/bashonly)
- [Extract all web formats](https://github.com/yt-dlp/yt-dlp/commit/4ccd73fea0f6f4be343e1ec7f22dd03799addcf8) ([#9960](https://github.com/yt-dlp/yt-dlp/issues/9960)) by [bashonly](https://github.com/bashonly)
- [Extract via mobile API only if extractor-arg is passed](https://github.com/yt-dlp/yt-dlp/commit/41ba4a808b597a3afed78c89675a30deb6844450) ([#9938](https://github.com/yt-dlp/yt-dlp/issues/9938)) by [bashonly](https://github.com/bashonly)
- [Fix subtitles extraction](https://github.com/yt-dlp/yt-dlp/commit/eef1e9f44ff14c5e65b759bb1eafa3946cdaf719) ([#9961](https://github.com/yt-dlp/yt-dlp/issues/9961)) by [bashonly](https://github.com/bashonly)
- collection: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/119d41f27061d220d276a2d38cfc8d873437452a) ([#9986](https://github.com/yt-dlp/yt-dlp/issues/9986)) by [bashonly](https://github.com/bashonly), [imanoreotwe](https://github.com/imanoreotwe)
- user: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/347f13dd9bccc2b4db3ea25689410d45d8370ed4) ([#9661](https://github.com/yt-dlp/yt-dlp/issues/9661)) by [bashonly](https://github.com/bashonly)
- **tv5monde**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/6db96268c521e945d42649607db1574f5d92e082) ([#9143](https://github.com/yt-dlp/yt-dlp/issues/9143)) by [alard](https://github.com/alard), [seproDev](https://github.com/seproDev)
- **twitter**
- [Fix auth for x.com migration](https://github.com/yt-dlp/yt-dlp/commit/3e35aa32c74bc108375be8c8b6b3bfc90dfff1b4) ([#9952](https://github.com/yt-dlp/yt-dlp/issues/9952)) by [bashonly](https://github.com/bashonly)
- [Support x.com URLs](https://github.com/yt-dlp/yt-dlp/commit/4813173e4544f125d6f2afc31e600727d761b8dd) ([#9926](https://github.com/yt-dlp/yt-dlp/issues/9926)) by [bashonly](https://github.com/bashonly)
- **vk**: [Improve format extraction](https://github.com/yt-dlp/yt-dlp/commit/df5c9e733aaba703cf285c0372b6d61629330c82) ([#9885](https://github.com/yt-dlp/yt-dlp/issues/9885)) by [seproDev](https://github.com/seproDev)
- **wrestleuniverse**: [Avoid partial stream formats](https://github.com/yt-dlp/yt-dlp/commit/c4853655cb9a793129280806af643de43c48f4d5) ([#9800](https://github.com/yt-dlp/yt-dlp/issues/9800)) by [bashonly](https://github.com/bashonly)
- **xiaohongshu**: [Add extractor](https://github.com/yt-dlp/yt-dlp/commit/a2e9031605d87c469be9ce98dbbdf4960b727338) ([#9646](https://github.com/yt-dlp/yt-dlp/issues/9646)) by [HobbyistDev](https://github.com/HobbyistDev)
- **xvideos**: quickies: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/b207d26f83fb8ab0ce56df74dff43ff583a3264f) ([#9834](https://github.com/yt-dlp/yt-dlp/issues/9834)) by [JakeFinley96](https://github.com/JakeFinley96)
- **youporn**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/351368cb9a6731b886a58f5a10fd6b302bbe47be) ([#8827](https://github.com/yt-dlp/yt-dlp/issues/8827)) by [The-MAGI](https://github.com/The-MAGI)
- **youtube**
- [Add `mediaconnect` client](https://github.com/yt-dlp/yt-dlp/commit/cf212d0a331aba05c32117573f760cdf3af8c62f) ([#9546](https://github.com/yt-dlp/yt-dlp/issues/9546)) by [clienthax](https://github.com/clienthax)
- [Extract upload timestamp if available](https://github.com/yt-dlp/yt-dlp/commit/96a134dea6397a5f2131947c427aac52c8b4e677) ([#9856](https://github.com/yt-dlp/yt-dlp/issues/9856)) by [coletdjnz](https://github.com/coletdjnz)
- [Fix comments extraction](https://github.com/yt-dlp/yt-dlp/commit/8e15177b4113c355989881e4e030f695a9b59c3a) ([#9775](https://github.com/yt-dlp/yt-dlp/issues/9775)) by [bbilly1](https://github.com/bbilly1), [jakeogh](https://github.com/jakeogh), [minamotorin](https://github.com/minamotorin), [shoxie007](https://github.com/shoxie007)
- [Remove `android` from default clients](https://github.com/yt-dlp/yt-dlp/commit/12d8ea8246fa901de302ff5cc748caddadc82f41) ([#9553](https://github.com/yt-dlp/yt-dlp/issues/9553)) by [bashonly](https://github.com/bashonly), [coletdjnz](https://github.com/coletdjnz)
- **zenyandex**: [Fix extractor](https://github.com/yt-dlp/yt-dlp/commit/c4b87dd885ee5391e5f481e7c8bd550a7c543623) ([#9813](https://github.com/yt-dlp/yt-dlp/issues/9813)) by [src-tinkerer](https://github.com/src-tinkerer)
#### Networking changes
- [Add `extensions` attribute to `Response`](https://github.com/yt-dlp/yt-dlp/commit/bec9a59e8ec82c18e3bf9268eaa436793dd52e35) ([#9756](https://github.com/yt-dlp/yt-dlp/issues/9756)) by [bashonly](https://github.com/bashonly)
- **Request Handler**
- requests
- [Patch support for `requests` 2.32.2+](https://github.com/yt-dlp/yt-dlp/commit/3f7999533ebe41c2a579d91b4e4cb211cfcd3bc0) ([#9992](https://github.com/yt-dlp/yt-dlp/issues/9992)) by [Grub4K](https://github.com/Grub4K)
- [Update to `requests` 2.32.0](https://github.com/yt-dlp/yt-dlp/commit/c36513f1be2ef3d3cec864accbffda1afaa06ffd) ([#9980](https://github.com/yt-dlp/yt-dlp/issues/9980)) by [coletdjnz](https://github.com/coletdjnz)
#### Misc. changes
- [Add `hatch`, `ruff`, `pre-commit` and improve dev docs](https://github.com/yt-dlp/yt-dlp/commit/e897bd8292a41999cf51dba91b390db5643c72db) ([#7409](https://github.com/yt-dlp/yt-dlp/issues/7409)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev)
- **build**
- [Migrate `linux_exe` to static musl builds](https://github.com/yt-dlp/yt-dlp/commit/ac817bc83efd939dca3e40c4b527d0ccfc77172b) ([#9811](https://github.com/yt-dlp/yt-dlp/issues/9811)) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K)
- [Normalize `curl_cffi` group to `curl-cffi`](https://github.com/yt-dlp/yt-dlp/commit/02483bea1c4dbe1bace8ca4d19700104fbb8a00f) ([#9698](https://github.com/yt-dlp/yt-dlp/issues/9698)) by [bashonly](https://github.com/bashonly) (With fixes in [89f535e](https://github.com/yt-dlp/yt-dlp/commit/89f535e2656964b4061c25a7739d4d6ba0a30568))
- [Run `macos_legacy` job on `macos-12`](https://github.com/yt-dlp/yt-dlp/commit/1a366403d9c26b992faa77e00f4d02ead57559e3) ([#9804](https://github.com/yt-dlp/yt-dlp/issues/9804)) by [bashonly](https://github.com/bashonly)
- [`macos` job requires `setuptools<70`](https://github.com/yt-dlp/yt-dlp/commit/78c57cc0e0998b8ed90e4306f410aa4be4115cd7) ([#9993](https://github.com/yt-dlp/yt-dlp/issues/9993)) by [bashonly](https://github.com/bashonly)
- **cleanup**
- [Remove questionable extractors](https://github.com/yt-dlp/yt-dlp/commit/01395a34345d1c6ba1b73ca92f94dd200dc45341) ([#9911](https://github.com/yt-dlp/yt-dlp/issues/9911)) by [seproDev](https://github.com/seproDev)
- Miscellaneous: [5c019f6](https://github.com/yt-dlp/yt-dlp/commit/5c019f6328ad40d66561eac3c4de0b3cd070d0f6), [ae2af11](https://github.com/yt-dlp/yt-dlp/commit/ae2af1104f80caf2f47544763a33db2c17a3e1de) by [bashonly](https://github.com/bashonly), [Grub4K](https://github.com/Grub4K), [seproDev](https://github.com/seproDev)
- **test**
- [Add HTTP proxy tests](https://github.com/yt-dlp/yt-dlp/commit/3c7a287e281d9f9a353dce8902ff78a84c24a040) ([#9578](https://github.com/yt-dlp/yt-dlp/issues/9578)) by [coletdjnz](https://github.com/coletdjnz)
- [Fix connect timeout test](https://github.com/yt-dlp/yt-dlp/commit/53b4d44f55cca66ac33dab092ef2a30b1164b684) ([#9906](https://github.com/yt-dlp/yt-dlp/issues/9906)) by [coletdjnz](https://github.com/coletdjnz)
### 2024.04.09
#### Important changes

11
Makefile
View File

@ -27,7 +27,7 @@ clean-dist:
yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS
clean-cache:
find . \( \
-type d -name .pytest_cache -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \
-type d -name ".*_cache" -o -type d -name __pycache__ -o -name "*.pyc" -o -name "*.class" \
\) -prune -exec rm -rf {} \;
completion-bash: completions/bash/yt-dlp
@ -70,14 +70,15 @@ uninstall:
rm -f $(DESTDIR)$(SHAREDIR)/fish/vendor_completions.d/yt-dlp.fish
codetest:
flake8 .
ruff check .
autopep8 --diff .
test:
$(PYTHON) -m pytest
$(PYTHON) -m pytest -Werror
$(MAKE) codetest
offlinetest: codetest
$(PYTHON) -m pytest -k "not download"
$(PYTHON) -m pytest -Werror -m "not download"
CODE_FOLDERS_CMD = find yt_dlp -type f -name '__init__.py' | sed 's,/__init__.py,,' | grep -v '/__' | sort
CODE_FOLDERS != $(CODE_FOLDERS_CMD)
@ -151,7 +152,7 @@ yt-dlp.tar.gz: all
--exclude '*.pyo' \
--exclude '*~' \
--exclude '__pycache__' \
--exclude '.pytest_cache' \
--exclude '.*_cache' \
--exclude '.git' \
-- \
README.md supportedsites.md Changelog.md LICENSE \

43
README.md
View File

@ -108,7 +108,6 @@ #### Alternatives
[yt-dlp_x86.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_x86.exe)|Windows (Win7 SP1+) standalone x86 (32-bit) binary
[yt-dlp_min.exe](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_min.exe)|Windows (Win7 SP1+) standalone x64 binary built with `py2exe`<br/> ([Not recommended](#standalone-py2exe-builds-windows))
[yt-dlp_linux](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux)|Linux standalone x64 binary
[yt-dlp_linux.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux.zip)|Unpackaged Linux executable (no auto-update)
[yt-dlp_linux_armv7l](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_armv7l)|Linux standalone armv7l (32-bit) binary
[yt-dlp_linux_aarch64](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64)|Linux standalone aarch64 (64-bit) binary
[yt-dlp_win.zip](https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_win.zip)|Unpackaged Windows executable (no auto-update)
@ -170,7 +169,7 @@ # To update to nightly from stable executable/binary:
yt-dlp --update-to nightly
# To install nightly with pip:
python3 -m pip install -U --pre yt-dlp[default]
python3 -m pip install -U --pre "yt-dlp[default]"
```
## DEPENDENCIES
@ -202,7 +201,7 @@ #### Impersonation
The following provide support for impersonating browser requests. This may be required for some sites that employ TLS fingerprinting.
* [**curl_cffi**](https://github.com/yifeikong/curl_cffi) (recommended) - Python binding for [curl-impersonate](https://github.com/lwthiker/curl-impersonate). Provides impersonation targets for Chrome, Edge and Safari. Licensed under [MIT](https://github.com/yifeikong/curl_cffi/blob/main/LICENSE)
* Can be installed with the `curl-cffi` group, e.g. `pip install yt-dlp[default,curl-cffi]`
* Can be installed with the `curl-cffi` group, e.g. `pip install "yt-dlp[default,curl-cffi]"`
* Currently only included in `yt-dlp.exe` and `yt-dlp_macos` builds
@ -263,7 +262,7 @@ ### Platform-independent Binary (UNIX)
### Standalone Py2Exe Builds (Windows)
While we provide the option to build with [py2exe](https://www.py2exe.org), it is recommended to build [using PyInstaller](#standalone-pyinstaller-builds) instead since the py2exe builds **cannot contain `pycryptodomex`/`certifi` and needs VC++14** on the target computer to run.
While we provide the option to build with [py2exe](https://www.py2exe.org), it is recommended to build [using PyInstaller](#standalone-pyinstaller-builds) instead since the py2exe builds **cannot contain `pycryptodomex`/`certifi`/`requests` and need VC++14** on the target computer to run.
If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:
@ -402,6 +401,9 @@ ## Network Options:
--impersonate CLIENT[:OS] Client to impersonate for requests. E.g.
chrome, chrome-110, chrome:windows-10. Pass
--impersonate="" to impersonate any client.
Note that forcing impersonation for all
requests may have a detrimental impact on
download speed and stability
--list-impersonate-targets List available clients to impersonate.
-4, --force-ipv4 Make all connections via IPv4
-6, --force-ipv6 Make all connections via IPv6
@ -666,16 +668,17 @@ ## Filesystem Options:
The name of the browser to load cookies
from. Currently supported browsers are:
brave, chrome, chromium, edge, firefox,
opera, safari, vivaldi, whale. Optionally, the
KEYRING used for decrypting Chromium cookies
on Linux, the name/path of the PROFILE to
load cookies from, and the CONTAINER name
(if Firefox) ("none" for no container) can
be given with their respective seperators.
By default, all containers of the most
recently accessed profile are used.
Currently supported keyrings are: basictext,
gnomekeyring, kwallet, kwallet5, kwallet6
opera, safari, vivaldi, whale. Optionally,
the KEYRING used for decrypting Chromium
cookies on Linux, the name/path of the
PROFILE to load cookies from, and the
CONTAINER name (if Firefox) ("none" for no
container) can be given with their
respective seperators. By default, all
containers of the most recently accessed
profile are used. Currently supported
keyrings are: basictext, gnomekeyring,
kwallet, kwallet5, kwallet6
--no-cookies-from-browser Do not load cookies from browser (default)
--cache-dir DIR Location in the filesystem where yt-dlp can
store some downloaded information (such as
@ -1751,7 +1754,7 @@ # Replace all spaces and "_" in title and uploader with a `-`
# EXTRACTOR ARGUMENTS
Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;include_live_dash" --extractor-args "funimation:version=uncut"`
Some extractors accept additional arguments which can be passed using `--extractor-args KEY:ARGS`. `ARGS` is a `;` (semicolon) separated string of `ARG=VAL1,VAL2`. E.g. `--extractor-args "youtube:player-client=android_embedded,web;formats=incomplete" --extractor-args "funimation:version=uncut"`
Note: In CLI, `ARG` can use `-` instead of `_`; e.g. `youtube:player-client"` becomes `youtube:player_client"`
@ -1815,6 +1818,7 @@ #### tiktok
* `manifest_app_version`: Default numeric app version to use with mobile API calls, e.g. `2023401020`
* `aid`: Default app ID to use with mobile API calls, e.g. `1180`
* `app_info`: Enable mobile API extraction with one or more app info strings in the format of `<iid>/[app_name]/[app_version]/[manifest_app_version]/[aid]`, where `iid` is the unique app install ID. `iid` is the only required value; all other values and their `/` separators can be omitted, e.g. `tiktok:app_info=1234567890123456789` or `tiktok:app_info=123,456/trill///1180,789//34.0.1/340001`
* `device_id`: Enable mobile API extraction with a genuine device ID to be used with mobile API calls. Default is a random 19-digit string
#### rokfinchannel
* `tab`: Which tab to download - one of `new`, `top`, `videos`, `podcasts`, `streams`, `stacks`
@ -1834,12 +1838,18 @@ #### nhkradirulive (NHK らじる★らじる LIVE)
#### nflplusreplay
* `type`: Type(s) of game replays to extract. Valid types are: `full_game`, `full_game_spanish`, `condensed_game` and `all_22`. You can use `all` to extract all available replay types, which is the default
#### jiocinema
* `refresh_token`: The `refreshToken` UUID from browser local storage can be passed to extend the life of your login session when logging in with `token` as username and the `accessToken` from browser local storage as password
#### jiosaavn
* `bitrate`: Audio bitrates to request. One or more of `16`, `32`, `64`, `128`, `320`. Default is `128,320`
#### afreecatvlive
* `cdn`: One or more CDN IDs to use with the API call for stream URLs, e.g. `gcp_cdn`, `gs_cdn_pc_app`, `gs_cdn_mobile_web`, `gs_cdn_pc_web`
#### soundcloud
* `formats`: Formats to request from the API. Requested values should be in the format of `{protocol}_{extension}` (omitting the bitrate), e.g. `hls_opus,http_aac`. The `*` character functions as a wildcard, e.g. `*_mp3`, and can passed by itself to request all formats. Known protocols include `http`, `hls` and `hls-aes`; known extensions include `aac`, `opus` and `mp3`. Original `download` formats are always extracted. Default is `http_aac,hls_aac,http_opus,hls_opus,http_mp3,hls_mp3`
**Note**: These options may be changed/removed in the future without concern for backward compatibility
<!-- MANPAGE: MOVE "INSTALLATION" SECTION HERE -->
@ -2116,7 +2126,7 @@ # CHANGES FROM YOUTUBE-DL
### New features
* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@be008e6**](https://github.com/ytdl-org/youtube-dl/commit/be008e657d79832642e2158557c899249c9e31cd) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
* Forked from [**yt-dlc@f9401f2**](https://github.com/blackjack4494/yt-dlc/commit/f9401f2a91987068139c5f757b12fc711d4c0cee) and merged with [**youtube-dl@a08f2b7**](https://github.com/ytdl-org/youtube-dl/commit/a08f2b7e4567cdc50c0614ee0a4ffdff49b8b6e6) ([exceptions](https://github.com/yt-dlp/yt-dlp/issues/21))
* **[SponsorBlock Integration](#sponsorblock-options)**: You can mark/remove sponsor sections in YouTube videos by utilizing the [SponsorBlock](https://sponsor.ajay.app) API
@ -2326,6 +2336,7 @@ #### No longer supported
--write-annotations No supported site has annotations now
--no-write-annotations Default
--compat-options seperate-video-versions No longer needed
--compat-options no-youtube-prefer-utc-upload-date No longer supported
#### Removed
These options were deprecated since 2014 and have now been entirely removed

6
bundle/py2exe.py
View File

@ -42,9 +42,9 @@ def main():
# py2exe cannot import Crypto
'Crypto',
'Cryptodome',
# py2exe appears to confuse this with our socks library.
# We don't use pysocks and urllib3.contrib.socks would fail to import if tried.
'urllib3.contrib.socks'
# requests >=2.32.0 breaks py2exe builds due to certifi dependency
'requests',
'urllib3'
],
'dll_excludes': ['w9xpopen.exe', 'crypt32.dll'],
# Modules that are only imported dynamically must be added here

22
devscripts/changelog_override.json
View File

@ -147,5 +147,27 @@
"action": "add",
"when": "9590cc6b4768e190183d7d071a6c78170889116a",
"short": "[priority] Security: [[CVE-2024-22423](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2024-22423)] [Prevent RCE when using `--exec` with `%q` on Windows](https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-hjq6-52gw-2g7p)\n - The shell escape function now properly escapes `%`, `\\` and `\\n`.\n - `utils.Popen` has been patched accordingly."
},
{
"action": "change",
"when": "41ba4a808b597a3afed78c89675a30deb6844450",
"short": "[ie/tiktok] Extract via mobile API only if extractor-arg is passed (#9938)",
"authors": ["bashonly"]
},
{
"action": "remove",
"when": "6e36d17f404556f0e3a43f441c477a71a91877d9"
},
{
"action": "change",
"when": "beaf832c7a9d57833f365ce18f6115b88071b296",
"short": "[ie/soundcloud] Add `formats` extractor-arg (#10004)",
"authors": ["bashonly", "Grub4K"]
},
{
"action": "change",
"when": "5c019f6328ad40d66561eac3c4de0b3cd070d0f6",
"short": "[cleanup] Misc (#9765)",
"authors": ["bashonly", "Grub4K", "seproDev"]
}
]

12
devscripts/install_deps.py
View File

@ -42,17 +42,25 @@ def parse_args():
def main():
args = parse_args()
project_table = parse_toml(read_file(args.input))['project']
recursive_pattern = re.compile(rf'{project_table["name"]}\[(?P<group_name>[\w-]+)\]')
optional_groups = project_table['optional-dependencies']
excludes = args.exclude or []
def yield_deps(group):
for dep in group:
if mobj := recursive_pattern.fullmatch(dep):
yield from optional_groups.get(mobj.group('group_name'), [])
else:
yield dep
targets = []
if not args.only_optional: # `-o` should exclude 'dependencies' and the 'default' group
targets.extend(project_table['dependencies'])
if 'default' not in excludes: # `--exclude default` should exclude entire 'default' group
targets.extend(optional_groups['default'])
targets.extend(yield_deps(optional_groups['default']))
for include in filter(None, map(optional_groups.get, args.include or [])):
targets.extend(include)
targets.extend(yield_deps(include))
targets = [t for t in targets if re.match(r'[\w-]+', t).group(0).lower() not in excludes]

4
devscripts/run_tests.bat
View File

@ -1,4 +0,0 @@
@echo off
>&2 echo run_tests.bat is deprecated. Please use `devscripts/run_tests.py` instead
python %~dp0run_tests.py %~1

14
devscripts/run_tests.py
View File

@ -4,6 +4,7 @@
import functools
import os
import re
import shlex
import subprocess
import sys
from pathlib import Path
@ -18,6 +19,8 @@ def parse_args():
'test', help='a extractor tests, or one of "core" or "download"', nargs='*')
parser.add_argument(
'-k', help='run a test matching EXPRESSION. Same as "pytest -k"', metavar='EXPRESSION')
parser.add_argument(
'--pytest-args', help='arguments to passthrough to pytest')
return parser.parse_args()
@ -26,15 +29,16 @@ def run_tests(*tests, pattern=None, ci=False):
run_download = 'download' in tests
tests = list(map(fix_test_name, tests))
arguments = ['pytest', '-Werror', '--tb=short']
pytest_args = args.pytest_args or os.getenv('HATCH_TEST_ARGS', '')
arguments = ['pytest', '-Werror', '--tb=short', *shlex.split(pytest_args)]
if ci:
arguments.append('--color=yes')
if pattern:
arguments.extend(['-k', pattern])
if run_core:
arguments.extend(['-m', 'not download'])
elif run_download:
arguments.extend(['-m', 'download'])
elif pattern:
arguments.extend(['-k', pattern])
else:
arguments.extend(
f'test/test_download.py::TestDownload::test_{test}' for test in tests)
@ -46,13 +50,13 @@ def run_tests(*tests, pattern=None, ci=False):
pass
arguments = [sys.executable, '-Werror', '-m', 'unittest']
if pattern:
arguments.extend(['-k', pattern])
if run_core:
print('"pytest" needs to be installed to run core tests', file=sys.stderr, flush=True)
return 1
elif run_download:
arguments.append('test.test_download')
elif pattern:
arguments.extend(['-k', pattern])
else:
arguments.extend(
f'test.test_download.TestDownload.test_{test}' for test in tests)

4
devscripts/run_tests.sh
View File

@ -1,4 +0,0 @@
#!/usr/bin/env sh
>&2 echo 'run_tests.sh is deprecated. Please use `devscripts/run_tests.py` instead'
python3 devscripts/run_tests.py "$1"

17
pyinst.py
View File

@ -1,17 +0,0 @@
#!/usr/bin/env python3
# Allow execution from anywhere
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import warnings
from bundle.pyinstaller import main
warnings.warn(DeprecationWarning('`pyinst.py` is deprecated and will be removed in a future version. '
'Use `bundle.pyinstaller` instead'))
if __name__ == '__main__':
main()

166
pyproject.toml
View File

@ -46,7 +46,7 @@ dependencies = [
"certifi",
"mutagen",
"pycryptodomex",
"requests>=2.31.0,<3",
"requests>=2.32.2,<3",
"urllib3>=1.26.17,<3",
"websockets>=12.0",
]
@ -62,18 +62,27 @@ build = [
"build",
"hatchling",
"pip",
"setuptools",
"wheel",
]
dev = [
"flake8",
"isort",
"pytest",
"pre-commit",
"yt-dlp[static-analysis]",
"yt-dlp[test]",
]
static-analysis = [
"autopep8~=2.0",
"ruff~=0.4.4",
]
test = [
"pytest~=8.1",
]
pyinstaller = [
"pyinstaller>=6.3; sys_platform!='darwin'",
"pyinstaller==5.13.2; sys_platform=='darwin'", # needed for curl_cffi
"pyinstaller>=6.7.0", # for compat with setuptools>=70
]
py2exe = [
"py2exe>=0.12",
]
py2exe = ["py2exe>=0.12"]
[project.urls]
Documentation = "https://github.com/yt-dlp/yt-dlp#readme"
@ -122,3 +131,146 @@ artifacts = ["/yt_dlp/extractor/lazy_extractors.py"]
[tool.hatch.version]
path = "yt_dlp/version.py"
pattern = "_pkg_version = '(?P<version>[^']+)'"
[tool.hatch.envs.default]
features = ["curl-cffi", "default"]
dependencies = ["pre-commit"]
path = ".venv"
installer = "uv"
[tool.hatch.envs.default.scripts]
setup = "pre-commit install --config .pre-commit-hatch.yaml"
yt-dlp = "python -Werror -Xdev -m yt_dlp {args}"
[tool.hatch.envs.hatch-static-analysis]
detached = true
features = ["static-analysis"]
dependencies = [] # override hatch ruff version
config-path = "pyproject.toml"
[tool.hatch.envs.hatch-static-analysis.scripts]
format-check = "autopep8 --diff {args:.}"
format-fix = "autopep8 --in-place {args:.}"
lint-check = "ruff check {args:.}"
lint-fix = "ruff check --fix {args:.}"
[tool.hatch.envs.hatch-test]
features = ["test"]
dependencies = [
"pytest-randomly~=3.15",
"pytest-rerunfailures~=14.0",
"pytest-xdist[psutil]~=3.5",
]
[tool.hatch.envs.hatch-test.scripts]
run = "python -m devscripts.run_tests {args}"
run-cov = "echo Code coverage not implemented && exit 1"
[[tool.hatch.envs.hatch-test.matrix]]
python = [
"3.8",
"3.9",
"3.10",
"3.11",
"3.12",
"pypy3.8",
"pypy3.9",
"pypy3.10",
]
[tool.ruff]
line-length = 120
[tool.ruff.lint]
ignore = [
"E402", # module level import not at top of file
"E501", # line too long
"E731", # do not assign a lambda expression, use a def
"E741", # ambiguous variable name
]
select = [
"E", # pycodestyle errors
"W", # pycodestyle warnings
"F", # pyflakes
"I", # import order
]
[tool.ruff.lint.per-file-ignores]
"devscripts/lazy_load_template.py" = ["F401"]
"!yt_dlp/extractor/**.py" = ["I"]
[tool.ruff.lint.isort]
known-first-party = [
"bundle",
"devscripts",
"test",
]
relative-imports-order = "closest-to-furthest"
[tool.autopep8]
max_line_length = 120
recursive = true
exit-code = true
jobs = 0
select = [
"E101",
"E112",
"E113",
"E115",
"E116",
"E117",
"E121",
"E122",
"E123",
"E124",
"E125",
"E126",
"E127",
"E128",
"E129",
"E131",
"E201",
"E202",
"E203",
"E211",
"E221",
"E222",
"E223",
"E224",
"E225",
"E226",
"E227",
"E228",
"E231",
"E241",
"E242",
"E251",
"E252",
"E261",
"E262",
"E265",
"E266",
"E271",
"E272",
"E273",
"E274",
"E275",
"E301",
"E302",
"E303",
"E304",
"E305",
"E306",
"E502",
"E701",
"E702",
"E704",
"W391",
"W504",
]
[tool.pytest.ini_options]
addopts = "-ra -v --strict-markers"
markers = [
"download",
]

6
setup.cfg
View File

@ -14,12 +14,6 @@ remove-duplicate-keys = true
remove-unused-variables = true
[tool:pytest]
addopts = -ra -v --strict-markers
markers =
download
[tox:tox]
skipsdist = true
envlist = py{38,39,310,311,312},pypy{38,39,310}

36
setup.py
View File

@ -1,36 +0,0 @@
#!/usr/bin/env python3
# Allow execution from anywhere
import os
import sys
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import warnings
if sys.argv[1:2] == ['py2exe']:
warnings.warn(DeprecationWarning('`setup.py py2exe` is deprecated and will be removed in a future version. '
'Use `bundle.py2exe` instead'))
import bundle.py2exe
bundle.py2exe.main()
elif 'build_lazy_extractors' in sys.argv:
warnings.warn(DeprecationWarning('`setup.py build_lazy_extractors` is deprecated and will be removed in a future version. '
'Use `devscripts.make_lazy_extractors` instead'))
import subprocess
os.chdir(sys.path[0])
print('running build_lazy_extractors')
subprocess.run([sys.executable, 'devscripts/make_lazy_extractors.py'])
else:
print(
'ERROR: Building by calling `setup.py` is deprecated. '
'Use a build frontend like `build` instead. ',
'Refer to https://build.pypa.io for more info', file=sys.stderr)
sys.exit(1)

37
supportedsites.md
View File

@ -14,7 +14,6 @@ # Supported sites
- **6play**
- **7plus**
- **8tracks**
- **91porn**
- **9c9media**
- **9gag**: 9GAG
- **9News**
@ -220,7 +219,7 @@ # Supported sites
- **BusinessInsider**
- **BuzzFeed**
- **BYUtv**: (**Currently broken**)
- **CableAV**
- **CaffeineTV**
- **Callin**
- **Caltrans**
- **CAM4**
@ -333,6 +332,8 @@ # Supported sites
- **DailyWirePodcast**
- **damtomo:record**
- **damtomo:video**
- **dangalplay**: [*dangalplay*](## "netrc machine")
- **dangalplay:season**: [*dangalplay*](## "netrc machine")
- **daum.net**
- **daum.net:clip**
- **daum.net:playlist**
@ -396,7 +397,6 @@ # Supported sites
- **EinsUndEinsTV**: [*1und1tv*](## "netrc machine")
- **EinsUndEinsTVLive**: [*1und1tv*](## "netrc machine")
- **EinsUndEinsTVRecordings**: [*1und1tv*](## "netrc machine")
- **Einthusan**
- **eitb.tv**
- **ElementorEmbed**
- **Elonet**
@ -498,6 +498,7 @@ # Supported sites
- **GameStar**
- **Gaskrank**
- **Gazeta**: (**Currently broken**)
- **GBNews**: GB News clips, features and live streams
- **GDCVault**: [*gdcvault*](## "netrc machine") (**Currently broken**)
- **GediDigital**
- **gem.cbc.ca**: [*cbcgem*](## "netrc machine")
@ -527,6 +528,7 @@ # Supported sites
- **GMANetworkVideo**
- **Go**
- **GoDiscovery**
- **GodResource**
- **GodTube**: (**Currently broken**)
- **Gofile**
- **Golem**
@ -630,11 +632,11 @@ # Supported sites
- **iwara:user**: [*iwara*](## "netrc machine")
- **Ixigua**
- **Izlesene**
- **Jable**
- **JablePlaylist**
- **Jamendo**
- **JamendoAlbum**
- **JeuxVideo**: (**Currently broken**)
- **jiocinema**: [*jiocinema*](## "netrc machine")
- **jiocinema:series**: [*jiocinema*](## "netrc machine")
- **jiosaavn:album**
- **jiosaavn:playlist**
- **jiosaavn:song**
@ -974,6 +976,7 @@ # Supported sites
- **NRKTVSeason**
- **NRKTVSeries**
- **NRLTV**: (**Currently broken**)
- **nts.live**
- **ntv.ru**
- **NubilesPorn**: [*nubiles-porn*](## "netrc machine")
- **nuum:live**
@ -1015,7 +1018,6 @@ # Supported sites
- **orf:on**
- **orf:podcast**
- **orf:radio**
- **orf:tvthek**: ORF TVthek
- **OsnatelTV**: [*osnateltv*](## "netrc machine")
- **OsnatelTVLive**: [*osnateltv*](## "netrc machine")
- **OsnatelTVRecordings**: [*osnateltv*](## "netrc machine")
@ -1394,6 +1396,10 @@ # Supported sites
- **SztvHu**
- **t-online.de**: (**Currently broken**)
- **Tagesschau**: (**Currently broken**)
- **TapTapApp**
- **TapTapAppIntl**
- **TapTapMoment**
- **TapTapPostIntl**
- **Tass**: (**Currently broken**)
- **TBS**
- **TBSJPEpisode**
@ -1412,7 +1418,7 @@ # Supported sites
- **TedSeries**
- **TedTalk**
- **Tele13**
- **Tele5**: (**Currently broken**)
- **Tele5**
- **TeleBruxelles**
- **TelecaribePlay**
- **Telecinco**: telecinco.es, cuatro.com and mediaset.es
@ -1452,11 +1458,12 @@ # Supported sites
- **ThreeSpeak**
- **ThreeSpeakUser**
- **TikTok**
- **tiktok:collection**
- **tiktok:effect**: (**Currently broken**)
- **tiktok:live**
- **tiktok:sound**: (**Currently broken**)
- **tiktok:tag**: (**Currently broken**)
- **tiktok:user**: (**Currently broken**)
- **tiktok:user**
- **TLC**
- **TMZ**
- **TNAFlix**
@ -1501,7 +1508,7 @@ # Supported sites
- **tv2play.hu**
- **tv2playseries.hu**
- **TV4**: tv4.se and tv4play.se
- **TV5MondePlus**: TV5MONDE+
- **TV5MONDE**
- **tv5unis**
- **tv5unis:video**
- **tv8.it**
@ -1639,8 +1646,6 @@ # Supported sites
- **voicy**: (**Currently broken**)
- **voicy:channel**: (**Currently broken**)
- **VolejTV**
- **Voot**: [*voot*](## "netrc machine") (**Currently broken**)
- **VootSeries**: [*voot*](## "netrc machine") (**Currently broken**)
- **VoxMedia**
- **VoxMediaVolume**
- **vpro**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
@ -1715,10 +1720,10 @@ # Supported sites
- **wykop:​post:comment**
- **Xanimu**
- **XboxClips**
- **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
- **XHamster**
- **XHamsterEmbed**
- **XHamsterUser**
- **XiaoHongShu**: 小红书
- **ximalaya**: 喜马拉雅FM
- **ximalaya:album**: 喜马拉雅FM 专辑
- **xinpianchang**: xinpianchang.com (**Currently broken**)
@ -1749,8 +1754,12 @@ # Supported sites
- **YouNowLive**
- **YouNowMoment**
- **YouPorn**
- **YourPorn**
- **YourUpload**
- **YouPornCategory**: YouPorn category, with sorting, filtering and pagination
- **YouPornChannel**: YouPorn channel, with sorting and pagination
- **YouPornCollection**: YouPorn collection (user playlist), with sorting and pagination
- **YouPornStar**: YouPorn Pornstar, with description, sorting and pagination
- **YouPornTag**: YouPorn tag (porntags), with sorting, filtering and pagination
- **YouPornVideos**: YouPorn video (browse) playlists, with sorting, filtering and pagination
- **youtube**: YouTube
- **youtube:clip**
- **youtube:favorites**: YouTube liked videos; ":ytfav" keyword (requires cookies)

2
test/test_InfoExtractor.py
View File

@ -1912,7 +1912,7 @@ def test_search_nextjs_data(self):
self.assertEqual(self.ie._search_nextjs_data('', None, fatal=False), {})
self.assertEqual(self.ie._search_nextjs_data('', None, default=None), None)
self.assertEqual(self.ie._search_nextjs_data('', None, default={}), {})
with self.assertRaises(DeprecationWarning):
with self.assertWarns(DeprecationWarning):
self.assertEqual(self.ie._search_nextjs_data('', None, default='{}'), {})

1
test/test_http_proxy.py
View File

@ -99,6 +99,7 @@ class SSLTransport(urllib3.util.ssltransport.SSLTransport):
This allows us to chain multiple TLS connections.
"""
def __init__(self, socket, ssl_context, server_hostname=None, suppress_ragged_eofs=True, server_side=False):
self.incoming = ssl.MemoryBIO()
self.outgoing = ssl.MemoryBIO()

23
test/test_networking.py
View File

@ -6,7 +6,7 @@
import pytest
from yt_dlp.networking.common import Features
from yt_dlp.networking.common import Features, DEFAULT_TIMEOUT
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -523,20 +523,17 @@ def test_read_timeout(self, handler):
def test_connect_timeout(self, handler):
# nothing should be listening on this port
connect_timeout_url = 'http://10.255.255.255'
with handler(timeout=0.01) as rh:
with handler(timeout=0.01) as rh, pytest.raises(TransportError):
now = time.time()
with pytest.raises(TransportError):
validate_and_send(
rh, Request(connect_timeout_url))
assert 0.01 <= time.time() - now < 20
validate_and_send(rh, Request(connect_timeout_url))
assert time.time() - now < DEFAULT_TIMEOUT
with handler() as rh:
with pytest.raises(TransportError):
# Per request timeout, should override handler timeout
now = time.time()
validate_and_send(
rh, Request(connect_timeout_url, extensions={'timeout': 0.01}))
assert 0.01 <= time.time() - now < 20
# Per request timeout, should override handler timeout
request = Request(connect_timeout_url, extensions={'timeout': 0.01})
with handler() as rh, pytest.raises(TransportError):
now = time.time()
validate_and_send(rh, request)
assert time.time() - now < DEFAULT_TIMEOUT
def test_source_address(self, handler):
source_address = f'127.0.0.{random.randint(5, 255)}'

7
test/test_utils.py
View File

@ -5,6 +5,7 @@
import sys
import unittest
import warnings
import datetime as dt
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -27,6 +28,7 @@
ExtractorError,
InAdvancePagedList,
LazyList,
NO_DEFAULT,
OnDemandPagedList,
Popen,
age_restricted,
@ -768,6 +770,11 @@ def test_encode_compat_str(self):
def test_parse_iso8601(self):
self.assertEqual(parse_iso8601('2014-03-23T23:04:26+0100'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T23:04:26-07:00'), 1395641066)
self.assertEqual(parse_iso8601('2014-03-23T23:04:26', timezone=dt.timedelta(hours=-7)), 1395641066)
self.assertEqual(parse_iso8601('2014-03-23T23:04:26', timezone=NO_DEFAULT), None)
# default does not override timezone in date_str
self.assertEqual(parse_iso8601('2014-03-23T23:04:26-07:00', timezone=dt.timedelta(hours=-10)), 1395641066)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26+0000'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26Z'), 1395612266)
self.assertEqual(parse_iso8601('2014-03-23T22:04:26.1234Z'), 1395612266)

20
test/test_websockets.py
View File

@ -3,11 +3,12 @@
# Allow direct execution
import os
import sys
import time
import pytest
from test.helper import verify_address_availability
from yt_dlp.networking.common import Features
from yt_dlp.networking.common import Features, DEFAULT_TIMEOUT
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -202,11 +203,26 @@ def test_raise_http_error(self, handler, status):
({'timeout': sys.float_info.min}, {}),
({}, {'timeout': sys.float_info.min}),
])
def test_timeout(self, handler, params, extensions):
def test_read_timeout(self, handler, params, extensions):
with handler(**params) as rh:
with pytest.raises(TransportError):
ws_validate_and_send(rh, Request(self.ws_base_url, extensions=extensions))
def test_connect_timeout(self, handler):
# nothing should be listening on this port
connect_timeout_url = 'ws://10.255.255.255'
with handler(timeout=0.01) as rh, pytest.raises(TransportError):
now = time.time()
ws_validate_and_send(rh, Request(connect_timeout_url))
assert time.time() - now < DEFAULT_TIMEOUT
# Per request timeout, should override handler timeout
request = Request(connect_timeout_url, extensions={'timeout': 0.01})
with handler() as rh, pytest.raises(TransportError):
now = time.time()
ws_validate_and_send(rh, request)
assert time.time() - now < DEFAULT_TIMEOUT
def test_cookies(self, handler):
cookiejar = YoutubeDLCookieJar()
cookiejar.set_cookie(http.cookiejar.Cookie(

1096
yt_dlp/extractor/_extractors.py
View File

File diff suppressed because it is too large Load Diff

4
yt_dlp/extractor/abc.py
View File

@ -6,10 +6,10 @@
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
dict_get,
ExtractorError,
js_to_json,
dict_get,
int_or_none,
js_to_json,
parse_iso8601,
str_or_none,
traverse_obj,

5
yt_dlp/extractor/abematv.py
View File

@ -12,20 +12,21 @@
import urllib.request
import urllib.response
import uuid
from ..utils.networking import clean_proxies
from .common import InfoExtractor
from ..aes import aes_ecb_decrypt
from ..utils import (
ExtractorError,
OnDemandPagedList,
bytes_to_intlist,
decode_base_n,
int_or_none,
intlist_to_bytes,
OnDemandPagedList,
time_seconds,
traverse_obj,
update_url_query,
)
from ..utils.networking import clean_proxies
def add_opener(ydl, handler): # FIXME: Create proper API in .networking

4
yt_dlp/extractor/acfun.py
View File

@ -3,10 +3,10 @@
float_or_none,
format_field,
int_or_none,
str_or_none,
traverse_obj,
parse_codecs,
parse_qs,
str_or_none,
traverse_obj,
)

4
yt_dlp/extractor/adn.py
View File

@ -10,18 +10,18 @@
from ..compat import compat_b64decode
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
ass_subtitles_timecode,
bytes_to_intlist,
bytes_to_long,
ExtractorError,
float_or_none,
int_or_none,
intlist_to_bytes,
long_to_bytes,
parse_iso8601,
pkcs1pad,
strip_or_none,
str_or_none,
strip_or_none,
try_get,
unified_strdate,
urlencode_postdata,

4
yt_dlp/extractor/adobetv.py
View File

@ -4,11 +4,11 @@
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
ISO639Utils,
OnDemandPagedList,
float_or_none,
int_or_none,
ISO639Utils,
join_nonempty,
OnDemandPagedList,
parse_duration,
str_or_none,
str_to_int,

2
yt_dlp/extractor/airtv.py
View File

@ -5,7 +5,7 @@
int_or_none,
mimetype2ext,
parse_iso8601,
traverse_obj
traverse_obj,
)

1
yt_dlp/extractor/allstar.py
View File

@ -12,7 +12,6 @@
)
from ..utils.traversal import traverse_obj
_FIELDS = '''
_id
clipImageSource

4
yt_dlp/extractor/alphaporno.py
View File

@ -1,9 +1,9 @@
from .common import InfoExtractor
from ..utils import (
parse_iso8601,
int_or_none,
parse_duration,
parse_filesize,
int_or_none,
parse_iso8601,
)

12
yt_dlp/extractor/alura.py
View File

@ -1,17 +1,13 @@
import re
from .common import InfoExtractor
from ..compat import (
compat_urlparse,
)
from ..compat import compat_urlparse
from ..utils import (
ExtractorError,
clean_html,
int_or_none,
urlencode_postdata,
urljoin,
int_or_none,
clean_html,
ExtractorError
)

2
yt_dlp/extractor/amara.py
View File

@ -1,6 +1,6 @@
from .common import InfoExtractor
from .youtube import YoutubeIE
from .vimeo import VimeoIE
from .youtube import YoutubeIE
from ..utils import (
int_or_none,
parse_iso8601,

2
yt_dlp/extractor/amp.py
View File

@ -1,7 +1,7 @@
from .common import InfoExtractor
from ..utils import (
determine_ext,
ExtractorError,
determine_ext,
int_or_none,
mimetype2ext,
parse_iso8601,

2
yt_dlp/extractor/anchorfm.py
View File

@ -5,7 +5,7 @@
int_or_none,
str_or_none,
traverse_obj,
unified_timestamp
unified_timestamp,
)

2
yt_dlp/extractor/angel.py
View File

@ -1,7 +1,7 @@
import re
from .common import InfoExtractor
from ..utils import url_or_none, merge_dicts
from ..utils import merge_dicts, url_or_none
class AngelIE(InfoExtractor):

5
yt_dlp/extractor/appleconnect.py
View File

@ -1,8 +1,5 @@
from .common import InfoExtractor
from ..utils import (
str_to_int,
ExtractorError
)
from ..utils import ExtractorError, str_to_int
class AppleConnectIE(InfoExtractor):

2
yt_dlp/extractor/appletrailers.py
View File

@ -1,5 +1,5 @@
import re
import json
import re
from .common import InfoExtractor
from ..compat import compat_urlparse

2
yt_dlp/extractor/arnes.py
View File

@ -4,8 +4,8 @@
compat_urllib_parse_urlparse,
)
from ..utils import (
format_field,
float_or_none,
format_field,
int_or_none,
parse_iso8601,
remove_start,

64
yt_dlp/extractor/arte.py
View File

@ -5,6 +5,7 @@
ExtractorError,
GeoRestrictedError,
int_or_none,
join_nonempty,
parse_iso8601,
parse_qs,
strip_or_none,
@ -31,20 +32,6 @@ class ArteTVIE(ArteTVBaseIE):
_TESTS = [{
'url': 'https://www.arte.tv/en/videos/088501-000-A/mexico-stealing-petrol-to-survive/',
'only_matching': True,
}, {
'url': 'https://www.arte.tv/pl/videos/100103-000-A/usa-dyskryminacja-na-porodowce/',
'info_dict': {
'id': '100103-000-A',
'title': 'USA: Dyskryminacja na porodówce',
'description': 'md5:242017b7cce59ffae340a54baefcafb1',
'alt_title': 'ARTE Reportage',
'upload_date': '20201103',
'duration': 554,
'thumbnail': r're:https://api-cdn\.arte\.tv/.+940x530',
'timestamp': 1604417980,
'ext': 'mp4',
},
'params': {'skip_download': 'm3u8'}
}, {
'note': 'No alt_title',
'url': 'https://www.arte.tv/fr/videos/110371-000-A/la-chaleur-supplice-des-arbres-de-rue/',
@ -58,6 +45,23 @@ class ArteTVIE(ArteTVBaseIE):
}, {
'url': 'https://www.arte.tv/de/videos/110203-006-A/zaz/',
'only_matching': True,
}, {
'url': 'https://www.arte.tv/fr/videos/109067-000-A/la-loi-de-teheran/',
'info_dict': {
'id': '109067-000-A',
'ext': 'mp4',
'description': 'md5:d2ca367b8ecee028dddaa8bd1aebc739',
'timestamp': 1713927600,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/3rR6PLzfbigSkkeHtkCZNF/940x530',
'duration': 7599,
'title': 'La loi de Téhéran',
'upload_date': '20240424',
'subtitles': {
'fr': 'mincount:1',
'fr-acc': 'mincount:1',
'fr-forced': 'mincount:1',
},
},
}, {
'note': 'age-restricted',
'url': 'https://www.arte.tv/de/videos/006785-000-A/the-element-of-crime/',
@ -71,23 +75,7 @@ class ArteTVIE(ArteTVBaseIE):
'upload_date': '20230930',
'ext': 'mp4',
},
}, {
'url': 'https://www.arte.tv/de/videos/085374-003-A/im-hohen-norden-geboren/',
'info_dict': {
'id': '085374-003-A',
'ext': 'mp4',
'description': 'md5:ab79ec7cc472a93164415b4e4916abf9',
'timestamp': 1702872000,
'thumbnail': 'https://api-cdn.arte.tv/img/v2/image/TnyHBfPxv3v2GEY3suXGZP/940x530',
'duration': 2594,
'title': 'Die kurze Zeit der Jugend',
'alt_title': 'Im hohen Norden geboren',
'upload_date': '20231218',
'subtitles': {
'fr': 'mincount:1',
'fr-acc': 'mincount:1',
},
},
'skip': '404 Not Found',
}]
_GEO_BYPASS = True
@ -143,16 +131,18 @@ def _fix_accessible_subs_locale(subs):
updated_subs = {}
for lang, sub_formats in subs.items():
for fmt in sub_formats:
if fmt.get('url', '').endswith('-MAL.m3u8'):
lang += '-acc'
updated_subs.setdefault(lang, []).append(fmt)
url = fmt.get('url') or ''
suffix = ('acc' if url.endswith('-MAL.m3u8')
else 'forced' if '_VO' not in url
else None)
updated_subs.setdefault(join_nonempty(lang, suffix), []).append(fmt)
return updated_subs
def _real_extract(self, url):
mobj = self._match_valid_url(url)
video_id = mobj.group('id')
lang = mobj.group('lang') or mobj.group('lang_2')
langauge_code = self._LANG_MAP.get(lang)
language_code = self._LANG_MAP.get(lang)
config = self._download_json(f'{self._API_BASE}/config/{lang}/{video_id}', video_id, headers={
'x-validated-age': '18'
@ -180,10 +170,10 @@ def _real_extract(self, url):
m = self._VERSION_CODE_RE.match(stream_version_code)
if m:
lang_pref = int(''.join('01'[x] for x in (
m.group('vlang') == langauge_code, # we prefer voice in the requested language
m.group('vlang') == language_code, # we prefer voice in the requested language
not m.group('audio_desc'), # and not the audio description version
bool(m.group('original_voice')), # but if voice is not in the requested language, at least choose the original voice
m.group('sub_lang') == langauge_code, # if subtitles are present, we prefer them in the requested language
m.group('sub_lang') == language_code, # if subtitles are present, we prefer them in the requested language
not m.group('has_sub'), # but we prefer no subtitles otherwise
not m.group('sdh_sub'), # and we prefer not the hard-of-hearing subtitles if there are subtitles
)))

2
yt_dlp/extractor/atvat.py
View File

@ -2,10 +2,10 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
float_or_none,
jwt_encode_hs256,
try_get,
ExtractorError,
)

2
yt_dlp/extractor/awaan.py
View File

@ -2,8 +2,8 @@
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse_urlencode,
compat_str,
compat_urllib_parse_urlencode,
)
from ..utils import (
format_field,

4
yt_dlp/extractor/banbye.py
View File

@ -2,12 +2,12 @@
from .common import InfoExtractor
from ..compat import (
compat_urllib_parse_urlparse,
compat_parse_qs,
compat_urllib_parse_urlparse,
)
from ..utils import (
format_field,
InAdvancePagedList,
format_field,
traverse_obj,
unified_timestamp,
)

6
yt_dlp/extractor/bannedvideo.py
View File

@ -2,11 +2,11 @@
from .common import InfoExtractor
from ..utils import (
try_get,
int_or_none,
url_or_none,
float_or_none,
int_or_none,
try_get,
unified_timestamp,
url_or_none,
)

1
yt_dlp/extractor/beeg.py
View File

@ -1,5 +1,4 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
str_or_none,

2
yt_dlp/extractor/bleacherreport.py
View File

@ -1,5 +1,5 @@
from .common import InfoExtractor
from .amp import AMPIE
from .common import InfoExtractor
from ..utils import (
ExtractorError,
int_or_none,

2
yt_dlp/extractor/blogger.py
View File

@ -1,3 +1,4 @@
from .common import InfoExtractor
from ..utils import (
mimetype2ext,
parse_duration,
@ -5,7 +6,6 @@
str_or_none,
traverse_obj,
)
from .common import InfoExtractor
class BloggerIE(InfoExtractor):

1
yt_dlp/extractor/bostonglobe.py
View File

@ -1,7 +1,6 @@
import re
from .common import InfoExtractor
from ..utils import (
extract_attributes,
)

6
yt_dlp/extractor/boxcast.py
View File

@ -1,9 +1,5 @@
from .common import InfoExtractor
from ..utils import (
js_to_json,
traverse_obj,
unified_timestamp
)
from ..utils import js_to_json, traverse_obj, unified_timestamp
class BoxCastVideoIE(InfoExtractor):

2
yt_dlp/extractor/brainpop.py
View File

@ -6,7 +6,7 @@
classproperty,
int_or_none,
traverse_obj,
urljoin
urljoin,
)

4
yt_dlp/extractor/brightcove.py
View File

@ -12,10 +12,11 @@
)
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
UnsupportedError,
clean_html,
dict_get,
extract_attributes,
ExtractorError,
find_xpath_attr,
fix_xml_ampersands,
float_or_none,
@ -29,7 +30,6 @@
try_get,
unescapeHTML,
unsmuggle_url,
UnsupportedError,
update_url_query,
url_or_none,
)

13
yt_dlp/extractor/brilliantpala.py
View File

@ -27,8 +27,17 @@ def _get_logged_in_username(self, url, video_id):
r'"username"\s*:\s*"(?P<username>[^"]+)"', webpage, 'logged-in username')
def _perform_login(self, username, password):
login_form = self._hidden_inputs(self._download_webpage(
self._LOGIN_API, None, 'Downloading login page'))
login_page, urlh = self._download_webpage_handle(
self._LOGIN_API, None, 'Downloading login page', expected_status=401)
if urlh.status != 401 and not urlh.url.startswith(self._LOGIN_API):
self.write_debug('Cookies are valid, no login required.')
return
if urlh.status == 401:
self.write_debug('Got HTTP Error 401; cookies have been invalidated')
login_page = self._download_webpage(self._LOGIN_API, None, 'Re-downloading login page')
login_form = self._hidden_inputs(login_page)
login_form.update({
'username': username,
'password': password,

74
yt_dlp/extractor/caffeinetv.py Normal file
View File

@ -0,0 +1,74 @@
from .common import InfoExtractor
from ..utils import (
determine_ext,
int_or_none,
parse_iso8601,
traverse_obj,
urljoin,
)
class CaffeineTVIE(InfoExtractor):
_VALID_URL = r'https?://(?:www\.)?caffeine\.tv/[^/?#]+/video/(?P<id>[\da-f-]+)'
_TESTS = [{
'url': 'https://www.caffeine.tv/TsuSurf/video/cffc0a00-e73f-11ec-8080-80017d29f26e',
'info_dict': {
'id': 'cffc0a00-e73f-11ec-8080-80017d29f26e',
'ext': 'mp4',
'title': 'GOOOOD MORNINNNNN #highlights',
'timestamp': 1654702180,
'upload_date': '20220608',
'uploader': 'RahJON Wicc',
'uploader_id': 'TsuSurf',
'duration': 3145,
'age_limit': 17,
'thumbnail': 'https://www.caffeine.tv/broadcasts/776b6f84-9cd5-42e3-af1d-4a776eeed697/replay/lobby.jpg',
'comment_count': int,
'view_count': int,
'like_count': int,
'tags': ['highlights', 'battlerap'],
},
'params': {
'skip_download': 'm3u8',
},
}]
def _real_extract(self, url):
video_id = self._match_id(url)
json_data = self._download_json(
f'https://api.caffeine.tv/social/public/activity/{video_id}', video_id)
broadcast_info = traverse_obj(json_data, ('broadcast_info', {dict})) or {}
video_url = broadcast_info['video_url']
ext = determine_ext(video_url)
if ext == 'm3u8':
formats = self._extract_m3u8_formats(video_url, video_id, 'mp4')
else:
formats = [{'url': video_url}]
return {
'id': video_id,
'formats': formats,
**traverse_obj(json_data, {
'like_count': ('like_count', {int_or_none}),
'view_count': ('view_count', {int_or_none}),
'comment_count': ('comment_count', {int_or_none}),
'tags': ('tags', ..., {str}, {lambda x: x or None}),
'uploader': ('user', 'name', {str}),
'uploader_id': (((None, 'user'), 'username'), {str}, any),
'is_live': ('is_live', {bool}),
}),
**traverse_obj(broadcast_info, {
'title': ('broadcast_title', {str}),
'duration': ('content_duration', {int_or_none}),
'timestamp': ('broadcast_start_time', {parse_iso8601}),
'thumbnail': ('preview_image_path', {lambda x: urljoin(url, x)}),
}),
'age_limit': {
# assume Apple Store ratings: https://en.wikipedia.org/wiki/Mobile_software_content_rating_system
'FOUR_PLUS': 0,
'NINE_PLUS': 9,
'TWELVE_PLUS': 12,
'SEVENTEEN_PLUS': 17,
}.get(broadcast_info.get('content_rating'), 17),
}

6
yt_dlp/extractor/cbs.py
View File

@ -5,14 +5,14 @@
from ..utils import (
ExtractorError,
extract_attributes,
find_xpath_attr,
get_element_html_by_id,
int_or_none,
find_xpath_attr,
smuggle_url,
xpath_element,
xpath_text,
update_url_query,
url_or_none,
xpath_element,
xpath_text,
)

2
yt_dlp/extractor/ceskatelevize.py
View File

@ -101,7 +101,7 @@ def _real_extract(self, url):
site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
playlist_title = self._og_search_title(webpage, default=None)
if site_name and playlist_title:
playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, 1)[0]
playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, maxsplit=1)[0]
playlist_description = self._og_search_description(webpage, default=None)
if playlist_description:
playlist_description = playlist_description.replace('\xa0', ' ')

1
yt_dlp/extractor/cinetecamilano.py
View File

@ -1,4 +1,5 @@
import json
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (

4
yt_dlp/extractor/clippit.py
View File

@ -1,11 +1,11 @@
import re
from .common import InfoExtractor
from ..utils import (
parse_iso8601,
qualities,
)
import re
class ClippitIE(InfoExtractor):

58
yt_dlp/extractor/common.py
View File

@ -1,5 +1,6 @@
import base64
import collections
import functools
import getpass
import hashlib
import http.client
@ -21,7 +22,6 @@
import urllib.request
import xml.etree.ElementTree
from ..compat import functools # isort: split
from ..compat import (
compat_etree_fromstring,
compat_expanduser,
@ -1036,13 +1036,14 @@ def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errno
webpage_bytes = urlh.read()
if prefix is not None:
webpage_bytes = prefix + webpage_bytes
url_or_request = self._create_request(url_or_request, data)
if self.get_param('dump_intermediate_pages', False):
self.to_screen('Dumping request to ' + urlh.url)
dump = base64.b64encode(webpage_bytes).decode('ascii')
self._downloader.to_screen(dump)
if self.get_param('write_pages'):
filename = self._request_dump_filename(urlh.url, video_id, url_or_request.data)
if isinstance(url_or_request, Request):
data = self._create_request(url_or_request, data).data
filename = self._request_dump_filename(urlh.url, video_id, data)
self.to_screen(f'Saving request to {filename}')
with open(filename, 'wb') as outf:
outf.write(webpage_bytes)
@ -2450,7 +2451,7 @@ def _parse_smil_formats_and_subtitles(
})
continue
src_url = src if src.startswith('http') else urllib.parse.urljoin(base, src)
src_url = src if src.startswith('http') else urllib.parse.urljoin(f'{base}/', src)
src_url = src_url.strip()
if proto == 'm3u8' or src_ext == 'm3u8':
@ -3383,23 +3384,16 @@ def manifest_url(manifest):
return formats
def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
mobj = re.search(
r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
webpage)
if mobj:
try:
jwplayer_data = self._parse_json(mobj.group('options'),
video_id=video_id,
transform_source=transform_source)
except ExtractorError:
pass
else:
if isinstance(jwplayer_data, dict):
return jwplayer_data
return self._search_json(
r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
webpage, 'JWPlayer data', video_id,
# must be a {...} or sequence, ending
contains_pattern=r'\{(?s:.*)}(?(load)(?:\s*,\s*\{(?s:.*)})*)', end_pattern=r'(?(load)\]|\))',
transform_source=transform_source, default=None)
def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
def _extract_jwplayer_data(self, webpage, video_id, *args, transform_source=js_to_json, **kwargs):
jwplayer_data = self._find_jwplayer_data(
webpage, video_id, transform_source=js_to_json)
webpage, video_id, transform_source=transform_source)
return self._parse_jwplayer_data(
jwplayer_data, video_id, *args, **kwargs)
@ -3431,22 +3425,14 @@ def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
subtitles = {}
tracks = video_data.get('tracks')
if tracks and isinstance(tracks, list):
for track in tracks:
if not isinstance(track, dict):
continue
track_kind = track.get('kind')
if not track_kind or not isinstance(track_kind, str):
continue
if track_kind.lower() not in ('captions', 'subtitles'):
continue
track_url = urljoin(base_url, track.get('file'))
if not track_url:
continue
subtitles.setdefault(track.get('label') or 'en', []).append({
'url': self._proto_relative_url(track_url)
})
for track in traverse_obj(video_data, (
'tracks', lambda _, v: v['kind'].lower() in ('captions', 'subtitles'))):
track_url = urljoin(base_url, track.get('file'))
if not track_url:
continue
subtitles.setdefault(track.get('label') or 'en', []).append({
'url': self._proto_relative_url(track_url)
})
entry = {
'id': this_video_id,
@ -3531,7 +3517,7 @@ def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
# See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
# of jwplayer.flash.swf
rtmp_url_parts = re.split(
r'((?:mp4|mp3|flv):)', source_url, 1)
r'((?:mp4|mp3|flv):)', source_url, maxsplit=1)
if len(rtmp_url_parts) == 3:
rtmp_url, prefix, play_path = rtmp_url_parts
a_format.update({

2
yt_dlp/extractor/corus.py
View File

@ -1,7 +1,7 @@
from .theplatform import ThePlatformFeedIE
from ..utils import (
dict_get,
ExtractorError,
dict_get,
float_or_none,
int_or_none,
)

2
yt_dlp/extractor/crackle.py
View File

@ -6,6 +6,7 @@
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
determine_ext,
float_or_none,
int_or_none,
@ -13,7 +14,6 @@
parse_age_limit,
parse_duration,
url_or_none,
ExtractorError
)

30
yt_dlp/extractor/crunchyroll.py
View File

@ -2,6 +2,7 @@
import uuid
from .common import InfoExtractor
from ..networking import Request
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
@ -24,6 +25,7 @@ class CrunchyrollBaseIE(InfoExtractor):
_BASE_URL = 'https://www.crunchyroll.com'
_API_BASE = 'https://api.crunchyroll.com'
_NETRC_MACHINE = 'crunchyroll'
_SWITCH_USER_AGENT = 'Crunchyroll/1.8.0 Nintendo Switch/12.3.12.0 UE4/4.27'
_REFRESH_TOKEN = None
_AUTH_HEADERS = None
_AUTH_EXPIRY = None
@ -179,10 +181,19 @@ def _extract_stream(self, identifier, display_id=None):
display_id = identifier
self._update_auth()
stream_response = self._download_json(
f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play',
display_id, note='Downloading stream info', errnote='Failed to download stream info',
headers=CrunchyrollBaseIE._AUTH_HEADERS)
headers = {**CrunchyrollBaseIE._AUTH_HEADERS, 'User-Agent': self._SWITCH_USER_AGENT}
try:
stream_response = self._download_json(
f'https://cr-play-service.prd.crunchyrollsvc.com/v1/{identifier}/console/switch/play',
display_id, note='Downloading stream info', errnote='Failed to download stream info', headers=headers)
except ExtractorError as error:
if self.get_param('ignore_no_formats_error'):
self.report_warning(error.orig_msg)
return [], {}
elif isinstance(error.cause, HTTPError) and error.cause.status == 420:
raise ExtractorError(
'You have reached the rate-limit for active streams; try again later', expected=True)
raise
available_formats = {'': ('', '', stream_response['url'])}
for hardsub_lang, stream in traverse_obj(stream_response, ('hardSubs', {dict.items}, lambda _, v: v[1]['url'])):
@ -211,7 +222,7 @@ def _extract_stream(self, identifier, display_id=None):
fatal=False, note=f'Downloading {f"{format_id} " if hardsub_lang else ""}MPD manifest')
self._merge_subtitles(dash_subs, target=subtitles)
else:
continue # XXX: Update this if/when meta mpd formats are working
continue # XXX: Update this if meta mpd formats work; will be tricky with token invalidation
for f in adaptive_formats:
if f.get('acodec') != 'none':
f['language'] = audio_locale
@ -221,6 +232,15 @@ def _extract_stream(self, identifier, display_id=None):
for locale, subtitle in traverse_obj(stream_response, (('subtitles', 'captions'), {dict.items}, ...)):
subtitles.setdefault(locale, []).append(traverse_obj(subtitle, {'url': 'url', 'ext': 'format'}))
# Invalidate stream token to avoid rate-limit
error_msg = 'Unable to invalidate stream token; you may experience rate-limiting'
if stream_token := stream_response.get('token'):
self._request_webpage(Request(
f'https://cr-play-service.prd.crunchyrollsvc.com/v1/token/{identifier}/{stream_token}/inactive',
headers=headers, method='PATCH'), display_id, 'Invalidating stream token', error_msg, fatal=False)
else:
self.report_warning(error_msg)
return formats, subtitles

6
yt_dlp/extractor/cspan.py
View File

@ -1,10 +1,12 @@
import re
from .common import InfoExtractor
from .senategov import SenateISVPIE
from .ustream import UstreamIE
from ..compat import compat_HTMLParseError
from ..utils import (
determine_ext,
ExtractorError,
determine_ext,
extract_attributes,
find_xpath_attr,
get_element_by_attribute,
@ -19,8 +21,6 @@
str_to_int,
unescapeHTML,
)
from .senategov import SenateISVPIE
from .ustream import UstreamIE
class CSpanIE(InfoExtractor):

2
yt_dlp/extractor/ctsnews.py
View File

@ -1,6 +1,6 @@
from .common import InfoExtractor
from ..utils import unified_timestamp
from .youtube import YoutubeIE
from ..utils import unified_timestamp
class CtsNewsIE(InfoExtractor):

2
yt_dlp/extractor/dailymail.py
View File

@ -1,8 +1,8 @@
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
int_or_none,
determine_protocol,
int_or_none,
try_get,
unescapeHTML,
)

2
yt_dlp/extractor/damtomo.py
View File

@ -1,8 +1,8 @@
import re
from .common import InfoExtractor
from ..utils import ExtractorError, clean_html, int_or_none, try_get, unified_strdate
from ..compat import compat_str
from ..utils import ExtractorError, clean_html, int_or_none, try_get, unified_strdate
class DamtomoBaseIE(InfoExtractor):

197
yt_dlp/extractor/dangalplay.py Normal file
View File

@ -0,0 +1,197 @@
import hashlib
import json
import re
import time
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import ExtractorError, int_or_none, join_nonempty, url_or_none
from ..utils.traversal import traverse_obj
class DangalPlayBaseIE(InfoExtractor):
_NETRC_MACHINE = 'dangalplay'
_OTV_USER_ID = None
_LOGIN_HINT = 'Pass credentials as -u "token" -p "USER_ID" where USER_ID is the `otv_user_id` in browser local storage'
_API_BASE = 'https://ottapi.dangalplay.com'
_AUTH_TOKEN = 'jqeGWxRKK7FK5zEk3xCM' # from https://www.dangalplay.com/main.48ad19e24eb46acccef3.js
_SECRET_KEY = 'f53d31a4377e4ef31fa0' # same as above
def _perform_login(self, username, password):
if self._OTV_USER_ID:
return
if username != 'token' or not re.fullmatch(r'[\da-f]{32}', password):
raise ExtractorError(self._LOGIN_HINT, expected=True)
self._OTV_USER_ID = password
def _real_initialize(self):
if not self._OTV_USER_ID:
self.raise_login_required(f'Login required. {self._LOGIN_HINT}', method=None)
def _extract_episode_info(self, metadata, episode_slug, series_slug):
return {
'display_id': episode_slug,
'episode_number': int_or_none(self._search_regex(
r'ep-(?:number-)?(\d+)', episode_slug, 'episode number', default=None)),
'season_number': int_or_none(self._search_regex(
r'season-(\d+)', series_slug, 'season number', default='1')),
'series': series_slug,
**traverse_obj(metadata, {
'id': ('content_id', {str}),
'title': ('display_title', {str}),
'episode': ('title', {str}),
'series': ('show_name', {str}, {lambda x: x or None}),
'series_id': ('catalog_id', {str}),
'duration': ('duration', {int_or_none}),
'release_timestamp': ('release_date_uts', {int_or_none}),
}),
}
def _call_api(self, path, display_id, note='Downloading JSON metadata', fatal=True, query={}):
return self._download_json(
f'{self._API_BASE}/{path}', display_id, note, fatal=fatal,
headers={'Accept': 'application/json'}, query={
'auth_token': self._AUTH_TOKEN,
'region': 'IN',
**query,
})
class DangalPlayIE(DangalPlayBaseIE):
IE_NAME = 'dangalplay'
_VALID_URL = r'https?://(?:www\.)?dangalplay.com/shows/(?P<series>[^/?#]+)/(?P<id>(?!episodes)[^/?#]+)/?(?:$|[?#])'
_TESTS = [{
'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-2/kitani-mohabbat-hai-season-2-ep-number-01',
'info_dict': {
'id': '647c61dc1e7171310dcd49b4',
'ext': 'mp4',
'release_timestamp': 1262304000,
'episode_number': 1,
'episode': 'EP 1 | KITANI MOHABBAT HAI SEASON 2',
'series': 'kitani-mohabbat-hai-season-2',
'season_number': 2,
'title': 'EP 1 | KITANI MOHABBAT HAI SEASON 2',
'release_date': '20100101',
'duration': 2325,
'season': 'Season 2',
'display_id': 'kitani-mohabbat-hai-season-2-ep-number-01',
'series_id': '645c9ea41e717158ca574966',
},
}, {
'url': 'https://www.dangalplay.com/shows/milke-bhi-hum-na-mile/milke-bhi-hum-na-mile-ep-number-01',
'info_dict': {
'id': '65d31d9ba73b9c3abd14a7f3',
'ext': 'mp4',
'episode': 'EP 1 | MILKE BHI HUM NA MILE',
'release_timestamp': 1708367411,
'episode_number': 1,
'season': 'Season 1',
'title': 'EP 1 | MILKE BHI HUM NA MILE',
'duration': 156048,
'release_date': '20240219',
'season_number': 1,
'series': 'MILKE BHI HUM NA MILE',
'series_id': '645c9ea41e717158ca574966',
'display_id': 'milke-bhi-hum-na-mile-ep-number-01',
},
}]
def _generate_api_data(self, data):
catalog_id = data['catalog_id']
content_id = data['content_id']
timestamp = str(int(time.time()))
unhashed = ''.join((catalog_id, content_id, self._OTV_USER_ID, timestamp, self._SECRET_KEY))
return json.dumps({
'catalog_id': catalog_id,
'content_id': content_id,
'category': '',
'region': 'IN',
'auth_token': self._AUTH_TOKEN,
'id': self._OTV_USER_ID,
'md5': hashlib.md5(unhashed.encode()).hexdigest(),
'ts': timestamp,
}, separators=(',', ':')).encode()
def _real_extract(self, url):
series_slug, episode_slug = self._match_valid_url(url).group('series', 'id')
metadata = self._call_api(
f'catalogs/shows/{series_slug}/episodes/{episode_slug}.gzip',
episode_slug, query={'item_language': ''})['data']
try:
details = self._download_json(
f'{self._API_BASE}/v2/users/get_all_details.gzip', episode_slug,
'Downloading playback details JSON', headers={
'Accept': 'application/json',
'Content-Type': 'application/json',
}, data=self._generate_api_data(metadata))['data']
except ExtractorError as e:
if isinstance(e.cause, HTTPError) and e.cause.status == 422:
error_info = traverse_obj(e.cause.response.read().decode(), ({json.loads}, 'error', {dict})) or {}
if error_info.get('code') == '1016':
self.raise_login_required(
f'Your token has expired or is invalid. {self._LOGIN_HINT}', method=None)
elif msg := error_info.get('message'):
raise ExtractorError(msg)
raise
m3u8_url = traverse_obj(details, (
('adaptive_url', ('adaptive_urls', 'hd', 'hls', ..., 'playback_url')), {url_or_none}, any))
formats, subtitles = self._extract_m3u8_formats_and_subtitles(m3u8_url, episode_slug, 'mp4')
return {
'formats': formats,
'subtitles': subtitles,
**self._extract_episode_info(metadata, episode_slug, series_slug),
}
class DangalPlaySeasonIE(DangalPlayBaseIE):
IE_NAME = 'dangalplay:season'
_VALID_URL = r'https?://(?:www\.)?dangalplay.com/shows/(?P<id>[^/?#]+)(?:/(?P<sub>ep-[^/?#]+)/episodes)?/?(?:$|[?#])'
_TESTS = [{
'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-1',
'playlist_mincount': 170,
'info_dict': {
'id': 'kitani-mohabbat-hai-season-1',
},
}, {
'url': 'https://www.dangalplay.com/shows/kitani-mohabbat-hai-season-1/ep-01-30-1/episodes',
'playlist_count': 30,
'info_dict': {
'id': 'kitani-mohabbat-hai-season-1-ep-01-30-1',
},
}, {
# 1 season only, series page is season page
'url': 'https://www.dangalplay.com/shows/milke-bhi-hum-na-mile',
'playlist_mincount': 15,
'info_dict': {
'id': 'milke-bhi-hum-na-mile',
},
}]
def _entries(self, subcategories, series_slug):
for subcategory in subcategories:
data = self._call_api(
f'catalogs/shows/items/{series_slug}/subcategories/{subcategory}/episodes.gzip',
series_slug, f'Downloading episodes JSON for {subcategory}', fatal=False, query={
'order_by': 'asc',
'status': 'published',
})
for ep in traverse_obj(data, ('data', 'items', lambda _, v: v['friendly_id'])):
episode_slug = ep['friendly_id']
yield self.url_result(
f'https://www.dangalplay.com/shows/{series_slug}/{episode_slug}',
DangalPlayIE, **self._extract_episode_info(ep, episode_slug, series_slug))
def _real_extract(self, url):
series_slug, subcategory = self._match_valid_url(url).group('id', 'sub')
subcategories = [subcategory] if subcategory else traverse_obj(
self._call_api(
f'catalogs/shows/items/{series_slug}.gzip', series_slug,
'Downloading season info JSON', query={'item_language': ''}),
('data', 'subcategories', ..., 'friendly_id', {str}))
return self.playlist_result(
self._entries(subcategories, series_slug), join_nonempty(series_slug, subcategory))

4
yt_dlp/extractor/democracynow.py
View File

@ -1,11 +1,11 @@
import re
import os.path
import re
from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
url_basename,
remove_start,
url_basename,
)

1
yt_dlp/extractor/digitalconcerthall.py
View File

@ -1,5 +1,4 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
parse_resolution,

2
yt_dlp/extractor/discoverygo.py
View File

@ -2,9 +2,9 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
determine_ext,
extract_attributes,
ExtractorError,
int_or_none,
parse_age_limit,
remove_end,

4
yt_dlp/extractor/disney.py
View File

@ -2,10 +2,10 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
unified_strdate,
determine_ext,
int_or_none,
join_nonempty,
unified_strdate,
update_url_query,
)

2
yt_dlp/extractor/douyutv.py
View File

@ -1,5 +1,5 @@
import time
import hashlib
import time
import urllib
import uuid

45
yt_dlp/extractor/dplay.py
View File

@ -4,8 +4,8 @@
from .common import InfoExtractor
from ..networking.exceptions import HTTPError
from ..utils import (
determine_ext,
ExtractorError,
determine_ext,
float_or_none,
int_or_none,
remove_start,
@ -355,12 +355,10 @@ def _download_video_playback_info(self, disco_base, video_id, headers):
video_id, headers=headers, data=json.dumps({
'deviceInfo': {
'adBlocker': False,
'drmSupported': False,
},
'videoId': video_id,
'wisteriaProperties': {
'platform': 'desktop',
'product': self._PRODUCT,
},
'wisteriaProperties': {},
}).encode('utf-8'))['data']['attributes']['streaming']
def _real_extract(self, url):
@ -878,10 +876,31 @@ def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
})
class DiscoveryNetworksDeIE(DPlayBaseIE):
class DiscoveryNetworksDeIE(DiscoveryPlusBaseIE):
_VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:tlc|dmax)\.de|dplay\.co\.uk)/(?:programme|show|sendungen)/(?P<programme>[^/]+)/(?:video/)?(?P<alternate_id>[^/]+)'
_TESTS = [{
'url': 'https://dmax.de/sendungen/goldrausch-in-australien/german-gold',
'info_dict': {
'id': '4756322',
'ext': 'mp4',
'title': 'German Gold',
'description': 'md5:f3073306553a8d9b40e6ac4cdbf09fc6',
'display_id': 'goldrausch-in-australien/german-gold',
'episode': 'Episode 1',
'episode_number': 1,
'season': 'Season 5',
'season_number': 5,
'series': 'Goldrausch in Australien',
'duration': 2648.0,
'upload_date': '20230517',
'timestamp': 1684357500,
'creators': ['DMAX'],
'thumbnail': 'https://eu1-prod-images.disco-api.com/2023/05/09/f72fb510-7992-3b12-af7f-f16a2c22d1e3.jpeg',
'tags': ['schatzsucher', 'schatz', 'nugget', 'bodenschätze', 'down under', 'australien', 'goldrausch'],
},
'params': {'skip_download': 'm3u8'},
}, {
'url': 'https://www.tlc.de/programme/breaking-amish/video/die-welt-da-drauen/DCB331270001100',
'info_dict': {
'id': '78867',
@ -901,9 +920,7 @@ class DiscoveryNetworksDeIE(DPlayBaseIE):
'season_number': 1,
'thumbnail': r're:https://.+\.jpg',
},
'params': {
'skip_download': True,
},
'skip': '404 Not Found',
}, {
'url': 'https://www.dmax.de/programme/dmax-highlights/video/tuning-star-sidney-hoffmann-exklusiv-bei-dmax/191023082312316',
'only_matching': True,
@ -920,8 +937,14 @@ def _real_extract(self, url):
country = 'GB' if domain == 'dplay.co.uk' else 'DE'
realm = 'questuk' if country == 'GB' else domain.replace('.', '')
return self._get_disco_api_info(
url, '%s/%s' % (programme, alternate_id),
'sonic-eu1-prod.disco-api.com', realm, country)
url, f'{programme}/{alternate_id}', 'eu1-prod.disco-api.com', realm, country)
def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
headers.update({
'x-disco-params': f'realm={realm}',
'x-disco-client': 'Alps:HyogaPlayer:0.0.0',
'Authorization': self._get_auth(disco_base, display_id, realm),
})
class DiscoveryPlusShowBaseIE(DPlayBaseIE):

2
yt_dlp/extractor/drtuber.py
View File

@ -2,8 +2,8 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
NO_DEFAULT,
int_or_none,
parse_duration,
str_to_int,
)

2
yt_dlp/extractor/duboku.py
View File

@ -5,9 +5,9 @@
from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
ExtractorError,
clean_html,
extract_attributes,
ExtractorError,
get_elements_by_class,
int_or_none,
js_to_json,

4
yt_dlp/extractor/dvtv.py
View File

@ -2,15 +2,15 @@
from .common import InfoExtractor
from ..utils import (
determine_ext,
ExtractorError,
determine_ext,
int_or_none,
join_nonempty,
js_to_json,
mimetype2ext,
parse_iso8601,
try_get,
unescapeHTML,
parse_iso8601,
)

2
yt_dlp/extractor/dw.py
View File

@ -1,10 +1,10 @@
from .common import InfoExtractor
from ..compat import compat_urlparse
from ..utils import (
int_or_none,
unified_strdate,
url_or_none,
)
from ..compat import compat_urlparse
class DWIE(InfoExtractor):

4
yt_dlp/extractor/ertgr.py
View File

@ -4,15 +4,15 @@
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
ExtractorError,
clean_html,
determine_ext,
ExtractorError,
dict_get,
int_or_none,
merge_dicts,
parse_qs,
parse_age_limit,
parse_iso8601,
parse_qs,
str_or_none,
try_get,
url_or_none,

2
yt_dlp/extractor/europa.py
View File

@ -8,7 +8,7 @@
qualities,
traverse_obj,
unified_strdate,
xpath_text
xpath_text,
)

3
yt_dlp/extractor/euscreen.py
View File

@ -1,8 +1,7 @@
from .common import InfoExtractor
from ..utils import (
parse_duration,
js_to_json,
parse_duration,
)

4
yt_dlp/extractor/eyedotv.py
View File

@ -1,8 +1,8 @@
from .common import InfoExtractor
from ..utils import (
xpath_text,
parse_duration,
ExtractorError,
parse_duration,
xpath_text,
)

8
yt_dlp/extractor/fancode.py
View File

@ -1,12 +1,6 @@
from .common import InfoExtractor
from ..compat import compat_str
from ..utils import (
parse_iso8601,
ExtractorError,
try_get,
mimetype2ext
)
from ..utils import ExtractorError, mimetype2ext, parse_iso8601, try_get
class FancodeVodIE(InfoExtractor):

2
yt_dlp/extractor/faz.py
View File

@ -3,9 +3,9 @@
from .common import InfoExtractor
from ..compat import compat_etree_fromstring
from ..utils import (
int_or_none,
xpath_element,
xpath_text,
int_or_none,
)

2
yt_dlp/extractor/fczenit.py
View File

@ -1,7 +1,7 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
float_or_none,
int_or_none,
)

1
yt_dlp/extractor/fifa.py
View File

@ -1,5 +1,4 @@
from .common import InfoExtractor
from ..utils import (
int_or_none,
traverse_obj,

4
yt_dlp/extractor/filmon.py
View File

@ -2,10 +2,10 @@
from ..compat import compat_str
from ..networking.exceptions import HTTPError
from ..utils import (
ExtractorError,
int_or_none,
qualities,
strip_or_none,
int_or_none,
ExtractorError,
)

2
yt_dlp/extractor/gab.py
View File

@ -7,7 +7,7 @@
parse_codecs,
parse_duration,
str_to_int,
unified_timestamp
unified_timestamp,
)

2
yt_dlp/extractor/gamejolt.py
View File

@ -10,7 +10,7 @@
int_or_none,
str_or_none,
traverse_obj,
try_get
try_get,
)

1
yt_dlp/extractor/gaskrank.py
View File

@ -1,4 +1,5 @@
import re
from .common import InfoExtractor
from ..utils import (
float_or_none,

107
yt_dlp/extractor/gbnews.py Normal file
View File

@ -0,0 +1,107 @@
import functools
from .common import InfoExtractor
from ..utils import (
ExtractorError,
extract_attributes,
get_elements_html_by_class,
url_or_none,
)
from ..utils.traversal import traverse_obj
class GBNewsIE(InfoExtractor):
IE_DESC = 'GB News clips, features and live streams'
_VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
_PLATFORM = 'safari'
_SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
_TESTS = [{
'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
'info_dict': {
'id': '52264136',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
'description': 'The post was criticised by former employers of the broadcaster',
'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
},
}, {
'url': 'https://www.gbnews.com/royal/prince-harry-in-love-with-kate-meghan-markle-jealous-royal',
'info_dict': {
'id': '52328390',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'prince-harry-in-love-with-kate-meghan-markle-jealous-royal',
'description': 'Ingrid Seward has published 17 books documenting the highs and lows of the Royal Family',
'title': 'Royal author claims Prince Harry was \'in love\' with Kate - Meghan was \'jealous\'',
}
}, {
'url': 'https://www.gbnews.uk/watchlive',
'info_dict': {
'id': '1069',
'ext': 'mp4',
'thumbnail': r're:https?://www\.gbnews\.\w+/.+\.(?:jpe?g|png|webp)',
'display_id': 'watchlive',
'live_status': 'is_live',
'title': r're:^GB News Live',
},
'params': {'skip_download': 'm3u8'},
}]
@functools.lru_cache
def _get_ss_endpoint(self, data_id, data_env):
if not data_id:
data_id = 'GB003'
if not data_env:
data_env = 'production'
json_data = self._download_json(
self._SSMP_URL, None, 'Downloading Simplestream JSON metadata', query={
'id': data_id,
'env': data_env,
})
meta_url = traverse_obj(json_data, ('response', 'api_hostname', {url_or_none}))
if not meta_url:
raise ExtractorError('No API host found')
return meta_url
def _real_extract(self, url):
display_id = self._match_id(url).rpartition('/')[2]
webpage = self._download_webpage(url, display_id)
video_data = None
elements = get_elements_html_by_class('simplestream', webpage)
for html_tag in elements:
attributes = extract_attributes(html_tag)
if 'sidebar' not in (attributes.get('class') or ''):
video_data = attributes
if not video_data:
raise ExtractorError('Could not find video element', expected=True)
endpoint_url = self._get_ss_endpoint(video_data.get('data-id'), video_data.get('data-env'))
uvid = video_data['data-uvid']
video_type = video_data.get('data-type')
if not video_type or video_type == 'vod':
video_type = 'show'
stream_data = self._download_json(
f'{endpoint_url}/api/{video_type}/stream/{uvid}',
uvid, 'Downloading stream JSON', query={
'key': video_data.get('data-key'),
'platform': self._PLATFORM,
})
if traverse_obj(stream_data, 'drm'):
self.report_drm(uvid)
return {
'id': uvid,
'display_id': display_id,
'title': self._og_search_title(webpage, default=None),
'description': self._og_search_description(webpage, default=None),
'formats': self._extract_m3u8_formats(traverse_obj(stream_data, (
'response', 'stream', {url_or_none})), uvid, 'mp4'),
'thumbnail': self._og_search_thumbnail(webpage, default=None),
'is_live': video_type == 'live',
}

2
yt_dlp/extractor/generic.py
View File

@ -4,7 +4,7 @@
import urllib.parse
import xml.etree.ElementTree
from .common import InfoExtractor # isort: split
from .common import InfoExtractor
from .commonprotocols import RtmpIE
from .youtube import YoutubeIE
from ..compat import compat_etree_fromstring

2
yt_dlp/extractor/gettr.py
View File

@ -1,7 +1,7 @@
from .common import InfoExtractor
from ..utils import (
bool_or_none,
ExtractorError,
bool_or_none,
dict_get,
float_or_none,
int_or_none,

1
yt_dlp/extractor/gigya.py
View File

@ -1,5 +1,4 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
urlencode_postdata,

2
yt_dlp/extractor/glomex.py
View File

@ -3,9 +3,9 @@
from .common import InfoExtractor
from ..utils import (
ExtractorError,
determine_ext,
extract_attributes,
ExtractorError,
int_or_none,
parse_qs,
smuggle_url,

16
yt_dlp/extractor/go.py
View File

@ -3,16 +3,16 @@
from .adobepass import AdobePassIE
from ..compat import compat_str
from ..utils import (
int_or_none,
determine_ext,
parse_age_limit,
remove_start,
remove_end,
try_get,
urlencode_postdata,
ExtractorError,
unified_timestamp,
determine_ext,
int_or_none,
parse_age_limit,
remove_end,
remove_start,
traverse_obj,
try_get,
unified_timestamp,
urlencode_postdata,
)

Some files were not shown because too many files have changed in this diff Show More