[cleanup] Misc cleanup

2024-12-29 06:35:50 +01:00 · 2022-01-04 01:07:24 +05:30 · 2022-01-04 01:07:24 +05:30 · 9e907ebddf
commit 9e907ebddf
parent 21df2117e4
8 changed files with 47 additions and 44 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@ -165,7 +165,7 @@ jobs:
    - name: Install Requirements
      run: |
          brew install coreutils
-          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.5.1 mutagen pycryptodomex websockets
+          /usr/bin/python3 -m pip install -U --user pip Pyinstaller==4.5.1 -r requirements.txt
    - name: Bump version
      id: bump_version
      run: /usr/bin/python3 devscripts/update-version.py
@ -208,7 +208,7 @@ jobs:
    - name: Get SHA2-256SUMS for yt-dlp_macos.zip
      id: sha256_macos_zip
      run: echo "::set-output name=sha256_macos_zip::$(sha256sum dist/yt-dlp_macos.zip | awk '{print $1}')"
-    - name: Get SHA2-512SUMS for yt-dlp_macos
+    - name: Get SHA2-512SUMS for yt-dlp_macos.zip
      id: sha512_macos_zip
      run: echo "::set-output name=sha512_macos_zip::$(sha512sum dist/yt-dlp_macos.zip | awk '{print $1}')"

@ -234,7 +234,7 @@ jobs:
      # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
      run: |
          python -m pip install --upgrade pip setuptools wheel py2exe
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-4.5.1-py3-none-any.whl" -r requirements.txt
    - name: Bump version
      id: bump_version
      env:
@ -321,7 +321,7 @@ jobs:
    - name: Install Requirements
      run: |
          python -m pip install --upgrade pip setuptools wheel
-          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" mutagen pycryptodomex websockets
+          pip install "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-4.5.1-py3-none-any.whl" -r requirements.txt
    - name: Bump version
      id: bump_version
      env:
--- a/5
+++ b/5
@ -1,5 +1,6 @@
 all: lazy-extractors yt-dlp doc pypi-files
-clean: clean-test clean-dist clean-cache
+clean: clean-test clean-dist
+clean-all: clean clean-cache
 completions: completion-bash completion-fish completion-zsh
 doc: README.md CONTRIBUTING.md issuetemplates supportedsites
 ot: offlinetest
@ -21,7 +22,7 @@ clean-dist:
 	rm -rf yt-dlp.1.temp.md yt-dlp.1 README.txt MANIFEST build/ dist/ .coverage cover/ yt-dlp.tar.gz completions/ \
 	yt_dlp/extractor/lazy_extractors.py *.spec CONTRIBUTING.md.tmp yt-dlp yt-dlp.exe yt_dlp.egg-info/ AUTHORS .mailmap
 clean-cache:
-	find . -name "*.pyc" -o -name "*.class" -delete
+	find . \( -name "*.pyc" -o -name "*.class" \) -delete

 completion-bash: completions/bash/yt-dlp
 completion-fish: completions/fish/yt-dlp.fish
--- a/README.md
+++ b/README.md
@ -112,7 +112,7 @@ # NEW FEATURES

 * **Other new options**: Many new options have been added such as `--print`, `--wait-for-video`, `--sleep-requests`, `--convert-thumbnails`, `--write-link`, `--force-download-archive`, `--force-overwrites`, `--break-on-reject` etc

-* **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio etc
+* **Improvements**: Regex and other operators in `--match-filter`, multiple `--postprocessor-args` and `--downloader-args`, faster archive checking, more [format selection options](#format-selection), merge multi-video/audio, multiple `--config-locations`, etc

 * **Plugins**: Extractors and PostProcessors can be loaded from an external file. See [plugins](#plugins) for details

@ -133,7 +133,7 @@ ### Differences in default behavior
 * `--ignore-errors` is enabled by default. Use `--abort-on-error` or `--compat-options abort-on-error` to abort on errors instead
 * When writing metadata files such as thumbnails, description or infojson, the same information (if available) is also written for playlists. Use `--no-write-playlist-metafiles` or `--compat-options no-playlist-metafiles` to not write these files
 * `--add-metadata` attaches the `infojson` to `mkv` files in addition to writing the metadata when used with `--write-info-json`. Use `--no-embed-info-json` or `--compat-options no-attach-info-json` to revert this
-* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](https://github.com/yt-dlp/yt-dlp#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
+* Some metadata are embedded into different fields when using `--add-metadata` as compared to youtube-dl. Most notably, `comment` field contains the `webpage_url` and `synopsis` contains the `description`. You can [use `--parse-metadata`](#modifying-metadata) to modify this to your liking or use `--compat-options embed-metadata` to revert this
 * `playlist_index` behaves differently when used with options like `--playlist-reverse` and `--playlist-items`. See [#302](https://github.com/yt-dlp/yt-dlp/issues/302) for details. You can use `--compat-options playlist-index` if you want to keep the earlier behavior
 * The output of `-F` is listed in a new format. Use `--compat-options list-formats` to revert this
 * All *experiences* of a funimation episode are considered as a single video. This behavior breaks existing archives. Use `--compat-options seperate-video-versions` to extract information from only the default player
@ -327,22 +327,26 @@ ## General Options:
                                     an error. The default value "fixup_error"
                                     repairs broken URLs, but emits an error if
                                     this is not possible instead of searching
-    --ignore-config, --no-config     Disable loading any configuration files
-                                     except the one provided by --config-location.
-                                     When given inside a configuration
-                                     file, no further configuration files are
-                                     loaded. Additionally, (for backward
-                                     compatibility) if this option is found
-                                     inside the system configuration file, the
-                                     user configuration is not loaded
-    --config-location PATH           Location of the main configuration file;
+    --ignore-config                  Don't load any more configuration files
+                                     except those given by --config-locations.
+                                     For backward compatibility, if this option
+                                     is found inside the system configuration
+                                     file, the user configuration is not loaded
+    --no-config-locations            Do not load any custom configuration files
+                                     (default). When given inside a
+                                     configuration file, ignore all previous
+                                     --config-locations defined in the current
+                                     file
+    --config-locations PATH          Location of the main configuration file;
                                     either the path to the config or its
-                                     containing directory
+                                     containing directory. Can be used multiple
+                                     times and inside other configuration files
    --flat-playlist                  Do not extract the videos of a playlist,
                                     only list them
    --no-flat-playlist               Extract the videos of a playlist
    --live-from-start                Download livestreams from the start.
                                     Currently only supported for YouTube
+                                     (Experimental)
    --no-live-from-start             Download livestreams from the current time
                                     (default)
    --wait-for-video MIN[-MAX]       Wait for scheduled streams to become
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@ -1154,6 +1154,7 @@ def _real_extract(self, url):
        self.assertTrue(entries[1] is None)
        self.assertEqual(len(ydl.downloaded_info_dicts), 1)
        downloaded = ydl.downloaded_info_dicts[0]
+        entries[2].pop('requested_downloads', None)
        self.assertEqual(entries[2], downloaded)
        self.assertEqual(downloaded['url'], TEST_URL)
        self.assertEqual(downloaded['title'], 'Video Transparent 2')
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@ -1161,7 +1161,7 @@ def create_key(outer_mobj):
            str_fmt = f'{fmt[:-1]}s'
            if fmt[-1] == 'l':  # list
                delim = '\n' if '#' in flags else ', '
-                value, fmt = delim.join(variadic(value, allowed_types=(str, bytes))), str_fmt
+                value, fmt = delim.join(map(str, variadic(value, allowed_types=(str, bytes)))), str_fmt
            elif fmt[-1] == 'j':  # json
                value, fmt = json.dumps(value, default=_dumpjson_default, indent=4 if '#' in flags else None), str_fmt
            elif fmt[-1] == 'q':  # quoted
@ -2396,6 +2396,9 @@ def sanitize_numeric_fields(info):
            if not get_from_start:
                info_dict['title'] += ' ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M')

+        # backward compatibility
+        info_dict['fulltitle'] = info_dict['title']
+
        if not formats:
            self.raise_no_formats(info_dict)

@ -2584,7 +2587,7 @@ def is_wellformed(f):
                if max_downloads_reached:
                    break

-            write_archive = set(f.get('_write_download_archive', False) for f in formats_to_download)
+            write_archive = set(f.get('__write_download_archive', False) for f in formats_to_download)
            assert write_archive.issubset({True, False, 'ignore'})
            if True in write_archive and False not in write_archive:
                self.record_download_archive(info_dict)
@ -2754,14 +2757,11 @@ def process_info(self, info_dict):
        assert info_dict.get('_type', 'video') == 'video'
        original_infodict = info_dict

-        # TODO: backward compatibility, to be removed
-        info_dict['fulltitle'] = info_dict['title']
-
        if 'format' not in info_dict and 'ext' in info_dict:
            info_dict['format'] = info_dict['ext']

        if self._match_entry(info_dict) is not None:
-            info_dict['_write_download_archive'] = 'ignore'
+            info_dict['__write_download_archive'] = 'ignore'
            return

        self.post_extract(info_dict)
@ -2776,7 +2776,7 @@ def process_info(self, info_dict):
        self.__forced_printings(info_dict, full_filename, incomplete=('format' not in info_dict))

        if self.params.get('simulate'):
-            info_dict['_write_download_archive'] = self.params.get('force_write_download_archive')
+            info_dict['__write_download_archive'] = self.params.get('force_write_download_archive')
            return

        if full_filename is None:
@ -2890,7 +2890,7 @@ def replace_info_dict(new_info):
            info_dict['__finaldir'] = os.path.dirname(os.path.abspath(encodeFilename(full_filename)))
            info_dict['__files_to_move'] = files_to_move
            replace_info_dict(self.run_pp(MoveFilesAfterDownloadPP(self, False), info_dict))
-            info_dict['_write_download_archive'] = self.params.get('force_write_download_archive')
+            info_dict['__write_download_archive'] = self.params.get('force_write_download_archive')
        else:
            # Download
            info_dict.setdefault('__postprocessors', [])
@ -3119,10 +3119,10 @@ def ffmpeg_fixup(cndn, msg, cls):
                except Exception as err:
                    self.report_error('post hooks: %s' % str(err))
                    return
-                info_dict['_write_download_archive'] = True
+                info_dict['__write_download_archive'] = True

        if self.params.get('force_write_download_archive'):
-            info_dict['_write_download_archive'] = True
+            info_dict['__write_download_archive'] = True

        # Make sure the info_dict was modified in-place
        assert info_dict is original_infodict
--- a/yt_dlp/extractor/zee5.py
+++ b/yt_dlp/extractor/zee5.py
@ -110,6 +110,8 @@ def _login(self):
                    raise ExtractorError(otp_request_json['message'], expected=True)
            elif username.lower() == 'token' and len(password) > 1198:
                self._USER_TOKEN = password
+            else:
+                raise ExtractorError(self._LOGIN_HINT, expected=True)

    def _real_initialize(self):
        self._login()
--- a/yt_dlp/options.py
+++ b/yt_dlp/options.py
@ -137,7 +137,7 @@ def _format_option_string(option):

    def _list_from_options_callback(option, opt_str, value, parser, append=True, delim=',', process=str.strip):
        # append can be True, False or -1 (prepend)
-        current = getattr(parser.values, option.dest) if append else []
+        current = list(getattr(parser.values, option.dest)) if append else []
        value = list(filter(None, [process(value)] if delim is None else map(process, value.split(delim))))
        setattr(
            parser.values, option.dest,
@ -146,7 +146,7 @@ def _list_from_options_callback(option, opt_str, value, parser, append=True, del
    def _set_from_options_callback(
            option, opt_str, value, parser, delim=',', allowed_values=None, aliases={},
            process=lambda x: x.lower().strip()):
-        current = getattr(parser.values, option.dest)
+        current = set(getattr(parser.values, option.dest))
        values = [process(value)] if delim is None else list(map(process, value.split(delim)[::-1]))
        while values:
            actual_val = val = values.pop()
@ -261,7 +261,7 @@ def _dict_from_options_callback(
        '--ignore-config', '--no-config',
        action='store_true', dest='ignoreconfig',
        help=(
-            'Disable loading any further configuration files except the one provided by --config-locations. '
+            'Don\'t load any more configuration files except those given by --config-locations. '
            'For backward compatibility, if this option is found inside the system configuration file, the user configuration is not loaded'))
    general.add_option(
        '--no-config-locations',
@ -286,7 +286,7 @@ def _dict_from_options_callback(
    general.add_option(
        '--live-from-start',
        action='store_true', dest='live_from_start',
-        help='Download livestreams from the start. Currently only supported for YouTube')
+        help='Download livestreams from the start. Currently only supported for YouTube (Experimental)')
    general.add_option(
        '--no-live-from-start',
        action='store_false', dest='live_from_start',
@ -811,7 +811,7 @@ def _dict_from_options_callback(
        metavar='NAME:ARGS', dest='external_downloader_args', default={}, type='str',
        action='callback', callback=_dict_from_options_callback,
        callback_kwargs={
-            'allowed_keys': r'ffmpeg_[io]\d*|%s' % '|'.join(list_external_downloaders()),
+            'allowed_keys': r'ffmpeg_[io]\d*|%s' % '|'.join(map(re.escape, list_external_downloaders())),
            'default_key': 'default',
            'process': compat_shlex_split
        }, help=(
@ -1050,7 +1050,7 @@ def _dict_from_options_callback(
        metavar='[TYPES:]PATH', dest='paths', default={}, type='str',
        action='callback', callback=_dict_from_options_callback,
        callback_kwargs={
-            'allowed_keys': 'home|temp|%s' % '|'.join(OUTTMPL_TYPES.keys()),
+            'allowed_keys': 'home|temp|%s' % '|'.join(map(re.escape, OUTTMPL_TYPES.keys())),
            'default_key': 'home'
        }, help=(
            'The paths where the files should be downloaded. '
@ -1065,7 +1065,7 @@ def _dict_from_options_callback(
        metavar='[TYPES:]TEMPLATE', dest='outtmpl', default={}, type='str',
        action='callback', callback=_dict_from_options_callback,
        callback_kwargs={
-            'allowed_keys': '|'.join(OUTTMPL_TYPES.keys()),
+            'allowed_keys': '|'.join(map(re.escape, OUTTMPL_TYPES.keys())),
            'default_key': 'default'
        }, help='Output filename template; see "OUTPUT TEMPLATE" for details')
    filesystem.add_option(
@ -1302,7 +1302,8 @@ def _dict_from_options_callback(
        metavar='NAME:ARGS', dest='postprocessor_args', default={}, type='str',
        action='callback', callback=_dict_from_options_callback,
        callback_kwargs={
-            'allowed_keys': r'\w+(?:\+\w+)?', 'default_key': 'default-compat',
+            'allowed_keys': r'\w+(?:\+\w+)?',
+            'default_key': 'default-compat',
            'process': compat_shlex_split,
            'multiple_keys': False
        }, help=(
--- a/yt_dlp/utils.py
+++ b/yt_dlp/utils.py
@ -2381,13 +2381,8 @@ def get_method(self):


 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
-    if get_attr:
-        if v is not None:
-            v = getattr(v, get_attr, None)
-    if v == '':
-        v = None
-    if v is None:
-        return default
+    if get_attr and v is not None:
+        v = getattr(v, get_attr, None)
    try:
        return int(v) * invscale // scale
    except (ValueError, TypeError, OverflowError):
@ -5036,7 +5031,6 @@ def _traverse_obj(obj, path, _current_depth=0):
    return default


-# Deprecated
 def traverse_dict(dictn, keys, casesense=True):
    write_string('DeprecationWarning: yt_dlp.utils.traverse_dict is deprecated '
                 'and may be removed in a future version. Use yt_dlp.utils.traverse_obj instead')