Coverage report: - 55% -
-
-
-
-
-
-
- 
- Hot-keys on this page
-- n - s - m - x - - c change column sorting -
-| Module | -statements | -missing | -excluded | - -coverage | -
|---|---|---|---|---|
| Total | -3474 | -1568 | -0 | - -55% | -
| youtube_dl | -330 | -303 | -0 | - -8% | -
| youtube_dl.FileDownloader | -477 | -209 | -0 | - -56% | -
| youtube_dl.InfoExtractors | -2213 | -923 | -0 | - -58% | -
| youtube_dl.PostProcessor | -130 | -111 | -0 | - -15% | -
| youtube_dl.utils | -323 | -22 | -0 | - -93% | -
| youtube_dl.version | -1 | -0 | -0 | - -100% | -
| t |
| - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | -
- #!/usr/bin/env python -# -*- coding: utf-8 -*- -- - - - - 'Ricardo Garcia Gonzalez', -'Danny Colligan', -'Benjamin Johnson', -'Vasyl\' Vavrychuk', -'Witold Baryluk', -'Paweł Paprota', -'Gergely Imreh', -'Rogério Brito', -'Philipp Hagemeister', -'Sören Schulze', -'Kevin Ngo', -'Ori Avtalion', -'shizeeg', -'Filippo Valsorda', -'Christian Albrecht', -'Dave Vasilevsky', -) -- - - - - - - - - - - - - - - - - - - - """Update the program file with the latest version from the repository""" -- # TODO: at least, check https certificates -- from zipimport import zipimporter -- API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" -BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" -EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" -- if hasattr(sys, "frozen"): # PY2EXE -if not os.access(filename, os.W_OK): -sys.exit('ERROR: no write permissions on %s' % filename) -- downloader.to_screen(u'Updating to latest version...') -- urla = compat_urllib_request.urlopen(API_URL) -download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read())) -if not download: -downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') -return -newversion = download[0]["description"].strip() -if newversion == __version__: -downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') -return -urla.close() -- exe = os.path.abspath(filename) -directory = os.path.dirname(exe) -if not os.access(directory, os.W_OK): -sys.exit('ERROR: no write permissions on %s' % directory) -- try: -urlh = compat_urllib_request.urlopen(EXE_URL) -newcontent = urlh.read() -urlh.close() -with open(exe + '.new', 'wb') as outf: -outf.write(newcontent) -except (IOError, OSError) as err: -sys.exit('ERROR: unable to download latest version') -- try: -bat = os.path.join(directory, 'youtube-dl-updater.bat') -b = open(bat, 'w') -b.write(""" -echo Updating youtube-dl... -ping 127.0.0.1 -n 5 -w 1000 > NUL -move /Y "%s.new" "%s" -del "%s" -\n""" %(exe, exe, bat)) -b.close() -- os.startfile(bat) -except (IOError, OSError) as err: -sys.exit('ERROR: unable to overwrite current version') -- elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP -if not os.access(filename, os.W_OK): -sys.exit('ERROR: no write permissions on %s' % filename) -- downloader.to_screen(u'Updating to latest version...') -- urla = compat_urllib_request.urlopen(API_URL) -download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"] -if not download: -downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') -return -newversion = download[0]["description"].strip() -if newversion == __version__: -downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') -return -urla.close() -- try: -urlh = compat_urllib_request.urlopen(BIN_URL) -newcontent = urlh.read() -urlh.close() -except (IOError, OSError) as err: -sys.exit('ERROR: unable to download latest version') -- try: -with open(filename, 'wb') as outf: -outf.write(newcontent) -except (IOError, OSError) as err: -sys.exit('ERROR: unable to overwrite current version') -- else: -downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.') -return -- downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') -- - def _readOptions(filename_bytes): -try: -optionf = open(filename_bytes) -except IOError: -return [] # silently skip if file is not present -try: -res = [] -for l in optionf: -res += shlex.split(l, comments=True) -finally: -optionf.close() -return res -- def _format_option_string(option): -''' ('-o', '--option') -> -o, --format METAVAR''' -- opts = [] -- if option._short_opts: -opts.append(option._short_opts[0]) -if option._long_opts: -opts.append(option._long_opts[0]) -if len(opts) > 1: -opts.insert(1, ', ') -- if option.takes_value(): opts.append(' %s' % option.metavar) -- return "".join(opts) -- def _find_term_columns(): -columns = os.environ.get('COLUMNS', None) -if columns: -return int(columns) -- try: -sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) -out,err = sp.communicate() -return int(out.split()[1]) -except: -pass -return None -- max_width = 80 -max_help_position = 80 -- # No need to wrap help messages if we're on a wide console -columns = _find_term_columns() -if columns: max_width = columns -- fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position) -fmt.format_option_strings = _format_option_string -- kw = { -'version' : __version__, -'formatter' : fmt, -'usage' : '%prog [options] url [url...]', -'conflict_handler' : 'resolve', -} -- parser = optparse.OptionParser(**kw) -- # option groups -general = optparse.OptionGroup(parser, 'General Options') -selection = optparse.OptionGroup(parser, 'Video Selection') -authentication = optparse.OptionGroup(parser, 'Authentication Options') -video_format = optparse.OptionGroup(parser, 'Video Format Options') -postproc = optparse.OptionGroup(parser, 'Post-processing Options') -filesystem = optparse.OptionGroup(parser, 'Filesystem Options') -verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options') -- general.add_option('-h', '--help', -action='help', help='print this help text and exit') -general.add_option('-v', '--version', -action='version', help='print program version and exit') -general.add_option('-U', '--update', -action='store_true', dest='update_self', help='update this program to latest version') -general.add_option('-i', '--ignore-errors', -action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) -general.add_option('-r', '--rate-limit', -dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)') -general.add_option('-R', '--retries', -dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10) -general.add_option('--buffer-size', -dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024") -general.add_option('--no-resize-buffer', -action='store_true', dest='noresizebuffer', -help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False) -general.add_option('--dump-user-agent', -action='store_true', dest='dump_user_agent', -help='display the current browser identification', default=False) -general.add_option('--user-agent', -dest='user_agent', help='specify a custom user agent', metavar='UA') -general.add_option('--list-extractors', -action='store_true', dest='list_extractors', -help='List all supported extractors and the URLs they would handle', default=False) -general.add_option('--test', action='store_true', dest='test', default=False, help=optparse.SUPPRESS_HELP) -- selection.add_option('--playlist-start', -dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) -selection.add_option('--playlist-end', -dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) -selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') -selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') -selection.add_option('--max-downloads', metavar='NUMBER', dest='max_downloads', help='Abort after downloading NUMBER files', default=None) -- authentication.add_option('-u', '--username', -dest='username', metavar='USERNAME', help='account username') -authentication.add_option('-p', '--password', -dest='password', metavar='PASSWORD', help='account password') -authentication.add_option('-n', '--netrc', -action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False) -- - video_format.add_option('-f', '--format', -action='store', dest='format', metavar='FORMAT', help='video format code') -video_format.add_option('--all-formats', -action='store_const', dest='format', help='download all available video formats', const='all') -video_format.add_option('--prefer-free-formats', -action='store_true', dest='prefer_free_formats', default=False, help='prefer free video formats unless a specific one is requested') -video_format.add_option('--max-quality', -action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download') -video_format.add_option('-F', '--list-formats', -action='store_true', dest='listformats', help='list all available formats (currently youtube only)') -video_format.add_option('--write-srt', -action='store_true', dest='writesubtitles', -help='write video closed captions to a .srt file (currently youtube only)', default=False) -video_format.add_option('--srt-lang', -action='store', dest='subtitleslang', metavar='LANG', -help='language of the closed captions to download (optional) use IETF language tags like \'en\'') -- - verbosity.add_option('-q', '--quiet', -action='store_true', dest='quiet', help='activates quiet mode', default=False) -verbosity.add_option('-s', '--simulate', -action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False) -verbosity.add_option('--skip-download', -action='store_true', dest='skip_download', help='do not download the video', default=False) -verbosity.add_option('-g', '--get-url', -action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False) -verbosity.add_option('-e', '--get-title', -action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) -verbosity.add_option('--get-thumbnail', -action='store_true', dest='getthumbnail', -help='simulate, quiet but print thumbnail URL', default=False) -verbosity.add_option('--get-description', -action='store_true', dest='getdescription', -help='simulate, quiet but print video description', default=False) -verbosity.add_option('--get-filename', -action='store_true', dest='getfilename', -help='simulate, quiet but print output filename', default=False) -verbosity.add_option('--get-format', -action='store_true', dest='getformat', -help='simulate, quiet but print output format', default=False) -verbosity.add_option('--no-progress', -action='store_true', dest='noprogress', help='do not print progress bar', default=False) -verbosity.add_option('--console-title', -action='store_true', dest='consoletitle', -help='display progress in console titlebar', default=False) -verbosity.add_option('-v', '--verbose', -action='store_true', dest='verbose', help='print various debugging information', default=False) -- - filesystem.add_option('-t', '--title', -action='store_true', dest='usetitle', help='use title in file name', default=False) -filesystem.add_option('--id', -action='store_true', dest='useid', help='use video ID in file name', default=False) -filesystem.add_option('-l', '--literal', -action='store_true', dest='usetitle', help='[deprecated] alias of --title', default=False) -filesystem.add_option('-A', '--auto-number', -action='store_true', dest='autonumber', -help='number downloaded files starting from 00000', default=False) -filesystem.add_option('-o', '--output', -dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .') -filesystem.add_option('--restrict-filenames', -action='store_true', dest='restrictfilenames', -help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False) -filesystem.add_option('-a', '--batch-file', -dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)') -filesystem.add_option('-w', '--no-overwrites', -action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) -filesystem.add_option('-c', '--continue', -action='store_true', dest='continue_dl', help='resume partially downloaded files', default=True) -filesystem.add_option('--no-continue', -action='store_false', dest='continue_dl', -help='do not resume partially downloaded files (restart from beginning)') -filesystem.add_option('--cookies', -dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in') -filesystem.add_option('--no-part', -action='store_true', dest='nopart', help='do not use .part files', default=False) -filesystem.add_option('--no-mtime', -action='store_false', dest='updatetime', -help='do not use the Last-modified header to set the file modification time', default=True) -filesystem.add_option('--write-description', -action='store_true', dest='writedescription', -help='write video description to a .description file', default=False) -filesystem.add_option('--write-info-json', -action='store_true', dest='writeinfojson', -help='write video metadata to a .info.json file', default=False) -- - postproc.add_option('-x', '--extract-audio', action='store_true', dest='extractaudio', default=False, -help='convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)') -postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best', -help='"best", "aac", "vorbis", "mp3", "m4a", or "wav"; best by default') -postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='5', -help='ffmpeg/avconv audio quality specification, insert a value between 0 (better) and 9 (worse) for VBR or a specific bitrate like 128K (default 5)') -postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False, -help='keeps the video file on disk after the post-processing; the video is erased by default') -- - parser.add_option_group(general) -parser.add_option_group(selection) -parser.add_option_group(filesystem) -parser.add_option_group(verbosity) -parser.add_option_group(video_format) -parser.add_option_group(authentication) -parser.add_option_group(postproc) -- xdg_config_home = os.environ.get('XDG_CONFIG_HOME') -if xdg_config_home: -userConf = os.path.join(xdg_config_home, 'youtube-dl.conf') -else: -userConf = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf') -argv = _readOptions('/etc/youtube-dl.conf') + _readOptions(userConf) + sys.argv[1:] -opts, args = parser.parse_args(argv) -- return parser, opts, args -- - """ Return a list of an instance of every supported extractor. -The order does matter; the first extractor matched is the one handling the URL. -""" - -YoutubePlaylistIE(), -YoutubeChannelIE(), -YoutubeUserIE(), -YoutubeSearchIE(), -YoutubeIE(), -MetacafeIE(), -DailymotionIE(), -GoogleSearchIE(), -PhotobucketIE(), -YahooIE(), -YahooSearchIE(), -DepositFilesIE(), -FacebookIE(), -BlipTVUserIE(), -BlipTVIE(), -VimeoIE(), -MyVideoIE(), -ComedyCentralIE(), -EscapistIE(), -CollegeHumorIE(), -XVideosIE(), -SoundcloudIE(), -InfoQIE(), -MixcloudIE(), -StanfordOpenClassroomIE(), -MTVIE(), -YoukuIE(), -XNXXIE(), -GooglePlusIE(), -ArteTvIE(), -NBAIE(), -JustinTVIE(), -GenericIE() -] -- - - - # Open appropriate CookieJar -if opts.cookiefile is None: -jar = compat_cookiejar.CookieJar() -else: -try: -jar = compat_cookiejar.MozillaCookieJar(opts.cookiefile) -if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK): -jar.load() -except (IOError, OSError) as err: -sys.exit(u'ERROR: unable to open cookie file') -# Set user agent -if opts.user_agent is not None: -std_headers['User-Agent'] = opts.user_agent -- # Dump user agent -if opts.dump_user_agent: -print(std_headers['User-Agent']) -sys.exit(0) -- # Batch file verification -batchurls = [] -if opts.batchfile is not None: -try: -if opts.batchfile == '-': -batchfd = sys.stdin -else: -batchfd = open(opts.batchfile, 'r') -batchurls = batchfd.readlines() -batchurls = [x.strip() for x in batchurls] -batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)] -except IOError: -sys.exit(u'ERROR: batch file could not be read') -all_urls = batchurls + args -all_urls = [url.strip() for url in all_urls] -- # General configuration -cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar) -proxy_handler = compat_urllib_request.ProxyHandler() -opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler()) -compat_urllib_request.install_opener(opener) -socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words) -- extractors = gen_extractors() -- if opts.list_extractors: -for ie in extractors: -print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) -matchedUrls = filter(lambda url: ie.suitable(url), all_urls) -all_urls = filter(lambda url: url not in matchedUrls, all_urls) -for mu in matchedUrls: -print(u' ' + mu) -sys.exit(0) -- # Conflicting, missing and erroneous options -if opts.usenetrc and (opts.username is not None or opts.password is not None): -parser.error(u'using .netrc conflicts with giving username/password') -if opts.password is not None and opts.username is None: -parser.error(u'account username missing') -if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): -parser.error(u'using output template conflicts with using title, video ID or auto number') -if opts.usetitle and opts.useid: -parser.error(u'using title conflicts with using video ID') -if opts.username is not None and opts.password is None: -opts.password = getpass.getpass(u'Type account password and press return:') -if opts.ratelimit is not None: -numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) -if numeric_limit is None: -parser.error(u'invalid rate limit specified') -opts.ratelimit = numeric_limit -if opts.retries is not None: -try: -opts.retries = int(opts.retries) -except (TypeError, ValueError) as err: -parser.error(u'invalid retry count specified') -if opts.buffersize is not None: -numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) -if numeric_buffersize is None: -parser.error(u'invalid buffer size specified') -opts.buffersize = numeric_buffersize -try: -opts.playliststart = int(opts.playliststart) -if opts.playliststart <= 0: -raise ValueError(u'Playlist start must be positive') -except (TypeError, ValueError) as err: -parser.error(u'invalid playlist start number specified') -try: -opts.playlistend = int(opts.playlistend) -if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): -raise ValueError(u'Playlist end must be greater than playlist start') -except (TypeError, ValueError) as err: -parser.error(u'invalid playlist end number specified') -if opts.extractaudio: -if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis', 'm4a', 'wav']: -parser.error(u'invalid audio format specified') -if opts.audioquality: -opts.audioquality = opts.audioquality.strip('k').strip('K') -if not opts.audioquality.isdigit(): -parser.error(u'invalid audio quality specified') -- if sys.version_info < (3,): -# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) -if opts.outtmpl is not None: -opts.outtmpl = opts.outtmpl.decode(preferredencoding()) -outtmpl =((opts.outtmpl is not None and opts.outtmpl) -or (opts.format == '-1' and opts.usetitle and u'%(title)s-%(id)s-%(format)s.%(ext)s') -or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s') -or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s') -or (opts.usetitle and u'%(title)s-%(id)s.%(ext)s') -or (opts.useid and u'%(id)s.%(ext)s') -or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') -or u'%(id)s.%(ext)s') -# File downloader -fd = FileDownloader({ -'usenetrc': opts.usenetrc, -'username': opts.username, -'password': opts.password, -'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), -'forceurl': opts.geturl, -'forcetitle': opts.gettitle, -'forcethumbnail': opts.getthumbnail, -'forcedescription': opts.getdescription, -'forcefilename': opts.getfilename, -'forceformat': opts.getformat, -'simulate': opts.simulate, -'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat), -'format': opts.format, -'format_limit': opts.format_limit, -'listformats': opts.listformats, -'outtmpl': outtmpl, -'restrictfilenames': opts.restrictfilenames, -'ignoreerrors': opts.ignoreerrors, -'ratelimit': opts.ratelimit, -'nooverwrites': opts.nooverwrites, -'retries': opts.retries, -'buffersize': opts.buffersize, -'noresizebuffer': opts.noresizebuffer, -'continuedl': opts.continue_dl, -'noprogress': opts.noprogress, -'playliststart': opts.playliststart, -'playlistend': opts.playlistend, -'logtostderr': opts.outtmpl == '-', -'consoletitle': opts.consoletitle, -'nopart': opts.nopart, -'updatetime': opts.updatetime, -'writedescription': opts.writedescription, -'writeinfojson': opts.writeinfojson, -'writesubtitles': opts.writesubtitles, -'subtitleslang': opts.subtitleslang, -'matchtitle': opts.matchtitle, -'rejecttitle': opts.rejecttitle, -'max_downloads': opts.max_downloads, -'prefer_free_formats': opts.prefer_free_formats, -'verbose': opts.verbose, -'test': opts.test, -}) -- if opts.verbose: -fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) -- for extractor in extractors: -fd.add_info_extractor(extractor) -- # PostProcessors -if opts.extractaudio: -fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo)) -- # Update version -if opts.update_self: -updateSelf(fd, sys.argv[0]) -- # Maybe do nothing -if len(all_urls) < 1: -if not opts.update_self: -parser.error(u'you must provide at least one URL') -else: -sys.exit() -- try: -retcode = fd.download(all_urls) -except MaxDownloadsReached: -fd.to_screen(u'--max-download limit reached, aborting.') -retcode = 101 -- # Dump cookie jar if requested -if opts.cookiefile is not None: - -jar.save() -except (IOError, OSError) as err: -sys.exit(u'ERROR: unable to save cookie jar') -- sys.exit(retcode) -- - - _real_main() -except DownloadError: -sys.exit(1) -except SameFileError: -sys.exit(u'ERROR: fixed output name but more than one file to download') -except KeyboardInterrupt: -sys.exit(u'\nERROR: Interrupted by user') - - |
-
-
-Hot-keys on this page
-- r - m - x - p toggle line displays -
-- j - k next/prev highlighted chunk -
-- 0 (zero) top of page -
-- 1 (one) first highlighted chunk -
-| - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | -
- #!/usr/bin/env python -# -*- coding: utf-8 -*- -- - - - - - - - - - - - - import ctypes -- - - - - """File Downloader class. -- File downloader objects are the ones responsible of downloading the -actual video file and writing it to disk if the user has requested -it, among some other tasks. In most cases there should be one per -program. As, given a video URL, the downloader doesn't know how to -extract all the needed information, task that InfoExtractors do, it -has to pass the URL to one of them. -- For this, file downloader objects have a method that allows -InfoExtractors to be registered in a given order. When it is passed -a URL, the file downloader handles it to the first InfoExtractor it -finds that reports being able to handle it. The InfoExtractor extracts -all the information about the video or videos the URL refers to, and -asks the FileDownloader to process the video information, possibly -downloading the video. -- File downloaders accept a lot of parameters. In order not to saturate -the object constructor with arguments, it receives a dictionary of -options instead. These options are available through the params -attribute for the InfoExtractors to use. The FileDownloader also -registers itself as the downloader in charge for the InfoExtractors -that are added to it, so this is a "mutual registration". -- Available options: -- username: Username for authentication purposes. -password: Password for authentication purposes. -usenetrc: Use netrc for authentication instead. -quiet: Do not print messages to stdout. -forceurl: Force printing final URL. -forcetitle: Force printing title. -forcethumbnail: Force printing thumbnail URL. -forcedescription: Force printing description. -forcefilename: Force printing final filename. -simulate: Do not download the video files. -format: Video format code. -format_limit: Highest quality format to try. -outtmpl: Template for output names. -restrictfilenames: Do not allow "&" and spaces in file names -ignoreerrors: Do not stop on download errors. -ratelimit: Download speed limit, in bytes/sec. -nooverwrites: Prevent overwriting files. -retries: Number of times to retry for HTTP error 5xx -buffersize: Size of download buffer in bytes. -noresizebuffer: Do not automatically resize the download buffer. -continuedl: Try to continue downloads if possible. -noprogress: Do not print the progress bar. -playliststart: Playlist item to start at. -playlistend: Playlist item to end at. -matchtitle: Download only matching titles. -rejecttitle: Reject downloads for matching titles. -logtostderr: Log messages to stderr instead of stdout. -consoletitle: Display progress in console window's titlebar. -nopart: Do not use temporary .part files. -updatetime: Use the Last-modified header to set output file timestamps. -writedescription: Write the video description to a .description file -writeinfojson: Write the video description to a .info.json file -writesubtitles: Write the video subtitles to a .srt file -subtitleslang: Language of the subtitles to download -test: Download only first bytes to test the downloader. -""" -- - - - - - - - - """Create a FileDownloader object with the given options.""" - - - - - - -- - self.to_stderr(u'WARNING: %(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') -- - def format_bytes(bytes): - -return 'N/A' - -bytes = float(bytes) - -exponent = 0 -else: - - - - -- - def calc_percent(byte_counter, data_len): - -return '---.-%' - -- - def calc_eta(start, now, total, current): - -return '--:--' - - - - - - - -return '--:--' - -- - def calc_speed(start, now, bytes): - - - - -- - def best_block_size(elapsed_time, bytes): - - - - - - - - - - -- - def parse_bytes(bytestr): -"""Parse a string indicating a byte quantity into an integer.""" -matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr) -if matchobj is None: -return None -number = float(matchobj.group(1)) -multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) -return int(round(number * multiplier)) -- - """Add an InfoExtractor object to the end of the list.""" - - -- - """Add a PostProcessor object to the end of the chain.""" -self._pps.append(pp) -pp.set_downloader(self) -- - """Print message to stdout if not in quiet mode.""" - - - - - - - - -- - """Print message to stderr.""" -assert type(message) == type(u'') -output = message + u'\n' -if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr -output = output.encode(preferredencoding()) -sys.stderr.write(output) -- - """Set console/terminal window title to message.""" - - -if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow(): -# c_wchar_p() might not be necessary if `message` is -# already of type unicode() -ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) -elif 'TERM' in os.environ: -sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding())) -- - """Checks if the output template is fixed.""" -return (re.search(u'(?u)%\\(.+?\\)s', self.params['outtmpl']) is None) -- - """Determine action to take when a download problem appears. -- Depending on if the downloader has been configured to ignore -download errors or not, this method may throw an exception or -not when errors are found, after printing the message. -""" -if message is not None: -self.to_stderr(message) -if self.params.get('verbose'): -self.to_stderr(u''.join(traceback.format_list(traceback.extract_stack()))) -if not self.params.get('ignoreerrors', False): -raise DownloadError(message) -self._download_retcode = 1 -- - """Sleep if the download speed is over the rate limit.""" - - - -now = time.time() -elapsed = now - start_time -if elapsed <= 0.0: -return -speed = float(byte_counter) / elapsed -if speed > rate_limit: -time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) -- - """Returns a temporary filename for the given filename.""" - -(os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): -return filename - -- - - - return filename -- - - - return - -except (IOError, OSError) as err: -self.trouble(u'ERROR: unable to rename file') -- - """Try to set the last-modified time of the given file.""" - -return - -return - - -return - - -return filetime - - -except: -pass - -- - """ Report that the description file is being written """ -self.to_screen(u'[info] Writing video description to: ' + descfn) -- - """ Report that the subtitles file is being written """ - -- - """ Report that the metadata file has been written """ -self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) -- - """Report destination filename.""" - -- - """Report download progress.""" - -return - -(percent_str, data_len_str, speed_str, eta_str), skip_eol=True) - -(percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip())) -- - """Report attempt to resume at given byte.""" -self.to_screen(u'[download] Resuming download at byte %s' % resume_len) -- - """Report retry in case of HTTP error 5xx""" -self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) -- - """Report file has already been fully downloaded.""" - - -except (UnicodeEncodeError) as err: -self.to_screen(u'[download] The file has already been downloaded') -- - """Report it was impossible to resume download.""" -self.to_screen(u'[download] Unable to resume') -- - """Report download finished.""" - -self.to_screen(u'[download] Download completed') -else: - -- - """Increment the ordinal that assigns a number to each file.""" - -- - """Generate the output filename.""" - - -- - - - - u'NA' if v is None else compat_str(v), -restricted=self.params.get('restrictfilenames'), -is_id=(k==u'id')) - -- - - except (ValueError, KeyError) as err: -self.trouble(u'ERROR: invalid system charset or erroneous output template') -return None -- - """ Returns None iff the file should be downloaded """ -- - - - matchtitle = matchtitle.decode('utf8') -if not re.search(matchtitle, title, re.IGNORECASE): -return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' - - -rejecttitle = rejecttitle.decode('utf8') -if re.search(rejecttitle, title, re.IGNORECASE): -return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' - -- - """Process a single dictionary returned by an InfoExtractor.""" -- # Keep for backwards compatibility - -- - - - - - self.to_screen(u'[download] ' + reason) -return -- - - if self._num_downloads > int(max_downloads): -raise MaxDownloadsReached() -- - - # Forced printings - -compat_print(info_dict['title']) - -compat_print(info_dict['url']) - -compat_print(info_dict['thumbnail']) - -compat_print(info_dict['description']) - -compat_print(filename) - -compat_print(info_dict['format']) -- # Do nothing else if in simulate mode - -return -- - return -- - - - os.makedirs(dn) -except (OSError, IOError) as err: -self.trouble(u'ERROR: unable to create directory ' + compat_str(err)) -return -- - try: -descfn = filename + u'.description' -self.report_writedescription(descfn) -descfile = open(encodeFilename(descfn), 'wb') -try: -descfile.write(info_dict['description'].encode('utf-8')) -finally: -descfile.close() -except (OSError, IOError): -self.trouble(u'ERROR: Cannot write description file ' + descfn) -return -- - # subtitles download errors are already managed as troubles in relevant IE -# that way it will silently go on when used with unsupporting IE - - - - - - -finally: - -except (OSError, IOError): -self.trouble(u'ERROR: Cannot write subtitles file ' + descfn) -return -- - infofn = filename + u'.info.json' -self.report_writeinfojson(infofn) -try: -json.dump -except (NameError,AttributeError): -self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.') -return -try: -infof = open(encodeFilename(infofn), 'w') -try: -json_info_dict = dict((k, info_dict[k]) for k in info_dict if not k in ['urlhandle']) -json.dump(json_info_dict, infof) -finally: -infof.close() -except (OSError, IOError): -self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn) -return -- - - success = True -else: - - -except (OSError, IOError) as err: -raise UnavailableVideoError() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self.trouble(u'ERROR: unable to download video data: %s' % str(err)) -return -except (ContentTooShortError, ) as err: -self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) -return -- - - - except (PostProcessingError) as err: -self.trouble(u'ERROR: postprocessing: %s' % str(err)) -return -- - """Download a given list of URLs.""" - -raise SameFileError(self.params['outtmpl']) -- - - - # Go to next InfoExtractor if not suitable - - -- # Warn if the _WORKING attribute is False - -self.trouble(u'WARNING: the program functionality for this site has been marked as broken, ' -u'and will probably not work. If you want to go on, use the -i option.') -- # Suitable InfoExtractor found - -- # Extract information from URL and process it - - - - - - -except UnavailableVideoError: -self.trouble(u'\nERROR: unable to download video') -- # Suitable InfoExtractor had been found; go to next URL - -- - self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) -- - - - """Run the postprocessing chain on the given file.""" - - - -info = pp.run(info) -if info is None: -break -- - self.report_destination(filename) -tmpfilename = self.temp_name(filename) -- # Check for rtmpdump first -try: -subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT) -except (OSError, IOError): -self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run') -return False -- # Download using rtmpdump. rtmpdump returns exit code 2 when -# the connection was interrumpted and resuming appears to be -# possible. This is part of rtmpdump's normal usage, AFAIK. -basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename] -args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)] -if self.params.get('verbose', False): -try: -import pipes -shell_quote = lambda args: ' '.join(map(pipes.quote, args)) -except ImportError: -shell_quote = repr -self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(args)) -retval = subprocess.call(args) -while retval == 2 or retval == 1: -prevsize = os.path.getsize(encodeFilename(tmpfilename)) -self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True) -time.sleep(5.0) # This seems to be needed -retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) -cursize = os.path.getsize(encodeFilename(tmpfilename)) -if prevsize == cursize and retval == 1: -break -# Some rtmp streams seem abort after ~ 99.8%. Don't complain for those -if prevsize == cursize and retval == 2 and cursize > 1024: -self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.') -retval = 0 -break -if retval == 0: -self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(encodeFilename(tmpfilename))) -self.try_rename(tmpfilename, filename) -return True -else: -self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval) -return False -- - - - - # Check file already present - - - -- # Attempt to download using rtmpdump - -return self._download_with_rtmpdump(filename, url, player_url) -- - - - # Do not include the Accept-Encoding header - - - -- - - - # Establish possible resume length - -resume_len = os.path.getsize(encodeFilename(tmpfilename)) -else: - -- - - if self.params.get('continuedl', False): -self.report_resuming_byte(resume_len) -request.add_header('Range','bytes=%d-' % resume_len) -open_mode = 'ab' -else: -resume_len = 0 -- - - - # Establish connection - - -data = info_dict['urlhandle'] - - -except (compat_urllib_error.HTTPError, ) as err: -if (err.code < 500 or err.code >= 600) and err.code != 416: -# Unexpected HTTP error -raise -elif err.code == 416: -# Unable to resume (requested range not satisfiable) -try: -# Open the connection again without the range header -data = compat_urllib_request.urlopen(basic_request) -content_length = data.info()['Content-Length'] -except (compat_urllib_error.HTTPError, ) as err: -if err.code < 500 or err.code >= 600: -raise -else: -# Examine the reported length -if (content_length is not None and -(resume_len - 100 < int(content_length) < resume_len + 100)): -# The file had already been fully downloaded. -# Explanation to the above condition: in issue #175 it was revealed that -# YouTube sometimes adds or removes a few bytes from the end of the file, -# changing the file size slightly and causing problems for some users. So -# I decided to implement a suggested change and consider the file -# completely downloaded if the file size differs less than 100 bytes from -# the one in the hard drive. -self.report_file_already_downloaded(filename) -self.try_rename(tmpfilename, filename) -return True -else: -# The length does not match, we start the download over -self.report_unable_to_resume() -open_mode = 'wb' -break -# Retry -count += 1 -if count <= retries: -self.report_retry(count, retries) -- - self.trouble(u'ERROR: giving up after %s retries' % retries) -return False -- - - - - - - - - # Download and write - - - - - - -- # Open file just in time - - - - - - -except (OSError, IOError) as err: -self.trouble(u'ERROR: unable to open for writing: %s' % str(err)) -return False - - -except (IOError, OSError) as err: -self.trouble(u'\nERROR: unable to write data: %s' % str(err)) -return False - - -- # Progress message - - -self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA') -else: - - - -- # Apply rate limit - -- - self.trouble(u'\nERROR: Did not get any data blocks') -return False - - - -raise ContentTooShortError(byte_counter, int(data_len)) - -- # Update file modification time - - -- - - |
-
-
-Hot-keys on this page
-- r - m - x - p toggle line displays -
-- j - k next/prev highlighted chunk -
-- 0 (zero) top of page -
-- 1 (one) first highlighted chunk -
-| - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | -
- #!/usr/bin/env python -# -*- coding: utf-8 -*- -- - - - - - - - - - - - - - - - - - """Information Extractor class. -- Information extractors are the classes that, given a URL, extract -information about the video (or videos) the URL refers to. This -information includes the real video URL, the video title, author and -others. The information is stored in a dictionary which is then -passed to the FileDownloader. The FileDownloader processes this -information possibly downloading the video to the file system, among -other possible outcomes. -- The dictionaries must include the following fields: -- id: Video identifier. -url: Final video URL. -uploader: Nickname of the video uploader, unescaped. -upload_date: Video upload date (YYYYMMDD). -title: Video title, unescaped. -ext: Video filename extension. -- The following fields are optional: -- format: The video format, defaults to ext (used for --get-format) -thumbnail: Full URL to a video thumbnail image. -description: One-line video description. -player_url: SWF Player URL (used for rtmpdump). -subtitles: The .srt file contents. -urlhandle: [internal] The urlHandle to be used to download the file, -like returned by urllib.request.urlopen -- The fields should all be Unicode strings. -- Subclasses of this one should re-define the _real_initialize() and -_real_extract() methods and define a _VALID_URL regexp. -Probably, they should also be added to the list of extractors. -- _real_extract() must return a *list* of information dictionaries as -described above. -- Finally, the _WORKING attribute should be set to False for broken IEs -in order to warn the users and skip the tests. -""" -- - - - - - """Constructor. Receives an optional downloader.""" - - -- - """Receives a URL and returns True if suitable for this IE.""" - -- - """Getter method for _WORKING.""" - -- - """Initializes an instance (authentication, etc).""" - - - -- - """Extracts URL information and returns it in list of dicts.""" - - -- - """Sets the downloader for this IE.""" - -- - """Real initialization process. Redefine in subclasses.""" - -- - """Real extraction process. Redefine in subclasses.""" -pass -- - - """Information extractor for youtube.com.""" -- - ( -(?:https?://)? # http(s):// (optional) -(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/| -tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains -(?:.*?\#/)? # handle anchor (#/) redirect urls -(?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs -(?: # the various things that can precede the ID: -(?:(?:v|embed|e)/) # v/ or embed/ or e/ -|(?: # or the v= param in all its forms -(?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx) -(?:\?|\#!?) # the params delimiter ? or # or #! -(?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx) -v= -) -)? # optional -> youtube.com/xxxx is OK -)? # all until now is optional -> you can pass the naked ID -([0-9A-Za-z_-]+) # here is it! the YouTube video ID -(?(1).+)? # if we found the ID, everything can follow -$""" - - - - - -# Listed in order of quality - - - -'13': '3gp', -'17': 'mp4', -'18': 'mp4', -'22': 'mp4', -'37': 'mp4', -'38': 'video', # You actually don't know if this will be MOV, AVI or whatever -'43': 'webm', -'44': 'webm', -'45': 'webm', -'46': 'webm', -} - -'5': '240x400', -'6': '???', -'13': '???', -'17': '144x176', -'18': '360x640', -'22': '720x1280', -'34': '360x640', -'35': '480x854', -'37': '1080x1920', -'38': '3072x4096', -'43': '360x640', -'44': '480x854', -'45': '720x1280', -'46': '1080x1920', -} - -- - """Receives a URL and returns True if suitable for this IE.""" - -- - """Report attempt to set language.""" - -- - """Report attempt to log in.""" -self._downloader.to_screen(u'[youtube] Logging in') -- - """Report attempt to confirm age.""" -self._downloader.to_screen(u'[youtube] Confirming age') -- - """Report attempt to download video webpage.""" - -- - """Report attempt to download video info webpage.""" - -- - """Report attempt to download video info webpage.""" - -- - """Report attempt to extract video information.""" - -- - """Report extracted video URL.""" -self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format)) -- - """Indicate the download will use the RTMP protocol.""" -self._downloader.to_screen(u'[youtube] RTMP download detected') -- - - - # TODO parse xml instead of regex - - - - - - - - - - - - -- - print('Available formats:') -for x in formats: -print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))) -- - - return -- - - - - # Attempt to use provided username and password or .netrc data - -username = downloader_params['username'] -password = downloader_params['password'] - -try: -info = netrc.netrc().authenticators(self._NETRC_MACHINE) -if info is not None: -username = info[0] -password = info[2] -else: -raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) -except (IOError, netrc.NetrcParseError) as err: -self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) -return -- # Set language - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err)) -return -- # No authentication to be performed - - -- # Log in -login_form = { -'current_form': 'loginForm', -'next': '/', -'action_login': 'Log In', -'username': username, -'password': password, -} -request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form)) -try: -self.report_login() -login_results = compat_urllib_request.urlopen(request).read().decode('utf-8') -if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None: -self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password') -return -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err)) -return -- # Confirm age -age_form = { -'next_url': '/', -'action_confirm': 'Confirm', -} -request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form)) -try: -self.report_age_confirmation() -age_results = compat_urllib_request.urlopen(request).read().decode('utf-8') -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) -return -- - # Extract original video URL from URL with redirection, like age verification, using next_url parameter - - -url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/') -- # Extract video id from URL - - -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return - -- # Get video webpage - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) -return -- - - # Attempt to extract SWF player URL - - -player_url = re.sub(r'\\(.)', r'\1', mobj.group(1)) -else: - -- # Get video info - - - -% (video_id, el_type)) - - - - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) -return - -if 'reason' in video_info: -self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0]) -else: -self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason') -return -- # Check for "rental" videos - -self._downloader.trouble(u'ERROR: "rental" videos not supported') -return -- # Start extracting information - -- # uploader - -self._downloader.trouble(u'ERROR: unable to extract uploader nickname') -return - -- # title - -self._downloader.trouble(u'ERROR: unable to extract video title') -return - -- # thumbnail image - -self._downloader.trouble(u'WARNING: unable to extract video thumbnail') -video_thumbnail = '' -else: # don't panic if we can't find it - -- # upload date - - - - - - - - - - -- # description - - - -else: -video_description = '' -- # closed captions - - - - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) - - - -raise Trouble(u'WARNING: video has no closed captions') - - - - -else: -srt_lang = srt_lang_list.keys()[0] - -raise Trouble(u'WARNING: no closed captions found in the specified language') - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err)) - -raise Trouble(u'WARNING: unable to download video subtitles') - -except Trouble as trouble: -self._downloader.trouble(str(trouble)) -- - self._downloader.trouble(u'WARNING: unable to extract video duration') -video_duration = '' -else: - -- # token - -- # Decide which formats to download - -- - self.report_rtmp_download() -video_url_list = [(None, video_info['conn'][0])] - - - - - -- - - - format_list = available_formats[available_formats.index(format_limit):] -else: - - - -self._downloader.trouble(u'ERROR: no known formats available for video') -return - -self._print_formats(existing_formats) -return - - -elif req_format == 'worst': -video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality -elif req_format in ('-1', 'all'): -video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats -else: -# Specific formats. We pick the first in a slash-delimeted sequence. -# For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'. -req_formats = req_format.split('/') -video_url_list = None -for rf in req_formats: -if rf in url_map: -video_url_list = [(rf, url_map[rf])] -break -if video_url_list is None: -self._downloader.trouble(u'ERROR: requested format not available') -return -else: -self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info') -return -- - - # Extension - -- - self._video_dimensions.get(format_param, '???')) -- - 'id': video_id, -'url': video_real_url, -'uploader': video_uploader, -'upload_date': upload_date, -'title': video_title, -'ext': video_extension, -'format': video_format, -'thumbnail': video_thumbnail, -'description': video_description, -'player_url': player_url, -'subtitles': video_subtitles, -'duration': video_duration -}) - -- - - """Information Extractor for metacafe.com.""" -- - - - - - - - - - """Report disclaimer retrieval.""" - -- - """Report attempt to confirm age.""" - -- - """Report webpage download.""" -self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id) -- - # Retrieve disclaimer - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err)) -return -- # Confirm age - -'filters': '0', -'submit': "Continue - I'm over 18", -} - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err)) -return -- - # Extract id and simplified title from URL - - -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -- - - # Check if video comes from YouTube - - - - -- # Retrieve video webpage to extract further information -request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) -try: -self.report_download_webpage(video_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) -return -- # Extract URL, uploader and title from webpage -self.report_extraction(video_id) -mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) -if mobj is not None: -mediaURL = compat_urllib_parse.unquote(mobj.group(1)) -video_extension = mediaURL[-3:] -- # Extract gdaKey if available -mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage) -if mobj is None: -video_url = mediaURL -else: -gdaKey = mobj.group(1) -video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) -else: -mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract media URL') -return -vardict = compat_parse_qs(mobj.group(1)) -if 'mediaData' not in vardict: -self._downloader.trouble(u'ERROR: unable to extract media URL') -return -mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0]) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract media URL') -return -mediaURL = mobj.group(1).replace('\\/', '/') -video_extension = mediaURL[-3:] -video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2)) -- mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract title') -return -video_title = mobj.group(1).decode('utf-8') -- mobj = re.search(r'submitter=(.*?);', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract uploader nickname') -return -video_uploader = mobj.group(1) -- return [{ -'id': video_id.decode('utf-8'), -'url': video_url.decode('utf-8'), -'uploader': video_uploader.decode('utf-8'), -'upload_date': None, -'title': video_title, -'ext': video_extension.decode('utf-8'), -}] -- - - """Information Extractor for Dailymotion""" -- - - - - - - - """Report webpage download.""" - -- - """Report information extraction.""" - -- - # Extract id and simplified title from URL - - -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -- - - - - # Retrieve video webpage to extract further information - - - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err)) -return -- # Extract URL, uploader and title from webpage - - - -self._downloader.trouble(u'ERROR: unable to extract media URL') - - -- - - - - - else: -self._downloader.trouble(u'ERROR: unable to extract video URL') -return -- - - self._downloader.trouble(u'ERROR: unable to extract video URL') -return -- - - # TODO: support choosing qualities -- - - self._downloader.trouble(u'ERROR: unable to extract title') - - -- - - - # lookin for official user -mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage) -if mobj_official is None: -self._downloader.trouble(u'WARNING: unable to extract uploader nickname') -else: -video_uploader = mobj_official.group(1) -else: - -- - - - video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1) -- - 'id': video_id, -'url': video_url, -'uploader': video_uploader, -'upload_date': video_upload_date, -'title': video_title, -'ext': video_extension, -}] -- - - """Information extractor for photobucket.com.""" -- - - - - InfoExtractor.__init__(self, downloader) -- - """Report webpage download.""" -self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id) -- - # Extract id from URL -mobj = re.match(self._VALID_URL, url) -if mobj is None: -self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) -return -- video_id = mobj.group(1) -- video_extension = 'flv' -- # Retrieve video webpage to extract further information -request = compat_urllib_request.Request(url) -try: -self.report_download_webpage(video_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- # Extract URL, uploader, and title from webpage -self.report_extraction(video_id) -mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract media URL') -return -mediaURL = compat_urllib_parse.unquote(mobj.group(1)) -- video_url = mediaURL -- mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract title') -return -video_title = mobj.group(1).decode('utf-8') -- video_uploader = mobj.group(2).decode('utf-8') -- return [{ -'id': video_id.decode('utf-8'), -'url': video_url.decode('utf-8'), -'uploader': video_uploader, -'upload_date': None, -'title': video_title, -'ext': video_extension.decode('utf-8'), -}] -- - - """Information extractor for video.yahoo.com.""" -- - # _VALID_URL matches all Yahoo! Video URLs -# _VPAGE_URL matches only the extractable '/watch/' URLs - - - -- - InfoExtractor.__init__(self, downloader) -- - """Report webpage download.""" -self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id) -- - # Extract ID from URL -mobj = re.match(self._VALID_URL, url) -if mobj is None: -self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) -return -- - - - # Rewrite valid but non-extractable URLs as -# extractable English language /watch/ URLs - - -try: - - -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- - if mobj is None: -self._downloader.trouble(u'ERROR: Unable to extract id field') - - -- mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: Unable to extract vid field') -return -yahoo_vid = mobj.group(1) -- url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id) -return self._real_extract(url, new_video=False) -- # Retrieve video webpage to extract further information -request = compat_urllib_request.Request(url) -try: -self.report_download_webpage(video_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- # Extract uploader and title from webpage -self.report_extraction(video_id) -mobj = re.search(r'<meta name="title" content="(.*)" />', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract video title') -return -video_title = mobj.group(1).decode('utf-8') -- mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract video uploader') -return -video_uploader = mobj.group(1).decode('utf-8') -- # Extract video thumbnail -mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract video thumbnail') -return -video_thumbnail = mobj.group(1).decode('utf-8') -- # Extract video description -mobj = re.search(r'<meta name="description" content="(.*)" />', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract video description') -return -video_description = mobj.group(1).decode('utf-8') - - -- # Extract video height and width -mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage) - - - - -- - if mobj is None: - - -yv_video_width = mobj.group(1) -- # Retrieve video playlist to extract media URL -# I'm not completely sure what all these options are, but we -# seem to need most of them, otherwise the server sends a 401. -yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents - - -'&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height + -'&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797') -try: -self.report_download_webpage(video_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- # Extract media URL from playlist XML -mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: Unable to extract media URL') -return -video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8') -video_url = unescapeHTML(video_url) -- return [{ -'id': video_id.decode('utf-8'), -'url': video_url, -'uploader': video_uploader, -'upload_date': None, -'title': video_title, -'ext': video_extension.decode('utf-8'), -'thumbnail': video_thumbnail.decode('utf-8'), -'description': video_description, -}] -- - - """Information extractor for vimeo.com.""" -- # _VALID_URL matches Vimeo URLs - - -- - - - - """Report webpage download.""" - -- - """Report information extraction.""" - -- - # Extract ID from URL - - -self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) -return -- - - # Retrieve video webpage to extract further information - - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- # Now we begin extracting as much information as we can from what we -# retrieved. First we extract the information common to all extractors, -# and latter we extract those that are Vimeo specific. - -- # Extract the config JSON - - - -except: -self._downloader.trouble(u'ERROR: unable to extract info section') -return -- # Extract title - -- # Extract uploader - -- # Extract video thumbnail - -- # Extract video description - - - -- # Extract upload date - - - -video_upload_date = mobj.group(1) -- # Vimeo specific: extract request signature and timestamp - - -- # Vimeo specific: extract video codec and quality information -# First consider quality, then codecs, then take everything -# TODO bind to format param - - - - - - -elif 'sd' in config["video"]["files"][codec_name]: -files['sd'].append((codec_name, codec_extension, 'sd')) -else: -files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0])) -- - - - - - - - else: - - -- - %(video_id, sig, timestamp, video_quality, video_codec.upper()) -- - 'id': video_id, -'url': video_url, -'uploader': video_uploader, -'upload_date': video_upload_date, -'title': video_title, -'ext': video_extension, -'thumbnail': video_thumbnail, -'description': video_description, -}] -- - - """arte.tv information extractor.""" -- - - - - - - - - - """Report webpage download.""" -self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id) -- - - - - - webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) - -except ValueError as err: - - -return webpage -- - page = self.fetch_webpage(url) - - -- - - - - - - - - else: - -- - - - video_lang = url.split('/')[-4] - -url, -r'src="(.*?/videothek_js.*?\.js)', -0, -[ -(1, 'url', u'ERROR: Invalid URL: %s' % url) -] -) -http_host = url.split('/')[2] -next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url'))) - -next_url, -r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' + -'(http://.*?\.swf).*?' + -'(rtmp://.*?)\'', -re.DOTALL, -[ -(1, 'path', u'ERROR: could not extract video path: %s' % url), -(2, 'player', u'ERROR: could not extract video player: %s' % url), -(3, 'url', u'ERROR: could not extract video url: %s' % url) -] -) - -- - - - url, -r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)', -0, -[ -(1, 'url', u'ERROR: Invalid URL: %s' % url) -] -) - - -next_url, -r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang, -0, -[ -(1, 'url', u'ERROR: Could not find <video> tag: %s' % url) -] -) - -- - next_url, -r'<video id="(.*?)".*?>.*?' + -'<name>(.*?)</name>.*?' + -'<dateVideo>(.*?)</dateVideo>.*?' + -'<url quality="hd">(.*?)</url>', -re.DOTALL, -[ -(1, 'id', u'ERROR: could not extract video id: %s' % url), -(2, 'title', u'ERROR: could not extract video title: %s' % url), -(3, 'date', u'ERROR: could not extract video date: %s' % url), -(4, 'url', u'ERROR: could not extract video url: %s' % url) -] -) -- return { -'id': info.get('id'), -'url': compat_urllib_parse.unquote(info.get('url')), -'uploader': u'arte.tv', -'upload_date': info.get('date'), -'title': info.get('title').decode('utf-8'), -'ext': u'mp4', -'format': u'NA', -'player_url': None, -} -- - - self.report_extraction(video_id) -- if re.search(self._LIVE_URL, video_id) is not None: -self.extractLiveStream(url) -return -else: -info = self.extractPlus7Stream(url) -- return [info] -- - - """Generic last-resort information extractor.""" -- - - - - InfoExtractor.__init__(self, downloader) -- - """Report webpage download.""" -self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.') -self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id) -- - """Report information extraction.""" -self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) -- - """Check if it is a redirect, like url shorteners, in case restart chain.""" -class HeadRequest(compat_urllib_request.Request): -def get_method(self): -return "HEAD" -- class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): -""" -Subclass the HTTPRedirectHandler to make it use our -HeadRequest also on the redirected URL -""" -def redirect_request(self, req, fp, code, msg, headers, newurl): -if code in (301, 302, 303, 307): -newurl = newurl.replace(' ', '%20') -newheaders = dict((k,v) for k,v in req.headers.items() -if k.lower() not in ("content-length", "content-type")) -return HeadRequest(newurl, -headers=newheaders, -origin_req_host=req.get_origin_req_host(), -unverifiable=True) -else: -raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) -- class HTTPMethodFallback(compat_urllib_request.BaseHandler): -""" -Fallback to GET if HEAD is not allowed (405 HTTP error) -""" -def http_error_405(self, req, fp, code, msg, headers): -fp.read() -fp.close() -- newheaders = dict((k,v) for k,v in req.headers.items() -if k.lower() not in ("content-length", "content-type")) -return self.parent.open(compat_urllib_request.Request(req.get_full_url(), -headers=newheaders, -origin_req_host=req.get_origin_req_host(), -unverifiable=True)) -- # Build our opener -opener = compat_urllib_request.OpenerDirector() -for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, -HTTPMethodFallback, HEADRedirectHandler, -compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: -opener.add_handler(handler()) -- response = opener.open(HeadRequest(url)) -new_url = response.geturl() -- if url == new_url: -return False -- self.report_following_redirect(new_url) -self._downloader.download([new_url]) -return True -- - if self._test_redirect(url): return -- video_id = url.split('/')[-1] -request = compat_urllib_request.Request(url) -try: -self.report_download_webpage(video_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -except ValueError as err: -# since this is the last-resort InfoExtractor, if -# this error is thrown, it'll be thrown here - - -- - # Start with something easy: JW Player in SWFObject - -if mobj is None: -# Broaden the search a little bit - -if mobj is None: - -return -- # It's possible that one of the regexes -# matched, but returned an empty group: - -self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) - -- - video_id = os.path.basename(video_url) -- # here's a fun little line of code for you: - -video_id = os.path.splitext(video_id)[0] -- # it's tempting to parse this further, but you would -# have to take into account all the variations like -# Video Title - Site Name -# Site Name | Video Title -# Video Title - Tagline | Site Name -# and so on and so forth; it's just not practical -mobj = re.search(r'<title>(.*)</title>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract title') -return -video_title = mobj.group(1) -- # video uploader is domain name -mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract title') -return -video_uploader = mobj.group(1) -- return [{ -'id': video_id, -'url': video_url, -'uploader': video_uploader, -'upload_date': None, -'title': video_title, -'ext': video_extension, -}] -- - - """Information Extractor for YouTube search queries.""" - - - - -- - InfoExtractor.__init__(self, downloader) -- - """Report attempt to download search page with given number.""" -query = query.decode(preferredencoding()) -self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum)) -- - mobj = re.match(self._VALID_URL, query) -if mobj is None: -self._downloader.trouble(u'ERROR: invalid search query "%s"' % query) -return -- - prefix = prefix[8:] - -if prefix == '': -self._download_n_results(query, 1) -return -elif prefix == 'all': -self._download_n_results(query, self._max_youtube_results) -return -else: -try: -n = int(prefix) -if n <= 0: -self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) -return -elif n > self._max_youtube_results: -self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n)) -n = self._max_youtube_results -self._download_n_results(query, n) -return -except ValueError: # parsing prefix as integer fails -self._download_n_results(query, 1) -return -- - """Downloads a specified number of results for a query""" -- video_ids = [] -pagenum = 0 -limit = n -- while (50 * pagenum) < limit: -self.report_download_page(query, pagenum+1) -result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1) -request = compat_urllib_request.Request(result_url) -try: -data = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err)) -return -api_response = json.loads(data)['data'] -- new_ids = list(video['id'] for video in api_response['items']) -video_ids += new_ids -- limit = min(n, api_response['totalItems']) -pagenum += 1 -- if len(video_ids) > n: -video_ids = video_ids[:n] -for id in video_ids: -self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) -return -- - - """Information Extractor for Google Video search queries.""" - - - - - - -- - InfoExtractor.__init__(self, downloader) -- - """Report attempt to download playlist page with given number.""" - - -- - - - - - - - - query = query.encode('utf-8') - -self._download_n_results(query, 1) -return - -self._download_n_results(query, self._max_google_results) - -else: -try: -n = int(prefix) -if n <= 0: -self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) -return -elif n > self._max_google_results: -self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n)) -n = self._max_google_results -self._download_n_results(query, n) -return -except ValueError: # parsing prefix as integer fails -self._download_n_results(query, 1) -return -- - """Downloads a specified number of results for a query""" -- video_ids = [] -pagenum = 0 -- while True: -self.report_download_page(query, pagenum) -result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10) -request = compat_urllib_request.Request(result_url) -try: -page = compat_urllib_request.urlopen(request).read() - -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) - -- # Extract video identifiers -for mobj in re.finditer(self._VIDEO_INDICATOR, page): -video_id = mobj.group(1) -if video_id not in video_ids: -video_ids.append(video_id) -if len(video_ids) == n: -# Specified n videos reached -for id in video_ids: -self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) -return -- if re.search(self._MORE_PAGES_INDICATOR, page) is None: -for id in video_ids: -self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id]) -return -- pagenum = pagenum + 1 -- - - """Information Extractor for Yahoo! Video search queries.""" -- - - - - - - - - - - - - """Report attempt to download playlist page with given number.""" - - -- - - if mobj is None: - -return -- - prefix = prefix[8:] - -if prefix == '': -self._download_n_results(query, 1) -return -elif prefix == 'all': -self._download_n_results(query, self._max_yahoo_results) -return -else: -try: -n = int(prefix) -if n <= 0: -self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query)) -return -elif n > self._max_yahoo_results: -self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n)) -n = self._max_yahoo_results -self._download_n_results(query, n) -return -except ValueError: # parsing prefix as integer fails -self._download_n_results(query, 1) -return -- - """Downloads a specified number of results for a query""" -- video_ids = [] -already_seen = set() -pagenum = 1 -- while True: - -result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum) -request = compat_urllib_request.Request(result_url) -try: -page = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) -return -- # Extract video identifiers -for mobj in re.finditer(self._VIDEO_INDICATOR, page): -video_id = mobj.group(1) -if video_id not in already_seen: -video_ids.append(video_id) -already_seen.add(video_id) -if len(video_ids) == n: -# Specified n videos reached -for id in video_ids: -self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) -return -- if re.search(self._MORE_PAGES_INDICATOR, page) is None: -for id in video_ids: -self._downloader.download(['http://video.yahoo.com/watch/%s' % id]) -return -- pagenum = pagenum + 1 -- - - """Information Extractor for YouTube playlists.""" -- - - - - - - - - - - """Report attempt to download playlist page with given number.""" - -- - # Extract playlist id - - -self._downloader.trouble(u'ERROR: invalid url: %s' % url) - -- # Single video case - -self._downloader.download([mobj.group(3)]) -return -- # Download playlist pages -# prefix is 'p' as default for playlists but there are other types that need extra care - - -playlist_access = 'artist' -else: - - - - - -- - - - - - - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) -return -- # Extract video identifiers - - - - - -- - - - - - - - - - - else: -video_ids = video_ids[playliststart:playlistend] -- - - else: -self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids))) -- - - - - - - """Information Extractor for YouTube channels.""" -- - - - - - - """Report attempt to download channel page with given number.""" - -- - # Extract channel id - - - - -- # Download channel pages - -video_ids = [] - -- - self.report_download_page(channel_id, pagenum) - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) - -- # Extract video identifiers -ids_in_page = [] -for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page): -if mobj.group(1) not in ids_in_page: - - -- - - - - - - - - - - - - """Information Extractor for YouTube users.""" -- - - - - - - - - - - - """Report attempt to download user page.""" - -(username, start_index, start_index + self._GDATA_PAGE_SIZE)) -- - # Extract username - - - - -- - - # Download video ids using YouTube Data API. Result size per -# query is limited (currently to 50 videos) so we need to query -# page by page until there are no video ids - it means we got -# all of them. -- - - - - - - - - - - - - - return -- # Extract video identifiers - -- - - - - - - # A little optimization - if current page is not -# "full", ie. does not contain PAGE_SIZE video ids then -# we can assume that this page is the last one - there -# are no more ids on further pages - no need to query -# again. -- - - - - - - - - - - - else: -video_ids = video_ids[playliststart:playlistend] -- - (username, all_ids_count, len(video_ids))) -- - - - - - """Information Extractor for blip.tv users.""" -- - - - - - - - - """Report attempt to download user page.""" - -(self.IE_NAME, username, pagenum)) -- - # Extract username - - -self._downloader.trouble(u'ERROR: invalid url: %s' % url) - -- - - page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1' -- - - - page = compat_urllib_request.urlopen(request).read().decode('utf-8') - -page_base = page_base % mobj.group(1) - -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) -return -- - # Download video ids using BlipTV Ajax calls. Result size per -# query is limited (currently to 12 videos) so we need to query -# page by page until there are no video ids - it means we got -# all of them. -- - - - while True: - -- - - try: -page = compat_urllib_request.urlopen(request).read().decode('utf-8') -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - -return -- # Extract video identifiers - -- - if mobj.group(1) not in ids_in_page: - -- video_ids.extend(ids_in_page) -- # A little optimization - if current page is not -# "full", ie. does not contain PAGE_SIZE video ids then -# we can assume that this page is the last one - there -# are no more ids on further pages - no need to query -# again. -- if len(ids_in_page) < self._PAGE_SIZE: - -- - - - - - - if playlistend == -1: - -else: - -- - (self.IE_NAME, username, all_ids_count, len(video_ids))) -- - - - - - """Information extractor for depositfiles.com""" -- - - - - InfoExtractor.__init__(self, downloader) -- - """Report webpage download.""" - -- - """Report information extraction.""" -self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id) -- - file_id = url.split('/')[-1] -# Rebuild url in english locale -url = 'http://depositfiles.com/en/files/' + file_id -- # Retrieve file webpage with 'Free download' button pressed -free_download_indication = { 'gateway_result' : '1' } -request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication)) -try: -self.report_download_webpage(file_id) -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err)) -return -- # Search for the real file URL -mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage) -if (mobj is None) or (mobj.group(1) is None): -# Try to figure out reason of the error. -mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL) -if (mobj is not None) and (mobj.group(1) is not None): -restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip() -self._downloader.trouble(u'ERROR: %s' % restriction_message) -else: -self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url) -return -- file_url = mobj.group(1) -file_extension = os.path.splitext(file_url)[1][1:] -- # Search for file title -mobj = re.search(r'<b title="(.*?)">', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract title') -return -file_title = mobj.group(1).decode('utf-8') -- return [{ -'id': file_id.decode('utf-8'), -'url': file_url.decode('utf-8'), -'uploader': None, -'upload_date': None, -'title': file_title, -'ext': file_extension.decode('utf-8'), -}] -- - - """Information Extractor for Facebook""" -- - - - - - - 'video': 'mp4', -'highqual': 'mp4', -'lowqual': 'mp4', -} - -- - InfoExtractor.__init__(self, downloader) -- - """Add header and report message.""" -self._downloader.to_screen(u'[facebook] %s' % message) -- - """Report attempt to log in.""" - -- - """Report attempt to download video webpage.""" - -- - """Report attempt to extract video information.""" - -- - """Extract video information from page""" -# General data - -'description': r'<div class="datawrap">(.*?)</div>', -'owner': r'\("video_owner_name", "(.*?)"\)', -'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)', -} -video_info = {} - -mobj = re.search(data[piece], video_webpage) -if mobj is not None: -video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) -- # Video urls -video_urls = {} -for fmt in self._available_formats: -mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage) -if mobj is not None: -# URL is in a Javascript segment inside an escaped Unicode format within -# the generally utf-8 page -video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape")) -video_info['video_urls'] = video_urls -- return video_info -- - if self._downloader is None: -return -- useremail = None -password = None -downloader_params = self._downloader.params -- # Attempt to use provided username and password or .netrc data -if downloader_params.get('username', None) is not None: -useremail = downloader_params['username'] -password = downloader_params['password'] -elif downloader_params.get('usenetrc', False): -try: -info = netrc.netrc().authenticators(self._NETRC_MACHINE) -if info is not None: -useremail = info[0] -password = info[2] -else: -raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE) -except (IOError, netrc.NetrcParseError) as err: -self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err)) -return -- if useremail is None: -return -- # Log in - -'email': useremail, -'pass': password, -'login': 'Log+In' -} - - - - - - - - - -return -- - - - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - -video_id = mobj.group('ID') -- # Get video webpage - -request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id) - -page = compat_urllib_request.urlopen(request) - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - -return -- # Start extracting information - -- # Extract information -video_info = self._parse_page(video_webpage) -- # uploader -if 'owner' not in video_info: -self._downloader.trouble(u'ERROR: unable to extract uploader nickname') -return -video_uploader = video_info['owner'] -- # title -if 'title' not in video_info: -self._downloader.trouble(u'ERROR: unable to extract video title') -return -video_title = video_info['title'] -video_title = video_title.decode('utf-8') -- # thumbnail image -if 'thumbnail' not in video_info: -self._downloader.trouble(u'WARNING: unable to extract video thumbnail') -video_thumbnail = '' -else: -video_thumbnail = video_info['thumbnail'] -- # upload date - -if 'upload_date' in video_info: - -timetuple = email.utils.parsedate_tz(upload_time) -if timetuple is not None: -try: -upload_date = time.strftime('%Y%m%d', timetuple[0:9]) -except: -pass -- # description -video_description = video_info.get('description', 'No description available.') -- url_map = video_info['video_urls'] -if len(url_map.keys()) > 0: -# Decide which formats to download -req_format = self._downloader.params.get('format', None) -format_limit = self._downloader.params.get('format_limit', None) -- if format_limit is not None and format_limit in self._available_formats: -format_list = self._available_formats[self._available_formats.index(format_limit):] -else: -format_list = self._available_formats -existing_formats = [x for x in format_list if x in url_map] -if len(existing_formats) == 0: -self._downloader.trouble(u'ERROR: no known formats available for video') -return -if req_format is None: -video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality -elif req_format == 'worst': -video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality -elif req_format == '-1': -video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats -else: -# Specific format -if req_format not in url_map: -self._downloader.trouble(u'ERROR: requested format not available') -return -video_url_list = [(req_format, url_map[req_format])] # Specific format -- results = [] -for format_param, video_real_url in video_url_list: -# Extension -video_extension = self._video_extensions.get(format_param, 'mp4') -- - 'id': video_id.decode('utf-8'), -'url': video_real_url.decode('utf-8'), -'uploader': video_uploader.decode('utf-8'), -'upload_date': upload_date, -'title': video_title, -'ext': video_extension.decode('utf-8'), -'format': (format_param is None and u'NA' or format_param.decode('utf-8')), -'thumbnail': video_thumbnail.decode('utf-8'), -'description': video_description.decode('utf-8'), -}) -return results -- - """Information extractor for blip.tv""" -- - - - - - """Report information extraction.""" - -- - """Report information extraction.""" -self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title)) -- - - - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -- - cchar = '&' -else: - - - - - - - - -basename = url.split('/')[-1] -title,ext = os.path.splitext(basename) -title = title.decode('UTF-8') -ext = ext.replace('.', '') -self.report_direct_download(title) -info = { -'id': title, -'url': url, -'uploader': None, -'upload_date': None, -'title': title, -'ext': ext, -'urlhandle': urlh -} -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err)) -return - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err)) -return -- - - - - else: -data = json_data -- - - - - raise ValueError('Can not determine filename extension') - -- - 'id': data['item_id'], -'url': video_url, -'uploader': data['display_name'], -'upload_date': upload_date, -'title': data['title'], -'ext': ext, -'format': data['media']['mimeType'], -'thumbnail': data['thumbnailUrl'], -'description': data['description'], -'player_url': data['embedUrl'] -} -except (ValueError,KeyError) as err: -self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err)) -return -- - - - - - """Information Extractor for myvideo.de.""" -- - - - - - - - """Report webpage download.""" - -- - """Report information extraction.""" - -- - - - - - - - - # Get video webpage - - - - - -self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err)) -return -- - - webpage) - -self._downloader.trouble(u'ERROR: unable to extract media URL') -return - -- - - self._downloader.trouble(u'ERROR: unable to extract title') -return -- - - - 'id': video_id, -'url': video_url, -'uploader': None, -'upload_date': None, -'title': video_title, -'ext': u'flv', -}] -- - """Information extractor for The Daily Show and Colbert Report """ -- # urls can be abbreviations like :thedailyshow or :colbert -# urls for episodes like: -# or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day -# or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news -# or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524 - -|(https?://)?(www\.)? -(?P<showname>thedailyshow|colbertnation)\.com/ -(full-episodes/(?P<episode>.*)| -(?P<clip> -(the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?)) -|(watch/(?P<date>[^/]*)/(?P<tdstitle>.*))))) -$""" - -- - - - '3500': 'mp4', -'2200': 'mp4', -'1700': 'mp4', -'1200': 'mp4', -'750': 'mp4', -'400': 'mp4', -} - -'3500': '1280x720', -'2200': '960x540', -'1700': '768x432', -'1200': '640x360', -'750': '512x288', -'400': '384x216', -} -- - """Receives a URL and returns True if suitable for this IE.""" - -- - self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id) -- - - - - self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id) -- - self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id) -- - - print('Available formats:') -for x in formats: -print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???'))) -- - - mobj = re.match(self._VALID_URL, url, re.VERBOSE) -if mobj is None: -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -- if mobj.group('shortname'): -if mobj.group('shortname') in ('tds', 'thedailyshow'): -url = u'http://www.thedailyshow.com/full-episodes/' -else: -url = u'http://www.colbertnation.com/full-episodes/' -mobj = re.match(self._VALID_URL, url, re.VERBOSE) -assert mobj is not None -- if mobj.group('clip'): -if mobj.group('showname') == 'thedailyshow': -epTitle = mobj.group('tdstitle') -else: -epTitle = mobj.group('cntitle') -dlNewest = False -else: -dlNewest = not mobj.group('episode') -if dlNewest: -epTitle = mobj.group('showname') -else: -epTitle = mobj.group('episode') -- req = compat_urllib_request.Request(url) - - - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) -return -if dlNewest: -url = htmlHandle.geturl() -mobj = re.match(self._VALID_URL, url, re.VERBOSE) -if mobj is None: -self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url) -return -if mobj.group('episode') == '': -self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url) - - -- - - - # The Colbert Report embeds the information in a without -# a URL prefix; so extract the alternate reference -# and then add the URL prefix manually. -- - - - - else: - -- - - - - - - self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err)) - -- - indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri}) - - -indexXml = compat_urllib_request.urlopen(indexUrl).read() - - -return -- - - - itemEls = idoc.findall('.//item') - -mediaId = itemEl.findall('./guid')[0].text - -showId = mediaId.split(':')[-2].replace('.com', '') -officialTitle = itemEl.findall('./title')[0].text -officialDate = itemEl.findall('./pubDate')[0].text -- - compat_urllib_parse.urlencode({'uri': mediaId})) -configReq = compat_urllib_request.Request(configUrl) -self.report_config_download(epTitle) -try: -configXml = compat_urllib_request.urlopen(configReq).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err)) -return -- cdoc = xml.etree.ElementTree.fromstring(configXml) -turls = [] -for rendition in cdoc.findall('.//rendition'): -finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text) -turls.append(finfo) -- if len(turls) == 0: -self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found') -continue -- if self._downloader.params.get('listformats', None): -self._print_formats([i[0] for i in turls]) -return -- # For now, just pick the highest bitrate -format,video_url = turls[-1] -- # Get the format arg from the arg stream -req_format = self._downloader.params.get('format', None) -- # Select format if we can find one -for f,v in turls: -if f == req_format: -format, video_url = f, v -break -- # Patch to download from alternative CDN, which does not -# break on current RTMPDump builds -broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/" -better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/" -- if video_url.startswith(broken_cdn): -video_url = video_url.replace(broken_cdn, better_cdn) -- effTitle = showId + u'-' + epTitle -info = { -'id': shortMediaId, -'url': video_url, -'uploader': showId, -'upload_date': officialDate, -'title': effTitle, -'ext': 'mp4', -'format': format, -'thumbnail': None, -'description': officialTitle, -'player_url': None #playerUrl -} -- results.append(info) -- return results -- - - """Information extractor for The Escapist """ -- - - - - - - - - - - - - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return - - -- - - - - - - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err)) -return -- - - - - - - - - - - - - - - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err)) -return -- # Technically, it's JavaScript, not JSON - -- - - except (ValueError,) as err: -self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err)) - -- - - - - 'id': videoId, -'url': videoUrl, -'uploader': showName, -'upload_date': None, -'title': showName, -'ext': 'flv', -'thumbnail': imgUrl, -'description': description, -'player_url': playerUrl, -} -- - - - - """Information extractor for collegehumor.com""" -- - - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - - - - - video_id = mobj.group('videoid') -- - 'id': video_id, -'uploader': None, -'upload_date': None, -} -- self.report_extraction(video_id) -xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id -try: -metaXml = compat_urllib_request.urlopen(xmlUrl).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) -return -- mdoc = xml.etree.ElementTree.fromstring(metaXml) -try: -videoNode = mdoc.findall('./video')[0] -info['description'] = videoNode.findall('./description')[0].text -info['title'] = videoNode.findall('./caption')[0].text -info['thumbnail'] = videoNode.findall('./thumbnail')[0].text -manifest_url = videoNode.findall('./file')[0].text -except IndexError: -self._downloader.trouble(u'\nERROR: Invalid metadata XML file') -return -- manifest_url += '?hdcore=2.10.3' -self.report_manifest(video_id) -try: -manifestXml = compat_urllib_request.urlopen(manifest_url).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err)) -return -- adoc = xml.etree.ElementTree.fromstring(manifestXml) -try: -media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0] -node_id = media_node.attrib['url'] -video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text -except IndexError as err: -self._downloader.trouble(u'\nERROR: Invalid manifest file') -return -- url_pr = compat_urllib_parse_urlparse(manifest_url) -url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1' -- info['url'] = url -info['ext'] = 'f4f' -return [info] -- - - """Information extractor for xvideos.com""" -- - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - - - - - - - - - - - - - - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - -- - - - # Extract video URL - - - -return - -- - # Extract title - - - -return - -- - # Extract video thumbnail - - - -return - -- - 'id': video_id, -'url': video_url, -'uploader': None, -'upload_date': None, -'title': video_title, -'ext': 'flv', -'thumbnail': video_thumbnail, -'description': None, -} -- - - - - """Information extractor for soundcloud.com -To access the media, the uid of the song and a stream token -must be extracted from the page source and the script must make -a request to media.soundcloud.com/crossdomain.xml. Then -the media can be grabbed by requesting from an url composed -of the stream token and uid -""" -- - - - - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - - - self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - -- # extract uploader (which is in the url) - -# extract simple title (uploader + slug of song title) - - -- - - - - - - - - - self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) - -- - - - - - - - - - - - - - - - - - 'id': info['id'], -'url': mediaURL, -'uploader': info['user']['username'], -'upload_date': info['created_at'], -'title': info['title'], -'ext': u'mp3', -'description': info['description'], -}] -- - - """Information extractor for infoq.com""" -- - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - mobj = re.match(self._VALID_URL, url) -if mobj is None: -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) - -- - - request = compat_urllib_request.Request(url) - - -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - - -- - - - # Extract video URL - - - - - -- - # Extract title - -if mobj is None: - - -video_title = mobj.group(1).decode('utf-8') -- # Extract description - - - - -- - - - - 'id': video_id, -'url': video_url, -'uploader': None, -'upload_date': None, -'title': video_title, -'ext': extension, # Extension is always(?) mp4, but seems to be flv -'thumbnail': None, -'description': video_description, -} -- - - - """Information extractor for www.mixcloud.com""" -- - - - - - - - - """Report JSON download.""" - -- - """Report information extraction.""" - -- - """Get urls from 'audio_formats' section in json""" - -try: - - -bitrate = max(bitrate_list) # select highest -- - - url_list = jsonData[fmt] - -- - """Returns 1st active url from list""" - -try: -compat_urllib_request.urlopen(url) -return url -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - -- - - - print('Available formats:') -for fmt in formats.keys(): -for b in formats[fmt]: -try: -ext = formats[fmt][b][0] -print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1])) -except TypeError: # we have no bitrate info - -print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1])) - -- - mobj = re.match(self._VALID_URL, url) -if mobj is None: -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -# extract uploader & filename from url -uploader = mobj.group(1).decode('utf-8') -file_id = uploader + "-" + mobj.group(2).decode('utf-8') -- # construct API request -file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json' -# retrieve .json file with links to files -request = compat_urllib_request.Request(file_url) -try: -self.report_download_json(file_url) -jsonData = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err)) -return -- # parse JSON -json_data = json.loads(jsonData) -player_url = json_data['player_swf_url'] -formats = dict(json_data['audio_formats']) -- req_format = self._downloader.params.get('format', None) -bitrate = None -- if self._downloader.params.get('listformats', None): -self._print_formats(formats) -return -- - - - # check urls - - -break # got it! -else: -if req_format not in formats.keys(): -self._downloader.trouble(u'ERROR: format is not available') - -- url_list = self.get_urls(formats, req_format) -file_url = self.check_urls(url_list) - -- - 'id': file_id.decode('utf-8'), -'url': file_url.decode('utf-8'), -'uploader': uploader.decode('utf-8'), -'upload_date': None, -'title': json_data['name'], -'ext': file_url.split('.')[-1].decode('utf-8'), -'format': (format_param is None and u'NA' or format_param.decode('utf-8')), -'thumbnail': json_data['thumbnail_url'], -'description': json_data['description'], -'player_url': player_url.decode('utf-8'), -}] -- - """Information extractor for Stanford's Open ClassRoom""" -- - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - - - - - - - - - - 'id': course + '_' + video, -'uploader': None, -'upload_date': None, -} -- - - - - - - - - - - - - - - - - - elif mobj.group('course'): # A course page -course = mobj.group('course') -info = { -'id': course, -'type': 'playlist', -'uploader': None, -'upload_date': None, -} -- self.report_download_webpage(info['id']) -try: -coursepage = compat_urllib_request.urlopen(url).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err)) -return -- m = re.search('<h1>([^<]+)</h1>', coursepage) -if m: -info['title'] = unescapeHTML(m.group(1)) -else: -info['title'] = info['id'] -- m = re.search('<description>([^<]+)</description>', coursepage) -if m: -info['description'] = unescapeHTML(m.group(1)) -- links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage)) -info['list'] = [ -{ -'type': 'reference', -'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage), -} -for vpage in links] -results = [] -for entry in info['list']: -assert entry['type'] == 'reference' -results += self.extract(entry['url']) - -- else: # Root page - -'id': 'Stanford OpenClassroom', -'type': 'playlist', -'uploader': None, -'upload_date': None, -} -- - rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php' - - - - - -- - - - - { -'type': 'reference', -'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage), -} -for cpage in links] -- - - - - - - - """Information extractor for MTV.com""" -- - - - - """Report information extraction.""" - -- - """Report information extraction.""" - -- - mobj = re.match(self._VALID_URL, url) -if mobj is None: -self._downloader.trouble(u'ERROR: invalid URL: %s' % url) -return -if not mobj.group('proto'): -url = 'http://' + url -video_id = mobj.group('videoid') -self.report_webpage(video_id) -- request = compat_urllib_request.Request(url) -try: -webpage = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err)) -return -- mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract song name') -return -song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1')) -mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract performer') -return -performer = unescapeHTML(mobj.group(1).decode('iso-8859-1')) -video_title = performer + ' - ' + song_name -- mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to mtvn_uri') -return -mtvn_uri = mobj.group(1) -- mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage) -if mobj is None: -self._downloader.trouble(u'ERROR: unable to extract content id') -return -content_id = mobj.group(1) -- videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri -self.report_extraction(video_id) -request = compat_urllib_request.Request(videogen_url) -try: -metadataXml = compat_urllib_request.urlopen(request).read() -except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: -self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err)) -return -- mdoc = xml.etree.ElementTree.fromstring(metadataXml) -renditions = mdoc.findall('.//rendition') -- # For now, always pick the highest quality. -rendition = renditions[-1] -- try: -_,_,ext = rendition.attrib['type'].partition('/') -format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate'] -video_url = rendition.find('./src').text -except KeyError: -self._downloader.trouble('Invalid rendition field.') -return -- - 'id': video_id, -'url': video_url, -'uploader': performer, -'upload_date': None, -'title': video_title, -'ext': ext, -'format': format, -} -- return [info] -- - - - - - - - - - - """Report webpage download.""" - -- - """Report information extraction.""" - -- - - - - - - - - - - - - - - - - #return ''.join(mixed) - -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - else: - - -elif format == 'worst': -format = 'mp4' -ext = u'mp4' -else: - - -- - - - - - - - - - - - #column 8,9 of fileid represent the segment number -#fileid[7:9] should be changed - -- - - - - 'id': '%s_part%02d' % (video_id, index), -'url': download_url, -'uploader': None, -'upload_date': None, -'title': video_title, -'ext': ext, -} - -- - - - - """Information extractor for xnxx.com""" -- - - - - - - - """Report information extraction""" - -- - """Report information extraction""" - -- - - - - - - - - - # Get webpage content - - - - - - -- - - - - - - - - - - - - - - - - - - - 'id': video_id, -'url': video_url, -'uploader': None, -'upload_date': None, -'title': video_title, -'ext': 'flv', -'thumbnail': video_thumbnail, -'description': None, -}] -- - - """Information extractor for plus.google.com.""" -- - - - - - - - """Report downloading extry""" - -- - """Report downloading extry""" - -- - """Report downloading extry""" - -- - """Report downloading extry""" - -- - """Report information extraction.""" - -- - # Extract id from URL - - - - -- - - - - - # Step 1, Retrieve post webpage to extract further information - - - - - - -return -- # Extract update date - - - - - -# Convert timestring to a format suitable for filename - - - -- # Extract uploader - - - - - - -- # Extract title -# Get the first line for title - - - - - - -- # Step 2, Stimulate clicking the image box to launch video - - - - -- - - - - - - - - - - # Extract video links on video page -"""Extract video links of all sizes""" - - - - -- # Sort in resolution - -- # Choose the lowest of the sort, i.e. highest resolution - -# Only get the url. The resolution part in the tuple has no use anymore - -# Treat escaped \u0026 style hex - - - - -- - - 'id': video_id, -'url': video_url, -'uploader': uploader, -'upload_date': upload_date, -'title': video_title, -'ext': video_extension, -}] -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - else: - -- - - - 'id': shortened_video_id, -'url': video_url, -'ext': 'mp4', -'title': title, -'uploader_date': _findProp(r'<b>Date:</b> (.*?)</div>'), -'description': _findProp(r'<div class="description">(.*?)</h1>'), -} - -- - """Information extractor for justin.tv and twitch.tv""" -# TODO: One broadcast may be split into multiple videos. The key -# 'broadcast_id' is the same for all parts, and 'broadcast_part' -# starts at 1 and increases. Can we treat all parts as one video? -- - ([^/]+)(?:/b/([^/]+))?/?(?:\#.*)?$""" - - -- - """Report information extraction.""" - -- - """Report attempt to download a single page of videos.""" - -(self.IE_NAME, channel, offset, offset + self._JUSTIN_PAGE_LIMIT)) -- # Return count of items, list of *valid* items - - - - - - - - -- - - - - - - - - 'id': clip['id'], -'url': video_url, -'title': clip['title'], -'uploader': clip.get('user_id', clip.get('channel_id')), -'upload_date': video_date, -'ext': video_extension, -}) - -- - - - - - - - - - - - - else: - - -- - - - - - - - - - - - - - - - - |
-
-
-Hot-keys on this page
-- r - m - x - p toggle line displays -
-- j - k next/prev highlighted chunk -
-- 0 (zero) top of page -
-- 1 (one) first highlighted chunk -
-| - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | -
- #!/usr/bin/env python -# -*- coding: utf-8 -*- -- - - - - - - - - - - - """Post Processor class. -- PostProcessor objects can be added to downloaders with their -add_post_processor() method. When the downloader has finished a -successful download, it will take its internal chain of PostProcessors -and start calling the run() method on each one of them, first with -an initial argument and then with the returned value of the previous -PostProcessor. -- The chain will be stopped if one of them ever returns None or the end -of the chain is reached. -- PostProcessor objects follow a "mutual registration" process similar -to InfoExtractor objects. -""" -- - - - self._downloader = downloader -- - """Sets the downloader for this PP.""" -self._downloader = downloader -- - """Run the PostProcessor. -- The "information" argument is a dictionary like the ones -composed by InfoExtractors. The only difference is that this -one has an extra field called "filepath" that points to the -downloaded file. -- When this method returns None, the postprocessing chain is -stopped. However, this method may return an information -dictionary that will be passed to the next postprocessing -object in the chain. It can be the one it received after -changing some fields. -- In addition, this method may raise a PostProcessingError -exception that will be taken into account by the downloader -it was called from. -""" -return information # by default, do nothing -- - - self.message = message -- - - PostProcessor.__init__(self, downloader) -if preferredcodec is None: -preferredcodec = 'best' -self._preferredcodec = preferredcodec -self._preferredquality = preferredquality -self._keepvideo = keepvideo -self._exes = self.detect_executables() -- - def detect_executables(): -def executable(exe): -try: -subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() -except OSError: -return False -return exe -programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe'] -return dict((program, executable(program)) for program in programs) -- - if not self._exes['ffprobe'] and not self._exes['avprobe']: return None -try: -cmd = [self._exes['avprobe'] or self._exes['ffprobe'], '-show_streams', '--', encodeFilename(path)] -handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE) -output = handle.communicate()[0] -if handle.wait() != 0: -return None -except (IOError, OSError): -return None -audio_codec = None -for line in output.decode('ascii', 'ignore').split('\n'): -if line.startswith('codec_name='): -audio_codec = line.split('=')[1].strip() -elif line.strip() == 'codec_type=audio' and audio_codec is not None: -return audio_codec -return None -- - if not self._exes['ffmpeg'] and not self._exes['avconv']: -raise AudioConversionError('ffmpeg or avconv not found. Please install one.') -if codec is None: -acodec_opts = [] -else: -acodec_opts = ['-acodec', codec] -cmd = ([self._exes['avconv'] or self._exes['ffmpeg'], '-y', '-i', encodeFilename(path), '-vn'] -+ acodec_opts + more_opts + -['--', encodeFilename(out_path)]) -p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) -stdout,stderr = p.communicate() -if p.returncode != 0: -msg = stderr.strip().split('\n')[-1] -raise AudioConversionError(msg) -- - path = information['filepath'] -- filecodec = self.get_audio_codec(path) -if filecodec is None: -self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe') -return None -- more_opts = [] -if self._preferredcodec == 'best' or self._preferredcodec == filecodec or (self._preferredcodec == 'm4a' and filecodec == 'aac'): -if self._preferredcodec == 'm4a' and filecodec == 'aac': -# Lossless, but in another container -acodec = 'copy' -extension = self._preferredcodec -more_opts = [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] -elif filecodec in ['aac', 'mp3', 'vorbis']: -# Lossless if possible -acodec = 'copy' -extension = filecodec -if filecodec == 'aac': -more_opts = ['-f', 'adts'] -if filecodec == 'vorbis': -extension = 'ogg' -else: -# MP3 otherwise. -acodec = 'libmp3lame' -extension = 'mp3' -more_opts = [] -if self._preferredquality is not None: -if int(self._preferredquality) < 10: -more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] -else: -more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] -else: -# We convert the audio (lossy) -acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'm4a': 'aac', 'vorbis': 'libvorbis', 'wav': None}[self._preferredcodec] -extension = self._preferredcodec -more_opts = [] -if self._preferredquality is not None: -if int(self._preferredquality) < 10: -more_opts += [self._exes['avconv'] and '-q:a' or '-aq', self._preferredquality] -else: -more_opts += [self._exes['avconv'] and '-b:a' or '-ab', self._preferredquality + 'k'] -if self._preferredcodec == 'aac': -more_opts += ['-f', 'adts'] -if self._preferredcodec == 'm4a': -more_opts += [self._exes['avconv'] and '-bsf:a' or '-absf', 'aac_adtstoasc'] -if self._preferredcodec == 'vorbis': -extension = 'ogg' -if self._preferredcodec == 'wav': -extension = 'wav' -more_opts += ['-f', 'wav'] -- prefix, sep, ext = path.rpartition(u'.') # not os.path.splitext, since the latter does not work on unicode in all setups -new_path = prefix + sep + extension -self._downloader.to_screen(u'[' + (self._exes['avconv'] and 'avconv' or 'ffmpeg') + '] Destination: ' + new_path) -try: -self.run_ffmpeg(path, new_path, acodec, more_opts) -except: -etype,e,tb = sys.exc_info() -if isinstance(e, AudioConversionError): -self._downloader.to_stderr(u'ERROR: audio conversion failed: ' + e.message) -else: -self._downloader.to_stderr(u'ERROR: error running ' + (self._exes['avconv'] and 'avconv' or 'ffmpeg')) -return None -- # Try to update the date time for extracted audio file. -if information.get('filetime') is not None: -try: -os.utime(encodeFilename(new_path), (time.time(), information['filetime'])) -except: -self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file') -- if not self._keepvideo: -try: -os.remove(encodeFilename(path)) -except (IOError, OSError): -self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file') -return None -- information['filepath'] = new_path -return information - - |
-
-
-Hot-keys on this page
-- r - m - x - p toggle line displays -
-- j - k next/prev highlighted chunk -
-- 0 (zero) top of page -
-- 1 (one) first highlighted chunk -
-| - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - | -
- #!/usr/bin/env python -# -*- coding: utf-8 -*- -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. -# Python 2's version is apparently totally broken - - - - - - - - - - -# pct_sequence: contiguous sequence of percent-encoded bytes, decoded - - - - - -raise ValueError - - - -# This segment was just a single percent-encoded character. -# May be part of a sequence of code units, so delay decoding. -# (Stored in pct_sequence). - - -rest = '%' + item -# Encountered non-percent-encoded characters. Flush the current -# pct_sequence. - - - -# Flush the final pct_sequence - - -- - encoding='utf-8', errors='replace'): - - - - - -continue - - -if strict_parsing: - -# Handle case of a control-name with no equal sign - - -else: - - - - - - - - - - -- - encoding='utf-8', errors='replace'): - - -encoding=encoding, errors=errors) - - - -else: - - -- - - - - - - - - - - - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0', -'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', -'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', -'Accept-Encoding': 'gzip, deflate', -'Accept-Language': 'en-us,en;q=0.5', -} - -"""Get preferred encoding. -- Returns the best encoding scheme for the system, based on -locale.getpreferredencoding() and some further tweaks. -""" - - - - -pref = 'UTF-8' -- - - - - - else: - -assert type(s) == type(u'') -print(s) -- - """Transforms an HTML entity to a character. -- This function receives a match object and is intended to be used with -the re.sub() function. -""" - -- # Known non-numeric HTML entity - - -- - - - - base = 16 -numstr = u'0%s' % numstr -else: - - -- # Unknown entity in name, return its literal representation - -- - - """Modified HTMLParser that isolates a tag with the specified id""" - - - - - - - - - -- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - """Needed to put the start position of the result (self.result[1]) -after the opening tag with the requested id""" - - - - -handle_decl = handle_pi = unknown_decl = find_startpos -- - - - - - - - - - - - - - - """Return the content of the tag with the specified id in the passed HTML document""" - - - - - - -- - - """Clean an HTML snippet into a readable string""" -# Newline vs <br /> - - -# Strip html tags - -# Replace html entities - - -- - - """Try to open the given filename, and slightly tweak it if this fails. -- Attempts to open the given filename. If this fails, it tries to change -the filename slightly, step by step, until it's either able to open it -or it fails and raises a final exception, like the standard open() -function. -- It returns the tuple (stream, definitive_file_name). -""" - - -if sys.platform == 'win32': -import msvcrt -msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) -return (sys.stdout, filename) - - -except (IOError, OSError) as err: -# In case of error, try to remove win32 forbidden chars -filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename) -- # An exception here should be caught in the caller - - -- - - """Convert RFC 2822 defined time string into system timestamp""" - - - - - -- - """Sanitizes a string so it could be used as part of a filename. -If restricted is set, use a stricter subset of allowed characters. -Set is_id if this is not an arbitrary string, but an ID that should be kept if possible -""" - - - - - - - - - - - - - - -- - - - - - # Common case of "Foreign band name - English song title" - - - - - -- - """ Remove all duplicates from the input iterable """ - - - - - -- - """ -@param s a string -""" - -- - - - - """ -@param s The name of the file -""" -- - - # Python 3 has a Unicode API - - -- - # Pass u'' directly to use Unicode APIs on Windows 2000 and up -# (Detecting Windows NT 4 is tricky because 'major >= 4' would -# match Windows 9x series as well. Besides, NT 4 is obsolete.) -return s -else: - -- - """Download Error exception. -- This exception may be thrown by FileDownloader objects if they are not -configured to continue on errors. They will contain the appropriate -error message. -""" - -- - - """Same File exception. -- This exception will be thrown by FileDownloader objects if they detect -multiple files would have to be downloaded to the same file on disk. -""" - -- - - """Post Processing exception. -- This exception may be raised by PostProcessor's .run() method to -indicate an error in the postprocessing task. -""" - -- - """ --max-downloads limit has been reached. """ - -- - - """Unavailable Format exception. -- This exception will be thrown when a video is requested -in a format that is not available for that video. -""" - -- - - """Content Too Short exception. -- This exception may be raised by FileDownloader objects when a file they -download is too small for what the server announced first, indicating -the connection was probably interrupted. -""" -# Both in bytes - - -- - self.downloaded = downloaded -self.expected = expected -- - - """Trouble helper exception -- This is an exception to be handled with -FileDownloader.trouble -""" -- - """Handler for HTTP requests and responses. -- This class, when installed with an OpenerDirector, automatically adds -the standard headers to every HTTP request and handles gzipped and -deflated responses from web servers. If compression is to be avoided in -a particular request, the original request in the program code only has -to include the HTTP header "Youtubedl-No-Compression", which will be -removed before making the real request. -- Part of this code was copied from: -- http://techknack.net/python-urllib2-handlers/ -- Andrew Rowls, the author of that code, agreed to release it to the -public domain. -""" -- - def deflate(data): -try: - - -return zlib.decompress(data) -- - def addinfourl_wrapper(stream, headers, url, code): - - - - - -- - - - - - - - - - - - - - # gzip - - - - -# deflate - - -resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) -resp.msg = old_resp.msg - -- - - - |
-
-
-Hot-keys on this page
-- r - m - x - p toggle line displays -
-- j - k next/prev highlighted chunk -
-- 0 (zero) top of page -
-- 1 (one) first highlighted chunk -
-| - - - - | -
- - - - |
-