MarcoBuster/yt-dlp
1
0
Fork 0
mirror of https://github.com/yt-dlp/yt-dlp.git synced 2024-11-04 12:07:12 +01:00
Code Issues Projects Releases Wiki Activity

reorganized the titles sanitizing: now title is the untouched title

Browse Source 
and stitle is created in process_info() and is cross-filesystem sanitized by sanitize_filename();
closes #164
This commit is contained in:
Filippo Valsorda 2012-05-09 14:47:28 +02:00
parent 0b8c922da9
commit 2c288bda42
4 changed files with 17 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
youtube-dl
View File

Binary file not shown.

2
youtube_dl/FileDownloader.py
View File

@ -344,6 +344,8 @@ def _match_entry(self, info_dict):
def process_info(self, info_dict): def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor.""" """Process a single dictionary returned by an InfoExtractor."""
info_dict['stitle'] = sanitize_filename(info_dict['title'])
reason = self._match_entry(info_dict) reason = self._match_entry(info_dict)
if reason is not None: if reason is not None:
self.to_screen(u'[download] ' + reason) self.to_screen(u'[download] ' + reason)

60
youtube_dl/InfoExtractors.py
View File

@ -39,7 +39,6 @@ class InfoExtractor(object):
url: Final video URL. url: Final video URL.
uploader: Nickname of the video uploader. uploader: Nickname of the video uploader.
title: Literal title. title: Literal title.
stitle: Simplified title.
ext: Video filename extension. ext: Video filename extension.
format: Video format. format: Video format.
player_url: SWF Player URL (may be None). player_url: SWF Player URL (may be None).
@ -327,10 +326,6 @@ def _real_extract(self, url):
return return
video_title = urllib.unquote_plus(video_info['title'][0]) video_title = urllib.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8') video_title = video_title.decode('utf-8')
video_title = sanitize_title(video_title)
# simplified title
simple_title = simplify_title(video_title)
# thumbnail image # thumbnail image
if 'thumbnail_url' not in video_info: if 'thumbnail_url' not in video_info:
@ -447,7 +442,6 @@ def _real_extract(self, url):
'uploader': video_uploader.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date, 'upload_date': upload_date,
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')), 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'), 'thumbnail': video_thumbnail.decode('utf-8'),
@ -523,8 +517,6 @@ def _real_extract(self, url):
self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)]) self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
return return
simple_title = mobj.group(2).decode('utf-8')
# Retrieve video webpage to extract further information # Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id) request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
try: try:
@ -570,7 +562,6 @@ def _real_extract(self, url):
self._downloader.trouble(u'ERROR: unable to extract title') self._downloader.trouble(u'ERROR: unable to extract title')
return return
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage) mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
if mobj is None: if mobj is None:
@ -584,7 +575,6 @@ def _real_extract(self, url):
'uploader': video_uploader.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -651,8 +641,6 @@ def _real_extract(self, url):
self._downloader.trouble(u'ERROR: unable to extract title') self._downloader.trouble(u'ERROR: unable to extract title')
return return
video_title = unescapeHTML(mobj.group('title').decode('utf-8')) video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage) mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
if mobj is None: if mobj is None:
@ -666,7 +654,6 @@ def _real_extract(self, url):
'uploader': video_uploader.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -730,8 +717,6 @@ def _real_extract(self, url):
self._downloader.trouble(u'ERROR: unable to extract title') self._downloader.trouble(u'ERROR: unable to extract title')
return return
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
# Extract video description # Extract video description
mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage) mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
@ -764,7 +749,6 @@ def _real_extract(self, url):
'uploader': u'NA', 'uploader': u'NA',
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -823,8 +807,6 @@ def _real_extract(self, url):
self._downloader.trouble(u'ERROR: unable to extract title') self._downloader.trouble(u'ERROR: unable to extract title')
return return
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
video_uploader = mobj.group(2).decode('utf-8') video_uploader = mobj.group(2).decode('utf-8')
@ -834,7 +816,6 @@ def _real_extract(self, url):
'uploader': video_uploader, 'uploader': video_uploader,
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -912,7 +893,6 @@ def _real_extract(self, url, new_video=True):
self._downloader.trouble(u'ERROR: unable to extract video title') self._downloader.trouble(u'ERROR: unable to extract video title')
return return
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
simple_title = simplify_title(video_title)
mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage) mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
if mobj is None: if mobj is None:
@ -978,7 +958,6 @@ def _real_extract(self, url, new_video=True):
'uploader': video_uploader, 'uploader': video_uploader,
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'), 'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description, 'description': video_description,
@ -1038,7 +1017,6 @@ def _real_extract(self, url, new_video=True):
# Extract title # Extract title
video_title = config["video"]["title"] video_title = config["video"]["title"]
simple_title = simplify_title(video_title)
# Extract uploader # Extract uploader
video_uploader = config["video"]["owner"]["name"] video_uploader = config["video"]["owner"]["name"]
@ -1084,7 +1062,6 @@ def _real_extract(self, url, new_video=True):
'uploader': video_uploader, 'uploader': video_uploader,
'upload_date': video_upload_date, 'upload_date': video_upload_date,
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension, 'ext': video_extension,
'thumbnail': video_thumbnail, 'thumbnail': video_thumbnail,
'description': video_description, 'description': video_description,
@ -1219,8 +1196,6 @@ def _real_extract(self, url):
self._downloader.trouble(u'ERROR: unable to extract title') self._downloader.trouble(u'ERROR: unable to extract title')
return return
video_title = mobj.group(1).decode('utf-8') video_title = mobj.group(1).decode('utf-8')
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
# video uploader is domain name # video uploader is domain name
mobj = re.match(r'(?:https?://)?([^/]*)/.*', url) mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
@ -1235,7 +1210,6 @@ def _real_extract(self, url):
'uploader': video_uploader, 'uploader': video_uploader,
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -1700,7 +1674,6 @@ def _real_extract(self, url):
'uploader': u'NA', 'uploader': u'NA',
'upload_date': u'NA', 'upload_date': u'NA',
'title': file_title, 'title': file_title,
'stitle': file_title,
'ext': file_extension.decode('utf-8'), 'ext': file_extension.decode('utf-8'),
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -1845,9 +1818,6 @@ def _real_extract(self, url):
return return
video_title = video_info['title'] video_title = video_info['title']
video_title = video_title.decode('utf-8') video_title = video_title.decode('utf-8')
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
# thumbnail image # thumbnail image
if 'thumbnail' not in video_info: if 'thumbnail' not in video_info:
@ -1908,7 +1878,6 @@ def _real_extract(self, url):
'uploader': video_uploader.decode('utf-8'), 'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date, 'upload_date': upload_date,
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': video_extension.decode('utf-8'), 'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')), 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'), 'thumbnail': video_thumbnail.decode('utf-8'),
@ -1958,7 +1927,6 @@ def _real_extract(self, url):
'id': title, 'id': title,
'url': url, 'url': url,
'title': title, 'title': title,
'stitle': simplify_title(title),
'ext': ext, 'ext': ext,
'urlhandle': urlh 'urlhandle': urlh
} }
@ -1992,7 +1960,6 @@ def _real_extract(self, url):
'uploader': data['display_name'], 'uploader': data['display_name'],
'upload_date': upload_date, 'upload_date': upload_date,
'title': data['title'], 'title': data['title'],
'stitle': simplify_title(data['title']),
'ext': ext, 'ext': ext,
'format': data['media']['mimeType'], 'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'], 'thumbnail': data['thumbnailUrl'],
@ -2054,9 +2021,6 @@ def _real_extract(self,url):
return return
video_title = mobj.group(1) video_title = mobj.group(1)
video_title = sanitize_title(video_title)
simple_title = simplify_title(video_title)
return [{ return [{
'id': video_id, 'id': video_id,
@ -2064,7 +2028,6 @@ def _real_extract(self,url):
'uploader': u'NA', 'uploader': u'NA',
'upload_date': u'NA', 'upload_date': u'NA',
'title': video_title, 'title': video_title,
'stitle': simple_title,
'ext': u'flv', 'ext': u'flv',
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -2191,7 +2154,6 @@ def _real_extract(self, url):
'uploader': showId, 'uploader': showId,
'upload_date': officialDate, 'upload_date': officialDate,
'title': effTitle, 'title': effTitle,
'stitle': simplify_title(effTitle),
'ext': 'mp4', 'ext': 'mp4',
'format': format, 'format': format,
'thumbnail': None, 'thumbnail': None,
@ -2265,7 +2227,6 @@ def _real_extract(self, url):
'uploader': showName, 'uploader': showName,
'upload_date': None, 'upload_date': None,
'title': showName, 'title': showName,
'stitle': simplify_title(showName),
'ext': 'flv', 'ext': 'flv',
'format': 'flv', 'format': 'flv',
'thumbnail': imgUrl, 'thumbnail': imgUrl,
@ -2329,7 +2290,6 @@ def _real_extract(self, url):
videoNode = mdoc.findall('./video')[0] videoNode = mdoc.findall('./video')[0]
info['description'] = videoNode.findall('./description')[0].text info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text info['title'] = videoNode.findall('./caption')[0].text
info['stitle'] = simplify_title(info['title'])
info['url'] = videoNode.findall('./file')[0].text info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
info['ext'] = info['url'].rpartition('.')[2] info['ext'] = info['url'].rpartition('.')[2]
@ -2403,7 +2363,6 @@ def _real_extract(self, url):
'uploader': None, 'uploader': None,
'upload_date': None, 'upload_date': None,
'title': video_title, 'title': video_title,
'stitle': simplify_title(video_title),
'ext': 'flv', 'ext': 'flv',
'format': 'flv', 'format': 'flv',
'thumbnail': video_thumbnail, 'thumbnail': video_thumbnail,
@ -2447,7 +2406,7 @@ def _real_extract(self, url):
uploader = mobj.group(1).decode('utf-8') uploader = mobj.group(1).decode('utf-8')
# extract simple title (uploader + slug of song title) # extract simple title (uploader + slug of song title)
slug_title = mobj.group(2).decode('utf-8') slug_title = mobj.group(2).decode('utf-8')
simple_title = uploader + '-' + slug_title simple_title = uploader + u'-' + slug_title
self.report_webpage('%s/%s' % (uploader, slug_title)) self.report_webpage('%s/%s' % (uploader, slug_title))
@ -2469,7 +2428,9 @@ def _real_extract(self, url):
# extract unsimplified title # extract unsimplified title
mobj = re.search('"title":"(.*?)",', webpage) mobj = re.search('"title":"(.*?)",', webpage)
if mobj: if mobj:
title = mobj.group(1) title = mobj.group(1).decode('utf-8')
else:
title = simple_title
# construct media url (with uid/token) # construct media url (with uid/token)
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s" mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
@ -2498,8 +2459,7 @@ def _real_extract(self, url):
'url': mediaURL, 'url': mediaURL,
'uploader': uploader.decode('utf-8'), 'uploader': uploader.decode('utf-8'),
'upload_date': upload_date, 'upload_date': upload_date,
'title': simple_title.decode('utf-8'), 'title': title,
'stitle': simple_title.decode('utf-8'),
'ext': u'mp3', 'ext': u'mp3',
'format': u'NA', 'format': u'NA',
'player_url': None, 'player_url': None,
@ -2569,7 +2529,6 @@ def _real_extract(self, url):
'uploader': None, 'uploader': None,
'upload_date': None, 'upload_date': None,
'title': video_title, 'title': video_title,
'stitle': simplify_title(video_title),
'ext': extension, 'ext': extension,
'format': extension, # Extension is always(?) mp4, but seems to be flv 'format': extension, # Extension is always(?) mp4, but seems to be flv
'thumbnail': None, 'thumbnail': None,
@ -2685,7 +2644,6 @@ def _real_extract(self, url):
'uploader': uploader.decode('utf-8'), 'uploader': uploader.decode('utf-8'),
'upload_date': u'NA', 'upload_date': u'NA',
'title': json_data['name'], 'title': json_data['name'],
'stitle': simplify_title(json_data['name']),
'ext': file_url.split('.')[-1].decode('utf-8'), 'ext': file_url.split('.')[-1].decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')), 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': json_data['thumbnail_url'], 'thumbnail': json_data['thumbnail_url'],
@ -2717,7 +2675,7 @@ def _real_extract(self, url):
course = mobj.group('course') course = mobj.group('course')
video = mobj.group('video') video = mobj.group('video')
info = { info = {
'id': simplify_title(course + '_' + video), 'id': course + '_' + video,
} }
self.report_extraction(info['id']) self.report_extraction(info['id'])
@ -2735,14 +2693,13 @@ def _real_extract(self, url):
except IndexError: except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file') self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return return
info['stitle'] = simplify_title(info['title'])
info['ext'] = info['url'].rpartition('.')[2] info['ext'] = info['url'].rpartition('.')[2]
info['format'] = info['ext'] info['format'] = info['ext']
return [info] return [info]
elif mobj.group('course'): # A course page elif mobj.group('course'): # A course page
course = mobj.group('course') course = mobj.group('course')
info = { info = {
'id': simplify_title(course), 'id': course,
'type': 'playlist', 'type': 'playlist',
} }
@ -2758,7 +2715,6 @@ def _real_extract(self, url):
info['title'] = unescapeHTML(m.group(1)) info['title'] = unescapeHTML(m.group(1))
else: else:
info['title'] = info['id'] info['title'] = info['id']
info['stitle'] = simplify_title(info['title'])
m = re.search('<description>([^<]+)</description>', coursepage) m = re.search('<description>([^<]+)</description>', coursepage)
if m: if m:
@ -2792,7 +2748,6 @@ def _real_extract(self, url):
return return
info['title'] = info['id'] info['title'] = info['id']
info['stitle'] = simplify_title(info['title'])
links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage)) links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
info['list'] = [ info['list'] = [
@ -2891,7 +2846,6 @@ def _real_extract(self, url):
'url': video_url, 'url': video_url,
'uploader': performer, 'uploader': performer,
'title': video_title, 'title': video_title,
'stitle': simplify_title(video_title),
'ext': ext, 'ext': ext,
'format': format, 'format': format,
} }

18
youtube_dl/utils.py
View File

@ -156,12 +156,6 @@ def clean_html(html):
return html return html
def sanitize_title(utitle):
"""Sanitizes a video title so it could be used as part of a filename."""
utitle = unescapeHTML(utitle)
return utitle.replace(unicode(os.sep), u'%')
def sanitize_open(filename, open_mode): def sanitize_open(filename, open_mode):
"""Try to open the given filename, and slightly tweak it if this fails. """Try to open the given filename, and slightly tweak it if this fails.
@ -196,10 +190,14 @@ def timeconvert(timestr):
if timetuple is not None: if timetuple is not None:
timestamp = email.utils.mktime_tz(timetuple) timestamp = email.utils.mktime_tz(timetuple)
return timestamp return timestamp
def simplify_title(title): def sanitize_filename(s):
expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) """Sanitizes a string so it could be used as part of a filename."""
return expr.sub(u'_', title).strip(u'_') def replace_insane(char):
if char in u' .\\/|?*<>:"' or ord(char) < 32:
return '_'
return char
return u''.join(map(replace_insane, s)).strip('_')
def orderedSet(iterable): def orderedSet(iterable):
""" Remove all duplicates from the input iterable """ """ Remove all duplicates from the input iterable """