Simplify tests

* Make them directly executable again
* Move common stuff (md5, parameters) to helper
* Never import *
* General clean up

test/helper.py

@@ -1,22 +1,27 @@
 import errno
 import io
+import hashlib
 import json
 import os.path
 import re
 import types
 
 import youtube_dl.extractor
-from youtube_dl import YoutubeDL, YoutubeDLHandler
-from youtube_dl.utils import (
-    compat_cookiejar,
-    compat_urllib_request,
-)
+from youtube_dl import YoutubeDL
 
+
+def global_setup():
     youtube_dl._setup_opener(timeout=10)
 
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+
+def get_params(override=None):
+    PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                   "parameters.json")
     with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
         parameters = json.load(pf)
+    if override:
+        parameters.update(override)
+    return parameters
 
 
 def try_rm(filename):
@@ -32,7 +37,7 @@ class FakeYDL(YoutubeDL):
     def __init__(self):
         # Different instances of the downloader can't share the same dictionary
         # some test set the "sublang" parameter, which would break the md5 checks.
-        params = dict(parameters)
+        params = get_params()
         super(FakeYDL, self).__init__(params)
         self.result = []
         
@@ -62,3 +67,6 @@ def get_testcases():
         for t in getattr(ie, '_TESTS', []):
             t['name'] = type(ie).__name__[:-len('IE')]
             yield t
+
+
+md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()

test/test_age_restriction.py

@@ -1,14 +1,16 @@
 #!/usr/bin/env python
 
-import sys
-import unittest
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import global_setup, try_rm
+global_setup()
+
 
 from youtube_dl import YoutubeDL
-from .helper import try_rm
 
 
 def _download_restricted(url, filename, age):

test/test_all_urls.py

@@ -1,14 +1,20 @@
 #!/usr/bin/env python
 
-import sys
-import unittest
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from test.helper import get_testcases
+
+from youtube_dl.extractor import (
+    gen_extractors,
+    JustinTVIE,
+    YoutubeIE,
+)
 
-from youtube_dl.extractor import YoutubeIE, YoutubePlaylistIE, YoutubeChannelIE, JustinTVIE, gen_extractors
-from .helper import get_testcases
 
 class TestAllURLsMatching(unittest.TestCase):
     def setUp(self):

test/test_dailymotion_subtitles.py

@@ -1,18 +1,16 @@
 #!/usr/bin/env python
 
-import sys
-import unittest
-import hashlib
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup, md5
+global_setup()
+
 
 from youtube_dl.extractor import DailymotionIE
-from youtube_dl.utils import *
-from .helper import FakeYDL
-
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 class TestDailymotionSubtitles(unittest.TestCase):
     def setUp(self):

test/test_download.py

@@ -1,26 +1,31 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import get_params, get_testcases, global_setup, try_rm, md5
+global_setup()
+
+
 import hashlib
 import io
-import os
 import json
-import unittest
-import sys
 import socket
-import binascii
-
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import youtube_dl.YoutubeDL
-from youtube_dl.utils import *
-
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
+from youtube_dl.utils import (
+    compat_str,
+    compat_urllib_error,
+    DownloadError,
+    ExtractorError,
+    UnavailableVideoError,
+)
 
 RETRIES = 3
 
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
-
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         self.to_stderr = self.to_screen
@@ -37,18 +42,12 @@ def _file_md5(fn):
     with open(fn, 'rb') as f:
         return hashlib.md5(f.read()).hexdigest()
 
-import test.helper as helper  # Set up remaining global configuration
-from .helper import get_testcases, try_rm
 defs = get_testcases()
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    parameters = json.load(pf)
-
 
 class TestDownload(unittest.TestCase):
     maxDiff = None
     def setUp(self):
-        self.parameters = parameters
         self.defs = defs
 
 ### Dynamically generate tests
@@ -68,8 +67,7 @@ def generator(test_case):
             print_skipping(test_case['skip'])
             return
 
-        params = self.parameters.copy()
-        params.update(test_case.get('params', {}))
+        params = get_params(test_case.get('params', {}))
 
         ydl = YoutubeDL(params)
         ydl.add_default_info_extractors()

test/test_playlists.py

@@ -1,13 +1,16 @@
 #!/usr/bin/env python
 # encoding: utf-8
 
-import sys
-import unittest
-import json
 
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup
+global_setup()
+
 
 from youtube_dl.extractor import (
     DailymotionPlaylistIE,
@@ -18,9 +21,7 @@ from youtube_dl.extractor import (
     LivestreamIE,
     NHLVideocenterIE,
 )
-from youtube_dl.utils import *
 
-from .helper import FakeYDL
 
 class TestPlaylists(unittest.TestCase):
     def assertIsPlaylist(self, info):

test/test_utils.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python
 
-# Various small unit tests
-
-import sys
-import unittest
-import xml.etree.ElementTree
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+# Various small unit tests
+import xml.etree.ElementTree
 
 #from youtube_dl.utils import htmlentity_transform
 from youtube_dl.utils import (

test/test_write_annotations.py

@@ -1,39 +1,38 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import xml.etree.ElementTree
+# Allow direct execution
 import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from test.helper import get_params, global_setup, try_rm
+global_setup()
+
+
+import io
+
+import xml.etree.ElementTree
 
 import youtube_dl.YoutubeDL
 import youtube_dl.extractor
-from youtube_dl.utils import *
-from .helper import try_rm
+from youtube_dl.utils import True
 
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
-
-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         super(YoutubeDL, self).__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    params = json.load(pf)
-params['writeannotations'] = True
-params['skip_download'] = True
-params['writeinfojson'] = False
-params['format'] = 'flv'
+params = get_params({
+    'writeannotations': True,
+    'skip_download': True,
+    'writeinfojson': False,
+    'format': 'flv',
+})
+
+
 
 TEST_ID = 'gr51aVj-mLg'
 ANNOTATIONS_FILE = TEST_ID + '.flv.annotations.xml'

test/test_write_info_json.py

@@ -1,37 +1,34 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-import json
+# Allow direct execution
 import os
 import sys
 import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-# Allow direct execution
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from test.helper import get_params, global_setup
+global_setup()
+
+
+import io
+import json
 
 import youtube_dl.YoutubeDL
 import youtube_dl.extractor
-from youtube_dl.utils import *
 
-PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "parameters.json")
-
-# General configuration (from __init__, not very elegant...)
-jar = compat_cookiejar.CookieJar()
-cookie_processor = compat_urllib_request.HTTPCookieProcessor(jar)
-proxy_handler = compat_urllib_request.ProxyHandler()
-opener = compat_urllib_request.build_opener(proxy_handler, cookie_processor, YoutubeDLHandler())
-compat_urllib_request.install_opener(opener)
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
         super(YoutubeDL, self).__init__(*args, **kwargs)
         self.to_stderr = self.to_screen
 
-with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
-    params = json.load(pf)
-params['writeinfojson'] = True
-params['skip_download'] = True
-params['writedescription'] = True
+params = get_params({
+    'writeinfojson': True,
+    'skip_download': True,
+    'writedescription': True,
+})
+
 
 TEST_ID = 'BaW_jenozKc'
 INFO_JSON_FILE = TEST_ID + '.mp4.info.json'
@@ -42,6 +39,7 @@ This is a test video for youtube-dl.
 
 For more information, contact phihag@phihag.de .'''
 
+
 class TestInfoJSON(unittest.TestCase):
     def setUp(self):
         # Clear old files

test/test_youtube_lists.py

@@ -1,17 +1,23 @@
 #!/usr/bin/env python
 
-import sys
-import unittest
-import json
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.extractor import YoutubeUserIE, YoutubePlaylistIE, YoutubeIE, YoutubeChannelIE, YoutubeShowIE
-from youtube_dl.utils import *
+from test.helper import FakeYDL, global_setup
+global_setup()
+
+
+from youtube_dl.extractor import (
+    YoutubeUserIE,
+    YoutubePlaylistIE,
+    YoutubeIE,
+    YoutubeChannelIE,
+    YoutubeShowIE,
+)
 
-from .helper import FakeYDL
 
 class TestYoutubeLists(unittest.TestCase):
     def assertIsPlaylist(self, info):

test/test_youtube_signature.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python
 
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import global_setup
+global_setup()
+
+
 import io
 import re
 import string
-import sys
-import unittest
-
-# Allow direct execution
-import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.utils import compat_str, compat_urlretrieve

test/test_youtube_subtitles.py

@@ -1,69 +1,79 @@
 #!/usr/bin/env python
 
-import sys
-import unittest
-import hashlib
-
 # Allow direct execution
 import os
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL, global_setup, md5
+global_setup()
+
 
 from youtube_dl.extractor import YoutubeIE
-from youtube_dl.utils import *
-from .helper import FakeYDL
 
-md5 = lambda s: hashlib.md5(s.encode('utf-8')).hexdigest()
 
 class TestYoutubeSubtitles(unittest.TestCase):
     def setUp(self):
         self.DL = FakeYDL()
         self.url = 'QRS8MkLhQmM'
+
     def getInfoDict(self):
         IE = YoutubeIE(self.DL)
         info_dict = IE.extract(self.url)
         return info_dict
+
     def getSubtitles(self):
         info_dict = self.getInfoDict()
         return info_dict[0]['subtitles']
+
     def test_youtube_no_writesubtitles(self):
         self.DL.params['writesubtitles'] = False
         subtitles = self.getSubtitles()
         self.assertEqual(subtitles, None)
+
     def test_youtube_subtitles(self):
         self.DL.params['writesubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['en']), '4cd9278a35ba2305f47354ee13472260')
+
     def test_youtube_subtitles_lang(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['subtitleslangs'] = ['it']
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['it']), '164a51f16f260476a05b50fe4c2f161d')
+
     def test_youtube_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
+
     def test_youtube_subtitles_sbv_format(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['subtitlesformat'] = 'sbv'
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['en']), '13aeaa0c245a8bed9a451cb643e3ad8b')
+
     def test_youtube_subtitles_vtt_format(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['subtitlesformat'] = 'vtt'
         subtitles = self.getSubtitles()
         self.assertEqual(md5(subtitles['en']), '356cdc577fde0c6783b9b822e7206ff7')
+
     def test_youtube_list_subtitles(self):
         self.DL.expect_warning(u'Video doesn\'t have automatic captions')
         self.DL.params['listsubtitles'] = True
         info_dict = self.getInfoDict()
         self.assertEqual(info_dict, None)
+
     def test_youtube_automatic_captions(self):
         self.url = '8YoUxe5ncPo'
         self.DL.params['writeautomaticsub'] = True
         self.DL.params['subtitleslangs'] = ['it']
         subtitles = self.getSubtitles()
         self.assertTrue(subtitles['it'] is not None)
+
     def test_youtube_nosubtitles(self):
         self.DL.expect_warning(u'video doesn\'t have subtitles')
         self.url = 'sAjKT8FhjI8'
@@ -71,6 +81,7 @@ class TestYoutubeSubtitles(unittest.TestCase):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles), 0)
+
     def test_youtube_multiple_langs(self):
         self.url = 'QRS8MkLhQmM'
         self.DL.params['writesubtitles'] = True