summaryrefslogtreecommitdiffstats
path: root/dom/media/test/external/external_media_harness
diff options
context:
space:
mode:
Diffstat (limited to 'dom/media/test/external/external_media_harness')
-rw-r--r--dom/media/test/external/external_media_harness/__init__.py5
-rw-r--r--dom/media/test/external/external_media_harness/runtests.py103
-rw-r--r--dom/media/test/external/external_media_harness/testcase.py362
3 files changed, 470 insertions, 0 deletions
diff --git a/dom/media/test/external/external_media_harness/__init__.py b/dom/media/test/external/external_media_harness/__init__.py
new file mode 100644
index 000000000..906473f0b
--- /dev/null
+++ b/dom/media/test/external/external_media_harness/__init__.py
@@ -0,0 +1,5 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from runtests import cli
diff --git a/dom/media/test/external/external_media_harness/runtests.py b/dom/media/test/external/external_media_harness/runtests.py
new file mode 100644
index 000000000..08d1a323b
--- /dev/null
+++ b/dom/media/test/external/external_media_harness/runtests.py
@@ -0,0 +1,103 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+import mozlog
+
+from manifestparser import read_ini
+from marionette_harness import (
+ BaseMarionetteTestRunner,
+ BaseMarionetteArguments,
+ BrowserMobProxyArguments,
+)
+from marionette_harness.runtests import MarionetteHarness, cli as mn_cli
+
+import external_media_tests
+from testcase import MediaTestCase
+from external_media_tests.media_utils.video_puppeteer import debug_script
+
+
+class MediaTestArgumentsBase(object):
+ name = 'Firefox Media Tests'
+ args = [
+ [['--urls'], {
+ 'help': 'ini file of urls to make available to all tests',
+ 'default': os.path.join(external_media_tests.urls, 'default.ini'),
+ }],
+ ]
+
+ def verify_usage_handler(self, args):
+ if args.urls:
+ if not os.path.isfile(args.urls):
+ raise ValueError('--urls must provide a path to an ini file')
+ else:
+ path = os.path.abspath(args.urls)
+ args.video_urls = MediaTestArgumentsBase.get_urls(path)
+ if not args.video_urls:
+ raise ValueError('list of video URLs cannot be empty')
+
+ def parse_args_handler(self, args):
+ if not args.tests:
+ args.tests = [external_media_tests.manifest]
+
+
+ @staticmethod
+ def get_urls(manifest):
+ with open(manifest, 'r'):
+ return [line[0] for line in read_ini(manifest)]
+
+
+class MediaTestArguments(BaseMarionetteArguments):
+ def __init__(self, **kwargs):
+ BaseMarionetteArguments.__init__(self, **kwargs)
+ self.register_argument_container(MediaTestArgumentsBase())
+ self.register_argument_container(BrowserMobProxyArguments())
+
+
+class MediaTestRunner(BaseMarionetteTestRunner):
+ def __init__(self, **kwargs):
+ BaseMarionetteTestRunner.__init__(self, **kwargs)
+ if not self.server_root:
+ self.server_root = external_media_tests.resources
+ # pick up prefs from marionette_driver.geckoinstance.DesktopInstance
+ self.app = 'fxdesktop'
+ self.test_handlers = [MediaTestCase]
+
+ # Used in HTML report (--log-html)
+ def gather_media_debug(test, status):
+ rv = {}
+ marionette = test._marionette_weakref()
+
+ if marionette.session is not None:
+ try:
+ with marionette.using_context(marionette.CONTEXT_CHROME):
+ debug_lines = marionette.execute_script(debug_script)
+ if debug_lines:
+ name = 'mozMediaSourceObject.mozDebugReaderData'
+ rv[name] = '\n'.join(debug_lines)
+ else:
+ logger = mozlog.get_default_logger()
+ logger.info('No data available about '
+ 'mozMediaSourceObject')
+ except:
+ logger = mozlog.get_default_logger()
+ logger.warning('Failed to gather test failure media debug',
+ exc_info=True)
+ return rv
+
+ self.result_callbacks.append(gather_media_debug)
+
+
+class FirefoxMediaHarness(MarionetteHarness):
+ def parse_args(self, *args, **kwargs):
+ return MarionetteHarness.parse_args(self, {'mach': sys.stdout})
+
+
+def cli():
+ mn_cli(MediaTestRunner, MediaTestArguments, FirefoxMediaHarness)
+
+if __name__ == '__main__':
+ cli()
diff --git a/dom/media/test/external/external_media_harness/testcase.py b/dom/media/test/external/external_media_harness/testcase.py
new file mode 100644
index 000000000..56350ccd9
--- /dev/null
+++ b/dom/media/test/external/external_media_harness/testcase.py
@@ -0,0 +1,362 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import os
+
+from marionette_driver import Wait
+from marionette_driver.errors import TimeoutException
+from marionette_harness import (
+ BrowserMobProxyTestCaseMixin,
+ MarionetteTestCase,
+ Marionette,
+ SkipTest,
+)
+
+from firefox_puppeteer import PuppeteerMixin
+from external_media_tests.utils import (timestamp_now, verbose_until)
+from external_media_tests.media_utils.video_puppeteer import (
+ VideoException,
+ VideoPuppeteer
+)
+
+
+class MediaTestCase(PuppeteerMixin, MarionetteTestCase):
+
+ """
+ Necessary methods for MSE playback
+
+ :param video_urls: Urls you are going to play as part of the tests.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.video_urls = kwargs.pop('video_urls', False)
+ super(MediaTestCase, self).__init__(*args, **kwargs)
+
+ def save_screenshot(self):
+ """
+ Make a screenshot of the window that is currently playing the video
+ element.
+ """
+ screenshot_dir = os.path.join(self.marionette.instance.workspace or '',
+ 'screenshots')
+ filename = ''.join([self.id().replace(' ', '-'),
+ '_',
+ str(timestamp_now()),
+ '.png'])
+ path = os.path.join(screenshot_dir, filename)
+ if not os.path.exists(screenshot_dir):
+ os.makedirs(screenshot_dir)
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ img_data = self.marionette.screenshot()
+ with open(path, 'wb') as f:
+ f.write(img_data.decode('base64'))
+ self.marionette.log('Screenshot saved in {}'
+ .format(os.path.abspath(path)))
+
+ def log_video_debug_lines(self, video):
+ """
+ Log the debugging information that Firefox provides for video elements.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ debug_lines = video.get_debug_lines()
+ if debug_lines:
+ self.marionette.log('\n'.join(debug_lines))
+
+ def run_playback(self, video):
+ """
+ Play the video all of the way through, or for the requested duration,
+ whichever comes first. Raises if the video stalls for too long.
+
+ :param video: VideoPuppeteer instance to play.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ self.logger.info(video.test_url)
+ try:
+ verbose_until(Wait(video, interval=video.interval,
+ timeout=video.expected_duration * 1.3 +
+ video.stall_wait_time),
+ video, VideoPuppeteer.playback_done)
+ except VideoException as e:
+ raise self.failureException(e)
+
+ def check_playback_starts(self, video):
+ """
+ Check to see if a given video will start. Raises if the video does not
+ start.
+
+ :param video: VideoPuppeteer instance to play.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ self.logger.info(video.test_url)
+ try:
+ verbose_until(Wait(video, timeout=video.timeout),
+ video, VideoPuppeteer.playback_started)
+ except TimeoutException as e:
+ raise self.failureException(e)
+
+ def skipTest(self, reason):
+ """
+ Skip this test.
+
+ Skip with marionette.marionette_test import SkipTest so that it
+ gets recognized a skip in marionette.marionette_test.CommonTestCase.run
+ """
+ raise SkipTest(reason)
+
+
+class NetworkBandwidthTestCase(MediaTestCase):
+ """
+ Test MSE playback while network bandwidth is limited. Uses browsermobproxy
+ (https://bmp.lightbody.net/). Please see
+ https://developer.mozilla.org/en-US/docs/Mozilla/QA/external-media-tests
+ for more information on setting up browsermob_proxy.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(NetworkBandwidthTestCase, self).__init__(*args, **kwargs)
+ BrowserMobProxyTestCaseMixin.__init__(self, *args, **kwargs)
+ self.proxy = None
+
+ def setUp(self):
+ super(NetworkBandwidthTestCase, self).setUp()
+ BrowserMobProxyTestCaseMixin.setUp(self)
+ self.proxy = self.create_browsermob_proxy()
+
+ def tearDown(self):
+ super(NetworkBandwidthTestCase, self).tearDown()
+ BrowserMobProxyTestCaseMixin.tearDown(self)
+ self.proxy = None
+
+ def run_videos(self, timeout=60):
+ """
+ Run each of the videos in the video list. Raises if something goes
+ wrong in playback.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ for url in self.video_urls:
+ video = VideoPuppeteer(self.marionette, url, stall_wait_time=60,
+ set_duration=60, timeout=timeout)
+ self.run_playback(video)
+
+
+class VideoPlaybackTestsMixin(object):
+
+ """
+ Test MSE playback in HTML5 video element.
+
+ These tests should pass on any site where a single video element plays
+ upon loading and is uninterrupted (by ads, for example).
+
+ This tests both starting videos and performing partial playback at one
+ minute each, and is the test that should be run frequently in automation.
+ """
+
+ def test_playback_starts(self):
+ """
+ Test to make sure that playback of the video element starts for each
+ video.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ for url in self.video_urls:
+ try:
+ video = VideoPuppeteer(self.marionette, url, timeout=60)
+ # Second playback_started check in case video._start_time
+ # is not 0
+ self.check_playback_starts(video)
+ video.pause()
+ except TimeoutException as e:
+ raise self.failureException(e)
+
+ def test_video_playback_partial(self):
+ """
+ Test to make sure that playback of 60 seconds works for each video.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ for url in self.video_urls:
+ video = VideoPuppeteer(self.marionette, url,
+ stall_wait_time=10,
+ set_duration=60)
+ self.run_playback(video)
+
+
+class NetworkBandwidthTestsMixin(object):
+
+ """
+ Test video urls with various bandwidth settings.
+ """
+
+ def test_playback_limiting_bandwidth_250(self):
+ self.proxy.limits({'downstream_kbps': 250})
+ self.run_videos(timeout=120)
+
+ def test_playback_limiting_bandwidth_500(self):
+ self.proxy.limits({'downstream_kbps': 500})
+ self.run_videos(timeout=120)
+
+ def test_playback_limiting_bandwidth_1000(self):
+ self.proxy.limits({'downstream_kbps': 1000})
+ self.run_videos(timeout=120)
+
+
+reset_adobe_gmp_script = """
+navigator.requestMediaKeySystemAccess('com.adobe.primetime',
+[{initDataTypes: ['cenc']}]).then(
+ function(access) {
+ marionetteScriptFinished('success');
+ },
+ function(ex) {
+ marionetteScriptFinished(ex);
+ }
+);
+"""
+
+
+reset_widevine_gmp_script = """
+navigator.requestMediaKeySystemAccess('com.widevine.alpha',
+[{initDataTypes: ['cenc']}]).then(
+ function(access) {
+ marionetteScriptFinished('success');
+ },
+ function(ex) {
+ marionetteScriptFinished(ex);
+ }
+);
+"""
+
+
+class EMESetupMixin(object):
+
+ """
+ An object that needs to use the Adobe or Widevine GMP system must inherit
+ from this class, and then call check_eme_system() to insure that everything
+ is setup correctly.
+ """
+
+ version_needs_reset = True
+
+ def check_eme_system(self):
+ """
+ Download the most current version of the Adobe and Widevine GMP
+ Plugins. Verify that all MSE and EME prefs are set correctly. Raises
+ if things are not OK.
+ """
+ self.set_eme_prefs()
+ self.reset_GMP_version()
+ assert(self.check_eme_prefs())
+
+ def set_eme_prefs(self):
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ # https://bugzilla.mozilla.org/show_bug.cgi?id=1187471#c28
+ # 2015-09-28 cpearce says this is no longer necessary, but in case
+ # we are working with older firefoxes...
+ self.marionette.set_pref('media.gmp.trial-create.enabled', False)
+
+ def reset_GMP_version(self):
+ if EMESetupMixin.version_needs_reset:
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ if self.marionette.get_pref('media.gmp-eme-adobe.version'):
+ self.marionette.reset_pref('media.gmp-eme-adobe.version')
+ if self.marionette.get_pref('media.gmp-widevinecdm.version'):
+ self.marionette.reset_pref('media.gmp-widevinecdm.version')
+ with self.marionette.using_context(Marionette.CONTEXT_CONTENT):
+ adobe_result = self.marionette.execute_async_script(
+ reset_adobe_gmp_script,
+ script_timeout=60000)
+ widevine_result = self.marionette.execute_async_script(
+ reset_widevine_gmp_script,
+ script_timeout=60000)
+ if not adobe_result == 'success':
+ raise VideoException(
+ 'ERROR: Resetting Adobe GMP failed {}'
+ .format(adobe_result))
+ if not widevine_result == 'success':
+ raise VideoException(
+ 'ERROR: Resetting Widevine GMP failed {}'
+ .format(widevine_result))
+
+ EMESetupMixin.version_needs_reset = False
+
+ def check_and_log_boolean_pref(self, pref_name, expected_value):
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ pref_value = self.marionette.get_pref(pref_name)
+
+ if pref_value is None:
+ self.logger.info('Pref {} has no value.'.format(pref_name))
+ return False
+ else:
+ self.logger.info('Pref {} = {}'.format(pref_name, pref_value))
+ if pref_value != expected_value:
+ self.logger.info('Pref {} has unexpected value.'
+ .format(pref_name))
+ return False
+
+ return True
+
+ def check_and_log_integer_pref(self, pref_name, minimum_value=0):
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ pref_value = self.marionette.get_pref(pref_name)
+
+ if pref_value is None:
+ self.logger.info('Pref {} has no value.'.format(pref_name))
+ return False
+ else:
+ self.logger.info('Pref {} = {}'.format(pref_name, pref_value))
+
+ match = re.search('^\d+$', pref_value)
+ if not match:
+ self.logger.info('Pref {} is not an integer'
+ .format(pref_name))
+ return False
+
+ return pref_value >= minimum_value
+
+ def chceck_and_log_version_string_pref(self, pref_name, minimum_value='0'):
+ """
+ Compare a pref made up of integers separated by stops .s, with a
+ version string of the same format. The number of integers in each
+ string does not need to match. The comparison is done by converting
+ each to an integer array and comparing those. Both version strings
+ must be made up of only integers, or this method will raise an
+ unhandled exception of type ValueError when the conversion to int
+ fails.
+ """
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ pref_value = self.marionette.get_pref(pref_name)
+
+ if pref_value is None:
+ self.logger.info('Pref {} has no value.'.format(pref_name))
+ return False
+ else:
+ self.logger.info('Pref {} = {}'.format(pref_name, pref_value))
+
+ match = re.search('^\d(.\d+)*$', pref_value)
+ if not match:
+ self.logger.info('Pref {} is not a version string'
+ .format(pref_name))
+ return False
+
+ pref_ints = [int(n) for n in pref_value.split('.')]
+ minumum_ints = [int(n) for n in minimum_value.split('.')]
+
+ return pref_ints >= minumum_ints
+
+ def check_eme_prefs(self):
+ with self.marionette.using_context(Marionette.CONTEXT_CHROME):
+ return all([
+ self.check_and_log_boolean_pref(
+ 'media.mediasource.enabled', True),
+ self.check_and_log_boolean_pref(
+ 'media.eme.enabled', True),
+ self.check_and_log_boolean_pref(
+ 'media.mediasource.mp4.enabled', True),
+ self.check_and_log_boolean_pref(
+ 'media.gmp-eme-adobe.enabled', True),
+ self.check_and_log_integer_pref(
+ 'media.gmp-eme-adobe.version', 1),
+ self.check_and_log_boolean_pref(
+ 'media.gmp-widevinecdm.enabled', True),
+ self.chceck_and_log_version_string_pref(
+ 'media.gmp-widevinecdm.version', '1.0.0.0')
+ ])