path: root/testing/mozharness/scripts/desktop_unittest.py

#!/usr/bin/env python
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
"""desktop_unittest.py
The goal of this is to extract desktop unittesting from buildbot's factory.py

author: Jordan Lund
"""

import os
import re
import sys
import copy
import shutil
import glob
import imp

# load modules from parent dir
sys.path.insert(1, os.path.dirname(sys.path[0]))

from mozharness.base.errors import BaseErrorList
from mozharness.base.log import INFO, ERROR
from mozharness.base.script import PreScriptAction
from mozharness.base.vcs.vcsbase import MercurialScript
from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
from mozharness.mozilla.buildbot import TBPL_EXCEPTION
from mozharness.mozilla.mozbase import MozbaseMixin
from mozharness.mozilla.structuredlog import StructuredOutputParser
from mozharness.mozilla.testing.errors import HarnessErrorList
from mozharness.mozilla.testing.unittest import DesktopUnittestOutputParser
from mozharness.mozilla.testing.codecoverage import (
    CodeCoverageMixin,
    code_coverage_config_options
)
from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options

SUITE_CATEGORIES = ['gtest', 'cppunittest', 'jittest', 'mochitest', 'reftest', 'xpcshell', 'mozbase', 'mozmill']
SUITE_DEFAULT_E10S = ['mochitest', 'reftest']


# DesktopUnittest {{{1
class DesktopUnittest(TestingMixin, MercurialScript, BlobUploadMixin, MozbaseMixin, CodeCoverageMixin):
    config_options = [
        [['--mochitest-suite', ], {
            "action": "extend",
            "dest": "specified_mochitest_suites",
            "type": "string",
            "help": "Specify which mochi suite to run. "
                    "Suites are defined in the config file.\n"
                    "Examples: 'all', 'plain1', 'plain5', 'chrome', or 'a11y'"}
         ],
        [['--reftest-suite', ], {
            "action": "extend",
            "dest": "specified_reftest_suites",
            "type": "string",
            "help": "Specify which reftest suite to run. "
                    "Suites are defined in the config file.\n"
                    "Examples: 'all', 'crashplan', or 'jsreftest'"}
         ],
        [['--xpcshell-suite', ], {
            "action": "extend",
            "dest": "specified_xpcshell_suites",
            "type": "string",
            "help": "Specify which xpcshell suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'xpcshell'"}
         ],
        [['--cppunittest-suite', ], {
            "action": "extend",
            "dest": "specified_cppunittest_suites",
            "type": "string",
            "help": "Specify which cpp unittest suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'cppunittest'"}
         ],
        [['--gtest-suite', ], {
            "action": "extend",
            "dest": "specified_gtest_suites",
            "type": "string",
            "help": "Specify which gtest suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'gtest'"}
         ],
        [['--jittest-suite', ], {
            "action": "extend",
            "dest": "specified_jittest_suites",
            "type": "string",
            "help": "Specify which jit-test suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'jittest'"}
         ],
        [['--mozbase-suite', ], {
            "action": "extend",
            "dest": "specified_mozbase_suites",
            "type": "string",
            "help": "Specify which mozbase suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'mozbase'"}
         ],
        [['--mozmill-suite', ], {
            "action": "extend",
            "dest": "specified_mozmill_suites",
            "type": "string",
            "help": "Specify which mozmill suite to run. "
                    "Suites are defined in the config file\n."
                    "Examples: 'mozmill'"}
         ],
        [['--run-all-suites', ], {
            "action": "store_true",
            "dest": "run_all_suites",
            "default": False,
            "help": "This will run all suites that are specified "
                    "in the config file. You do not need to specify "
                    "any other suites.\nBeware, this may take a while ;)"}
         ],
        [['--e10s', ], {
            "action": "store_true",
            "dest": "e10s",
            "default": False,
            "help": "Run tests with multiple processes."}
         ],
        [['--strict-content-sandbox', ], {
            "action": "store_true",
            "dest": "strict_content_sandbox",
            "default": False,
            "help": "Run tests with a more strict content sandbox (Windows only)."}
         ],
        [['--no-random', ], {
            "action": "store_true",
            "dest": "no_random",
            "default": False,
            "help": "Run tests with no random intermittents and bisect in case of real failure."}
         ],
        [["--total-chunks"], {
            "action": "store",
            "dest": "total_chunks",
            "help": "Number of total chunks"}
         ],
        [["--this-chunk"], {
            "action": "store",
            "dest": "this_chunk",
            "help": "Number of this chunk"}
         ],
        [["--allow-software-gl-layers"], {
            "action": "store_true",
            "dest": "allow_software_gl_layers",
            "default": False,
            "help": "Permits a software GL implementation (such as LLVMPipe) to use the GL compositor."}
         ],
    ] + copy.deepcopy(testing_config_options) + \
        copy.deepcopy(blobupload_config_options) + \
        copy.deepcopy(code_coverage_config_options)

    def __init__(self, require_config_file=True):
        # abs_dirs defined already in BaseScript but is here to make pylint happy
        self.abs_dirs = None
        super(DesktopUnittest, self).__init__(
            config_options=self.config_options,
            all_actions=[
                'clobber',
                'read-buildbot-config',
                'download-and-extract',
                'create-virtualenv',
                'install',
                'stage-files',
                'run-tests',
            ],
            require_config_file=require_config_file,
            config={'require_test_zip': True})

        c = self.config
        self.global_test_options = []
        self.installer_url = c.get('installer_url')
        self.test_url = c.get('test_url')
        self.test_packages_url = c.get('test_packages_url')
        self.symbols_url = c.get('symbols_url')
        # this is so mozinstall in install() doesn't bug out if we don't run
        # the download_and_extract action
        self.installer_path = c.get('installer_path')
        self.binary_path = c.get('binary_path')
        self.abs_app_dir = None
        self.abs_res_dir = None

        # Construct an identifier to be used to identify Perfherder data
        # for resource monitoring recording. This attempts to uniquely
        # identify this test invocation configuration.
        perfherder_parts = []
        perfherder_options = []
        suites = (
            ('specified_mochitest_suites', 'mochitest'),
            ('specified_reftest_suites', 'reftest'),
            ('specified_xpcshell_suites', 'xpcshell'),
            ('specified_cppunittest_suites', 'cppunit'),
            ('specified_gtest_suites', 'gtest'),
            ('specified_jittest_suites', 'jittest'),
            ('specified_mozbase_suites', 'mozbase'),
            ('specified_mozmill_suites', 'mozmill'),
        )
        for s, prefix in suites:
            if s in c:
                perfherder_parts.append(prefix)
                perfherder_parts.extend(c[s])

        if 'this_chunk' in c:
            perfherder_parts.append(c['this_chunk'])

        if c['e10s']:
            perfherder_options.append('e10s')

        self.resource_monitor_perfherder_id = ('.'.join(perfherder_parts),
                                               perfherder_options)

    # helper methods {{{2
    def _pre_config_lock(self, rw_config):
        super(DesktopUnittest, self)._pre_config_lock(rw_config)
        c = self.config
        if not c.get('run_all_suites'):
            return  # configs are valid
        for category in SUITE_CATEGORIES:
            specific_suites = c.get('specified_%s_suites' % (category))
            if specific_suites:
                if specific_suites != 'all':
                    self.fatal("Config options are not valid. Please ensure"
                               " that if the '--run-all-suites' flag was enabled,"
                               " then do not specify to run only specific suites "
                               "like:\n '--mochitest-suite browser-chrome'")

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(DesktopUnittest, self).query_abs_dirs()

        c = self.config
        dirs = {}
        dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
        dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
        dirs['abs_test_extensions_dir'] = os.path.join(dirs['abs_test_install_dir'], 'extensions')
        dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
        dirs['abs_test_bin_plugins_dir'] = os.path.join(dirs['abs_test_bin_dir'],
                                                        'plugins')
        dirs['abs_test_bin_components_dir'] = os.path.join(dirs['abs_test_bin_dir'],
                                                           'components')
        dirs['abs_mochitest_dir'] = os.path.join(dirs['abs_test_install_dir'], "mochitest")
        dirs['abs_reftest_dir'] = os.path.join(dirs['abs_test_install_dir'], "reftest")
        dirs['abs_xpcshell_dir'] = os.path.join(dirs['abs_test_install_dir'], "xpcshell")
        dirs['abs_cppunittest_dir'] = os.path.join(dirs['abs_test_install_dir'], "cppunittest")
        dirs['abs_gtest_dir'] = os.path.join(dirs['abs_test_install_dir'], "gtest")
        dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
        dirs['abs_jittest_dir'] = os.path.join(dirs['abs_test_install_dir'], "jit-test", "jit-test")
        dirs['abs_mozbase_dir'] = os.path.join(dirs['abs_test_install_dir'], "mozbase")
        dirs['abs_mozmill_dir'] = os.path.join(dirs['abs_test_install_dir'], "mozmill")

        if os.path.isabs(c['virtualenv_path']):
            dirs['abs_virtualenv_dir'] = c['virtualenv_path']
        else:
            dirs['abs_virtualenv_dir'] = os.path.join(abs_dirs['abs_work_dir'],
                                                      c['virtualenv_path'])
        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs

        return self.abs_dirs

    def query_abs_app_dir(self):
        """We can't set this in advance, because OSX install directories
        change depending on branding and opt/debug.
        """
        if self.abs_app_dir:
            return self.abs_app_dir
        if not self.binary_path:
            self.fatal("Can't determine abs_app_dir (binary_path not set!)")
        self.abs_app_dir = os.path.dirname(self.binary_path)
        return self.abs_app_dir

    def query_abs_res_dir(self):
        """The directory containing resources like plugins and extensions. On
        OSX this is Contents/Resources, on all other platforms its the same as
        the app dir.

        As with the app dir, we can't set this in advance, because OSX install
        directories change depending on branding and opt/debug.
        """
        if self.abs_res_dir:
            return self.abs_res_dir

        abs_app_dir = self.query_abs_app_dir()
        if self._is_darwin():
            res_subdir = self.config.get("mac_res_subdir", "Resources")
            self.abs_res_dir = os.path.join(os.path.dirname(abs_app_dir), res_subdir)
        else:
            self.abs_res_dir = abs_app_dir
        return self.abs_res_dir

    @PreScriptAction('create-virtualenv')
    def _pre_create_virtualenv(self, action):
        dirs = self.query_abs_dirs()

        self.register_virtualenv_module(name='pip>=1.5')
        self.register_virtualenv_module('psutil==3.1.1', method='pip')
        self.register_virtualenv_module(name='mock')
        self.register_virtualenv_module(name='simplejson')

        requirements_files = [
                os.path.join(dirs['abs_test_install_dir'],
                    'config',
                    'marionette_requirements.txt')]

        if os.path.isdir(dirs['abs_mochitest_dir']):
            # mochitest is the only thing that needs this
            requirements_files.append(
                os.path.join(dirs['abs_mochitest_dir'],
                             'websocketprocessbridge',
                             'websocketprocessbridge_requirements.txt'))

        for requirements_file in requirements_files:
            self.register_virtualenv_module(requirements=[requirements_file],
                                            two_pass=True)

    def _query_symbols_url(self):
        """query the full symbols URL based upon binary URL"""
        # may break with name convention changes but is one less 'input' for script
        if self.symbols_url:
            return self.symbols_url

        symbols_url = None
        self.info("finding symbols_url based upon self.installer_url")
        if self.installer_url:
            for ext in ['.zip', '.dmg', '.tar.bz2']:
                if ext in self.installer_url:
                    symbols_url = self.installer_url.replace(
                        ext, '.crashreporter-symbols.zip')
            if not symbols_url:
                self.fatal("self.installer_url was found but symbols_url could \
                        not be determined")
        else:
            self.fatal("self.installer_url was not found in self.config")
        self.info("setting symbols_url as %s" % (symbols_url))
        self.symbols_url = symbols_url
        return self.symbols_url

    def _query_abs_base_cmd(self, suite_category, suite):
        if self.binary_path:
            c = self.config
            dirs = self.query_abs_dirs()
            run_file = c['run_file_names'][suite_category]
            base_cmd = [self.query_python_path('python'), '-u']
            base_cmd.append(os.path.join(dirs["abs_%s_dir" % suite_category], run_file))
            abs_app_dir = self.query_abs_app_dir()
            abs_res_dir = self.query_abs_res_dir()

            raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
                                        '%s_raw.log' % suite)

            error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
                                              '%s_errorsummary.log' % suite)
            str_format_values = {
                'binary_path': self.binary_path,
                'symbols_path': self._query_symbols_url(),
                'abs_app_dir': abs_app_dir,
                'abs_res_dir': abs_res_dir,
                'raw_log_file': raw_log_file,
                'error_summary_file': error_summary_file,
                'gtest_dir': os.path.join(dirs['abs_test_install_dir'],
                                          'gtest'),
            }

            # TestingMixin._download_and_extract_symbols() will set
            # self.symbols_path when downloading/extracting.
            if self.symbols_path:
                str_format_values['symbols_path'] = self.symbols_path

            if suite_category in SUITE_DEFAULT_E10S and not c['e10s']:
                base_cmd.append('--disable-e10s')
            elif suite_category not in SUITE_DEFAULT_E10S and c['e10s']:
                base_cmd.append('--e10s')

            if c.get('strict_content_sandbox'):
                if suite_category == "mochitest":
                    base_cmd.append('--strict-content-sandbox')
                else:
                    self.fatal("--strict-content-sandbox only works with mochitest suites.")

            if c.get('total_chunks') and c.get('this_chunk'):
                base_cmd.extend(['--total-chunks', c['total_chunks'],
                                 '--this-chunk', c['this_chunk']])

            if c['no_random']:
                if suite_category == "mochitest":
                    base_cmd.append('--bisect-chunk=default')
                else:
                    self.warning("--no-random does not currently work with suites other than mochitest.")

            # set pluginsPath
            abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
            str_format_values['test_plugin_path'] = abs_res_plugins_dir

            if suite_category not in c["suite_definitions"]:
                self.fatal("'%s' not defined in the config!")

            if suite in ('browser-chrome-coverage', 'xpcshell-coverage', 'mochitest-devtools-chrome-coverage'):
                base_cmd.append('--jscov-dir-prefix=%s' %
                                dirs['abs_blob_upload_dir'])

            options = c["suite_definitions"][suite_category]["options"]
            if options:
                for option in options:
                    option = option % str_format_values
                    if not option.endswith('None'):
                        base_cmd.append(option)
                if self.structured_output(
                    suite_category,
                    self._query_try_flavor(suite_category, suite)
                ):
                    base_cmd.append("--log-raw=-")
                return base_cmd
            else:
                self.warning("Suite options for %s could not be determined."
                             "\nIf you meant to have options for this suite, "
                             "please make sure they are specified in your "
                             "config under %s_options" %
                             (suite_category, suite_category))

            return base_cmd
        else:
            self.fatal("'binary_path' could not be determined.\n This should "
                       "be like '/path/build/application/firefox/firefox'"
                       "\nIf you are running this script without the 'install' "
                       "action (where binary_path is set), please ensure you are"
                       " either:\n(1) specifying it in the config file under "
                       "binary_path\n(2) specifying it on command line with the"
                       " '--binary-path' flag")

    def _query_specified_suites(self, category):
        # logic goes: if at least one '--{category}-suite' was given,
        # then run only that(those) given suite(s). Elif no suites were
        # specified and the --run-all-suites flag was given,
        # run all {category} suites. Anything else, run no suites.
        c = self.config
        all_suites = c.get('all_%s_suites' % (category))
        specified_suites = c.get('specified_%s_suites' % (category))  # list
        suites = None

        if specified_suites:
            if 'all' in specified_suites:
                # useful if you want a quick way of saying run all suites
                # of a specific category.
                suites = all_suites
            else:
                # suites gets a dict of everything from all_suites where a key
                # is also in specified_suites
                suites = dict((key, all_suites.get(key)) for key in
                              specified_suites if key in all_suites.keys())
        else:
            if c.get('run_all_suites'):  # needed if you dont specify any suites
                suites = all_suites

        return suites

    def _query_try_flavor(self, category, suite):
        flavors = {
            "mochitest": [("plain.*", "mochitest"),
                          ("browser-chrome.*", "browser-chrome"),
                          ("mochitest-devtools-chrome.*", "devtools-chrome"),
                          ("chrome", "chrome"),
                          ("jetpack.*", "jetpack")],
            "xpcshell": [("xpcshell", "xpcshell")],
            "reftest": [("reftest", "reftest"),
                        ("crashtest", "crashtest")]
        }
        for suite_pattern, flavor in flavors.get(category, []):
            if re.compile(suite_pattern).match(suite):
                return flavor

    def structured_output(self, suite_category, flavor=None):
        unstructured_flavors = self.config.get('unstructured_flavors')
        if not unstructured_flavors:
            return False
        if suite_category not in unstructured_flavors:
            return True
        if not unstructured_flavors.get(suite_category) or flavor in unstructured_flavors.get(suite_category):
            return False
        return True

    def get_test_output_parser(self, suite_category, flavor=None, strict=False,
                               **kwargs):
        if not self.structured_output(suite_category, flavor):
            return DesktopUnittestOutputParser(suite_category=suite_category, **kwargs)
        self.info("Structured output parser in use for %s." % suite_category)
        return StructuredOutputParser(suite_category=suite_category, strict=strict, **kwargs)

    # Actions {{{2

    # clobber defined in BaseScript, deletes mozharness/build if exists
    # read_buildbot_config is in BuildbotMixin.
    # postflight_read_buildbot_config is in TestingMixin.
    # preflight_download_and_extract is in TestingMixin.
    # create_virtualenv is in VirtualenvMixin.
    # preflight_install is in TestingMixin.
    # install is in TestingMixin.
    # upload_blobber_files is in BlobUploadMixin

    @PreScriptAction('download-and-extract')
    def _pre_download_and_extract(self, action):
        """Abort if --artifact try syntax is used with compiled-code tests"""
        if not self.try_message_has_flag('artifact'):
            return
        self.info('Artifact build requested in try syntax.')
        rejected = []
        compiled_code_suites = [
            "cppunit",
            "gtest",
            "jittest",
        ]
        for category in SUITE_CATEGORIES:
            suites = self._query_specified_suites(category) or []
            for suite in suites:
                if any([suite.startswith(c) for c in compiled_code_suites]):
                    rejected.append(suite)
                    break
        if rejected:
            self.buildbot_status(TBPL_EXCEPTION)
            self.fatal("There are specified suites that are incompatible with "
                      "--artifact try syntax flag: {}".format(', '.join(rejected)),
                       exit_code=self.return_code)


    def download_and_extract(self):
        """
        download and extract test zip / download installer
        optimizes which subfolders to extract from tests zip
        """
        c = self.config

        extract_dirs = None
        if c['specific_tests_zip_dirs']:
            extract_dirs = list(c['minimum_tests_zip_dirs'])
            for category in c['specific_tests_zip_dirs'].keys():
                if c['run_all_suites'] or self._query_specified_suites(category) \
                        or 'run-tests' not in self.actions:
                    extract_dirs.extend(c['specific_tests_zip_dirs'][category])

        if c.get('run_all_suites'):
            target_categories = SUITE_CATEGORIES
        else:
            target_categories = [cat for cat in SUITE_CATEGORIES
                                 if self._query_specified_suites(cat) is not None]
        super(DesktopUnittest, self).download_and_extract(extract_dirs=extract_dirs,
                                                          suite_categories=target_categories)

    def stage_files(self):
        for category in SUITE_CATEGORIES:
            suites = self._query_specified_suites(category)
            stage = getattr(self, '_stage_{}'.format(category), None)
            if suites and stage:
                stage(suites)

    def _stage_files(self, bin_name=None):
        dirs = self.query_abs_dirs()
        abs_app_dir = self.query_abs_app_dir()

        # For mac these directories are in Contents/Resources, on other
        # platforms abs_res_dir will point to abs_app_dir.
        abs_res_dir = self.query_abs_res_dir()
        abs_res_components_dir = os.path.join(abs_res_dir, 'components')
        abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
        abs_res_extensions_dir = os.path.join(abs_res_dir, 'extensions')

        if bin_name:
            self.info('copying %s to %s' % (os.path.join(dirs['abs_test_bin_dir'],
                      bin_name), os.path.join(abs_app_dir, bin_name)))
            shutil.copy2(os.path.join(dirs['abs_test_bin_dir'], bin_name),
                         os.path.join(abs_app_dir, bin_name))

        self.copytree(dirs['abs_test_bin_components_dir'],
                      abs_res_components_dir,
                      overwrite='overwrite_if_exists')
        self.mkdir_p(abs_res_plugins_dir)
        self.copytree(dirs['abs_test_bin_plugins_dir'],
                      abs_res_plugins_dir,
                      overwrite='overwrite_if_exists')
        if os.path.isdir(dirs['abs_test_extensions_dir']):
            self.mkdir_p(abs_res_extensions_dir)
            self.copytree(dirs['abs_test_extensions_dir'],
                          abs_res_extensions_dir,
                          overwrite='overwrite_if_exists')

    def _stage_xpcshell(self, suites):
        self._stage_files(self.config['xpcshell_name'])

    def _stage_cppunittest(self, suites):
        abs_res_dir = self.query_abs_res_dir()
        dirs = self.query_abs_dirs()
        abs_cppunittest_dir = dirs['abs_cppunittest_dir']

        # move manifest and js fils to resources dir, where tests expect them
        files = glob.glob(os.path.join(abs_cppunittest_dir, '*.js'))
        files.extend(glob.glob(os.path.join(abs_cppunittest_dir, '*.manifest')))
        for f in files:
            self.move(f, abs_res_dir)

    def _stage_gtest(self, suites):
        abs_res_dir = self.query_abs_res_dir()
        abs_app_dir = self.query_abs_app_dir()
        dirs = self.query_abs_dirs()
        abs_gtest_dir = dirs['abs_gtest_dir']
        dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')

        files = glob.glob(os.path.join(dirs['abs_test_bin_plugins_dir'], 'gmp-*'))
        files.append(os.path.join(abs_gtest_dir, 'dependentlibs.list.gtest'))
        for f in files:
            self.move(f, abs_res_dir)

        self.copytree(os.path.join(abs_gtest_dir, 'gtest_bin'),
                      os.path.join(abs_app_dir))

    def _stage_mozmill(self, suites):
        self._stage_files()
        dirs = self.query_abs_dirs()
        modules = ['jsbridge', 'mozmill']
        for module in modules:
            self.install_module(module=os.path.join(dirs['abs_mozmill_dir'],
                                                    'resources',
                                                    module))

    # pull defined in VCSScript.
    # preflight_run_tests defined in TestingMixin.

    def run_tests(self):
        for category in SUITE_CATEGORIES:
            self._run_category_suites(category)

    def get_timeout_for_category(self, suite_category):
        if suite_category == 'cppunittest':
            return 2500
        return self.config["suite_definitions"][suite_category].get('run_timeout', 1000)

    def _run_category_suites(self, suite_category):
        """run suite(s) to a specific category"""
        dirs = self.query_abs_dirs()
        suites = self._query_specified_suites(suite_category)
        abs_app_dir = self.query_abs_app_dir()
        abs_res_dir = self.query_abs_res_dir()

        if suites:
            self.info('#### Running %s suites' % suite_category)
            for suite in suites:
                abs_base_cmd = self._query_abs_base_cmd(suite_category, suite)
                cmd = abs_base_cmd[:]
                replace_dict = {
                    'abs_app_dir': abs_app_dir,

                    # Mac specific, but points to abs_app_dir on other
                    # platforms.
                    'abs_res_dir': abs_res_dir,
                }
                options_list = []
                env = {}
                if isinstance(suites[suite], dict):
                    options_list = suites[suite].get('options', [])
                    tests_list = suites[suite].get('tests', [])
                    env = copy.deepcopy(suites[suite].get('env', {}))
                else:
                    options_list = suites[suite]
                    tests_list = []

                flavor = self._query_try_flavor(suite_category, suite)
                try_options, try_tests = self.try_args(flavor)

                cmd.extend(self.query_options(options_list,
                                              try_options,
                                              str_format_values=replace_dict))
                cmd.extend(self.query_tests_args(tests_list,
                                                 try_tests,
                                                 str_format_values=replace_dict))

                suite_name = suite_category + '-' + suite
                tbpl_status, log_level = None, None
                error_list = BaseErrorList + HarnessErrorList
                parser = self.get_test_output_parser(suite_category,
                                                     flavor=flavor,
                                                     config=self.config,
                                                     error_list=error_list,
                                                     log_obj=self.log_obj)

                if suite_category == "reftest":
                    ref_formatter = imp.load_source(
                        "ReftestFormatter",
                        os.path.abspath(
                            os.path.join(dirs["abs_reftest_dir"], "output.py")))
                    parser.formatter = ref_formatter.ReftestFormatter()

                if self.query_minidump_stackwalk():
                    env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path
                if self.query_nodejs():
                    env['MOZ_NODE_PATH'] = self.nodejs_path
                env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
                env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
                if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
                    self.mkdir_p(env['MOZ_UPLOAD_DIR'])

                if self.config['allow_software_gl_layers']:
                    env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1'

                env = self.query_env(partial_env=env, log_level=INFO)
                cmd_timeout = self.get_timeout_for_category(suite_category)
                return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'],
                                               output_timeout=cmd_timeout,
                                               output_parser=parser,
                                               env=env)

                # mochitest, reftest, and xpcshell suites do not return
                # appropriate return codes. Therefore, we must parse the output
                # to determine what the tbpl_status and worst_log_level must
                # be. We do this by:
                # 1) checking to see if our mozharness script ran into any
                #    errors itself with 'num_errors' <- OutputParser
                # 2) if num_errors is 0 then we look in the subclassed 'parser'
                #    findings for harness/suite errors <- DesktopUnittestOutputParser
                # 3) checking to see if the return code is in success_codes

                success_codes = None
                if self._is_windows() and suite_category != 'gtest':
                    # bug 1120644
                    success_codes = [0, 1]

                tbpl_status, log_level = parser.evaluate_parser(return_code,
                                                                success_codes=success_codes)
                parser.append_tinderboxprint_line(suite_name)

                self.buildbot_status(tbpl_status, level=log_level)
                self.log("The %s suite: %s ran with return status: %s" %
                         (suite_category, suite, tbpl_status), level=log_level)
        else:
            self.debug('There were no suites to run for %s' % suite_category)


# main {{{1
if __name__ == '__main__':
    desktop_unittest = DesktopUnittest()
    desktop_unittest.run_and_exit()