diff options
Diffstat (limited to 'build/valgrind')
| -rw-r--r-- | build/valgrind/__init__.py | 0 | ||||
| -rw-r--r-- | build/valgrind/cross-architecture.sup | 124 | ||||
| -rw-r--r-- | build/valgrind/i386-redhat-linux-gnu.sup | 63 | ||||
| -rw-r--r-- | build/valgrind/mach_commands.py | 175 | ||||
| -rw-r--r-- | build/valgrind/output_handler.py | 116 | ||||
| -rwxr-xr-x | build/valgrind/valgrind.sh | 45 | ||||
| -rw-r--r-- | build/valgrind/x86_64-redhat-linux-gnu.sup | 210 |
7 files changed, 733 insertions, 0 deletions
diff --git a/build/valgrind/__init__.py b/build/valgrind/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/build/valgrind/__init__.py diff --git a/build/valgrind/cross-architecture.sup b/build/valgrind/cross-architecture.sup new file mode 100644 index 000000000..1e9d7ab35 --- /dev/null +++ b/build/valgrind/cross-architecture.sup @@ -0,0 +1,124 @@ +# Full list is tracked through meta bug 793882 + +#################### +# Intended leaks # +#################### + +{ + PR_SetEnv requires its argument to be leaked, but does not appear on stacks. (See bug 793534.) + Memcheck:Leak + ... + fun:_ZL9SaveToEnvPKc + ... +} +{ + PR_SetEnv requires its argument to be leaked, but does not appear on stacks. (See bug 793549.) + Memcheck:Leak + ... + fun:_ZL13SaveWordToEnvPKcRK19nsACString_internal + ... +} +{ + PR_SetEnv requires its argument to be leaked, but does not appear on stacks. (See bug 944133.) + Memcheck:Leak + ... + fun:_ZN13CrashReporter14SetRestartArgsEiPPc + ... +} +{ + PR_SetEnv requires its argument to be leaked, but does not appear on stacks. (See bug 793548.) + Memcheck:Leak + fun:malloc + ... + fun:_Z12ToNewCStringRK19nsACString_internal + fun:_ZN13CrashReporter14SetupExtraDataEP7nsIFileRK19nsACString_internal + ... +} +{ + We purposely leak the StatisticsReporter object + Memcheck:Leak + fun:malloc + fun:moz_xmalloc + fun:operator new + fun:_Z21XRE_CreateStatsObjectv + ... +} + +#################################### +# Leaks in third party libraries # +#################################### + +{ + See bug 793535 + Memcheck:Leak + fun:calloc + fun:_dlerror_run + fun:dlsym + ... +} +{ + See bug 793611 + Memcheck:Leak + fun:memalign + fun:tls_get_addr_tail + fun:__tls_get_addr + ... +} +{ + See bug 793611 + Memcheck:Leak + fun:memalign + fun:tls_get_addr_tail + fun:___tls_get_addr + ... +} + +################# +# Other leaks # +################# + +{ + Bug 794369 + Memcheck:Leak + fun:malloc + fun:moz_xmalloc + ... + fun:_ZN20mozJSComponentLoader10LoadModuleERN7mozilla12FileLocationE + ... +} +{ + Bug 794370 + Memcheck:Leak + fun:malloc + fun:moz_xmalloc + ... + fun:_ZN22nsComponentManagerImpl15RegisterFactoryERK4nsIDPKcS4_P10nsIFactory + ... +} +{ + Bug 794372 + Memcheck:Leak + fun:malloc + fun:moz_xmalloc + ... + fun:_ZN22nsComponentManagerImpl22RegisterCIDEntryLockedEPKN7mozilla6Module8CIDEntryEPNS_11KnownModuleE + ... +} +{ + Bug 794374 + Memcheck:Leak + fun:malloc + fun:moz_xmalloc + ... + fun:_ZN22nsComponentManagerImpl17ManifestComponentERNS_25ManifestProcessingContextEiPKPc + ... +} +{ + Bug 1017112 + Memcheck:Leak + fun:malloc + ... + fun:PK11_InitPin + fun:_ZN11nsPK11Token12InitPasswordEPKDs + ... +} diff --git a/build/valgrind/i386-redhat-linux-gnu.sup b/build/valgrind/i386-redhat-linux-gnu.sup new file mode 100644 index 000000000..c1d39cd24 --- /dev/null +++ b/build/valgrind/i386-redhat-linux-gnu.sup @@ -0,0 +1,63 @@ +# Full list is tracked through meta bug 793882 + +#################################### +# Leaks in third party libraries # +#################################### + +{ + Bug 793537 + Memcheck:Leak + ... + obj:/usr/lib/libpango-1.0.so.0.2800.1 + ... +} +{ + Bug 793598 + Memcheck:Leak + ... + obj:/lib/libdbus-1.so.3.4.0 + ... +} +{ + Bug 793600 + Memcheck:Leak + fun:realloc + obj:/usr/lib/libfontconfig.so.1.4.4 + ... + fun:FcDefaultSubstitute + fun:_ZN17gfxPangoFontGroup11MakeFontSetEP14_PangoLanguagedP9nsAutoRefI10_FcPatternE + ... +} +{ + Bug 794366 + Memcheck:Leak + ... + obj:/usr/lib/libgtk-x11-2.0.so.0.1800.9 + ... +} +{ + Bug 794368 + Memcheck:Leak + ... + obj:/usr/lib/libXrandr.so.2.2.0 + ... +} +{ + Bug 794373 + Memcheck:Leak + ... + obj:/lib/libgobject-2.0.so.0.2200.5 + ... +} + +################################### +# Leaks in short lived precesses # +################################### + +{ + Bug 984196 + Memcheck:Leak + ... + fun:glxtest + ... +} diff --git a/build/valgrind/mach_commands.py b/build/valgrind/mach_commands.py new file mode 100644 index 000000000..ba2575247 --- /dev/null +++ b/build/valgrind/mach_commands.py @@ -0,0 +1,175 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import subprocess + +from mach.decorators import ( + Command, + CommandArgument, + CommandProvider, +) +from mozbuild.base import ( + MachCommandBase, + MachCommandConditions as conditions, +) + + +def is_valgrind_build(cls): + '''Must be a build with --enable-valgrind and --disable-jemalloc.''' + defines = cls.config_environment.defines + return 'MOZ_VALGRIND' in defines and 'MOZ_MEMORY' not in defines + + +@CommandProvider +class MachCommands(MachCommandBase): + ''' + Run Valgrind tests. + ''' + def __init__(self, context): + MachCommandBase.__init__(self, context) + + @Command('valgrind-test', category='testing', + conditions=[conditions.is_firefox, is_valgrind_build], + description='Run the Valgrind test job (memory-related errors).') + @CommandArgument('--suppressions', default=[], action='append', + metavar='FILENAME', + help='Specify a suppression file for Valgrind to use. Use ' + '--suppression multiple times to specify multiple suppression ' + 'files.') + def valgrind_test(self, suppressions): + import json + import sys + import tempfile + + from mozbuild.base import MozbuildObject + from mozfile import TemporaryDirectory + from mozhttpd import MozHttpd + from mozprofile import FirefoxProfile, Preferences + from mozprofile.permissions import ServerLocations + from mozrunner import FirefoxRunner + from mozrunner.utils import findInPath + from valgrind.output_handler import OutputHandler + + build_dir = os.path.join(self.topsrcdir, 'build') + + # XXX: currently we just use the PGO inputs for Valgrind runs. This may + # change in the future. + httpd = MozHttpd(docroot=os.path.join(build_dir, 'pgo')) + httpd.start(block=False) + + with TemporaryDirectory() as profilePath: + #TODO: refactor this into mozprofile + prefpath = os.path.join(self.topsrcdir, 'testing', 'profiles', 'prefs_general.js') + prefs = {} + prefs.update(Preferences.read_prefs(prefpath)) + interpolation = { 'server': '%s:%d' % httpd.httpd.server_address, + 'OOP': 'false'} + prefs = json.loads(json.dumps(prefs) % interpolation) + for pref in prefs: + prefs[pref] = Preferences.cast(prefs[pref]) + + quitter = os.path.join(self.topsrcdir, 'tools', 'quitter', 'quitter@mozilla.org.xpi') + + locations = ServerLocations() + locations.add_host(host='127.0.0.1', + port=httpd.httpd.server_port, + options='primary') + + profile = FirefoxProfile(profile=profilePath, + preferences=prefs, + addons=[quitter], + locations=locations) + + firefox_args = [httpd.get_url()] + + env = os.environ.copy() + env['G_SLICE'] = 'always-malloc' + env['MOZ_CC_RUN_DURING_SHUTDOWN'] = '1' + env['MOZ_CRASHREPORTER_NO_REPORT'] = '1' + env['XPCOM_DEBUG_BREAK'] = 'warn' + + env.update(self.extra_environment_variables) + + outputHandler = OutputHandler(self.log) + kp_kwargs = {'processOutputLine': [outputHandler]} + + valgrind = 'valgrind' + if not os.path.exists(valgrind): + valgrind = findInPath(valgrind) + + valgrind_args = [ + valgrind, + '--smc-check=all-non-file', + '--vex-iropt-register-updates=allregs-at-mem-access', + '--gen-suppressions=all', + '--num-callers=36', + '--leak-check=full', + '--show-possibly-lost=no', + '--track-origins=yes', + '--trace-children=yes', + '-v', # Enable verbosity to get the list of used suppressions + # Avoid excessive delays in the presence of spinlocks. + # See bug 1309851. + '--fair-sched=yes', + ] + + for s in suppressions: + valgrind_args.append('--suppressions=' + s) + + supps_dir = os.path.join(build_dir, 'valgrind') + supps_file1 = os.path.join(supps_dir, 'cross-architecture.sup') + valgrind_args.append('--suppressions=' + supps_file1) + + # MACHTYPE is an odd bash-only environment variable that doesn't + # show up in os.environ, so we have to get it another way. + machtype = subprocess.check_output(['bash', '-c', 'echo $MACHTYPE']).rstrip() + supps_file2 = os.path.join(supps_dir, machtype + '.sup') + if os.path.isfile(supps_file2): + valgrind_args.append('--suppressions=' + supps_file2) + + exitcode = None + timeout = 1800 + try: + runner = FirefoxRunner(profile=profile, + binary=self.get_binary_path(), + cmdargs=firefox_args, + env=env, + process_args=kp_kwargs) + runner.start(debug_args=valgrind_args) + exitcode = runner.wait(timeout=timeout) + + finally: + errs = outputHandler.error_count + supps = outputHandler.suppression_count + if errs != supps: + status = 1 # turns the TBPL job orange + self.log(logging.ERROR, 'valgrind-fail-parsing', + {'errs': errs, 'supps': supps}, + 'TEST-UNEXPECTED-FAIL | valgrind-test | error parsing: {errs} errors seen, but {supps} generated suppressions seen') + + elif errs == 0: + status = 0 + self.log(logging.INFO, 'valgrind-pass', {}, + 'TEST-PASS | valgrind-test | valgrind found no errors') + else: + status = 1 # turns the TBPL job orange + # We've already printed details of the errors. + + if exitcode == None: + status = 2 # turns the TBPL job red + self.log(logging.ERROR, 'valgrind-fail-timeout', + {'timeout': timeout}, + 'TEST-UNEXPECTED-FAIL | valgrind-test | Valgrind timed out (reached {timeout} second limit)') + elif exitcode != 0: + status = 2 # turns the TBPL job red + self.log(logging.ERROR, 'valgrind-fail-errors', {}, + 'TEST-UNEXPECTED-FAIL | valgrind-test | non-zero exit code from Valgrind') + + httpd.stop() + + return status diff --git a/build/valgrind/output_handler.py b/build/valgrind/output_handler.py new file mode 100644 index 000000000..df24ff28b --- /dev/null +++ b/build/valgrind/output_handler.py @@ -0,0 +1,116 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +import logging +import re + +class OutputHandler(object): + ''' + A class for handling Valgrind output. + + Valgrind errors look like this: + + ==60741== 40 (24 direct, 16 indirect) bytes in 1 blocks are definitely lost in loss record 2,746 of 5,235 + ==60741== at 0x4C26B43: calloc (vg_replace_malloc.c:593) + ==60741== by 0x63AEF65: PR_Calloc (prmem.c:443) + ==60741== by 0x69F236E: PORT_ZAlloc_Util (secport.c:117) + ==60741== by 0x69F1336: SECITEM_AllocItem_Util (secitem.c:28) + ==60741== by 0xA04280B: ffi_call_unix64 (in /builds/slave/m-in-l64-valgrind-000000000000/objdir/toolkit/library/libxul.so) + ==60741== by 0xA042443: ffi_call (ffi64.c:485) + + For each such error, this class extracts most or all of the first (error + kind) line, plus the function name in each of the first few stack entries. + With this data it constructs and prints a TEST-UNEXPECTED-FAIL message that + TBPL will highlight. + + It buffers these lines from which text is extracted so that the + TEST-UNEXPECTED-FAIL message can be printed before the full error. + + Parsing the Valgrind output isn't ideal, and it may break in the future if + Valgrind changes the format of the messages, or introduces new error kinds. + To protect against this, we also count how many lines containing + "<insert_a_suppression_name_here>" are seen. Thanks to the use of + --gen-suppressions=yes, exactly one of these lines is present per error. If + the count of these lines doesn't match the error count found during + parsing, then the parsing has missed one or more errors and we can fail + appropriately. + ''' + + def __init__(self, logger): + # The regexps in this list match all of Valgrind's errors. Note that + # Valgrind is English-only, so we don't have to worry about + # localization. + self.logger = logger + self.re_error = \ + r'==\d+== (' + \ + r'(Use of uninitialised value of size \d+)|' + \ + r'(Conditional jump or move depends on uninitialised value\(s\))|' + \ + r'(Syscall param .* contains uninitialised byte\(s\))|' + \ + r'(Syscall param .* points to (unaddressable|uninitialised) byte\(s\))|' + \ + r'((Unaddressable|Uninitialised) byte\(s\) found during client check request)|' + \ + r'(Invalid free\(\) / delete / delete\[\] / realloc\(\))|' + \ + r'(Mismatched free\(\) / delete / delete \[\])|' + \ + r'(Invalid (read|write) of size \d+)|' + \ + r'(Jump to the invalid address stated on the next line)|' + \ + r'(Source and destination overlap in .*)|' + \ + r'(.* bytes in .* blocks are .* lost)' + \ + r')' + # Match identifer chars, plus ':' for namespaces, and '\?' in order to + # match "???" which Valgrind sometimes produces. + self.re_stack_entry = r'^==\d+==.*0x[A-Z0-9]+: ([A-Za-z0-9_:\?]+)' + self.re_suppression = r' *<insert_a_suppression_name_here>' + self.error_count = 0 + self.suppression_count = 0 + self.number_of_stack_entries_to_get = 0 + self.curr_error = None + self.curr_location = None + self.buffered_lines = None + + def log(self, line): + self.logger(logging.INFO, 'valgrind-output', {'line': line}, '{line}') + + def __call__(self, line): + if self.number_of_stack_entries_to_get == 0: + # Look for the start of a Valgrind error. + m = re.search(self.re_error, line) + if m: + self.error_count += 1 + self.number_of_stack_entries_to_get = 4 + self.curr_error = m.group(1) + self.curr_location = "" + self.buffered_lines = [line] + else: + self.log(line) + + else: + # We've recently found a Valgrind error, and are now extracting + # details from the first few stack entries. + self.buffered_lines.append(line) + m = re.match(self.re_stack_entry, line) + if m: + self.curr_location += m.group(1) + else: + self.curr_location += '?!?' + + self.number_of_stack_entries_to_get -= 1 + if self.number_of_stack_entries_to_get != 0: + self.curr_location += ' / ' + else: + # We've finished getting the first few stack entries. Print the + # failure message and the buffered lines, and then reset state. + self.logger(logging.ERROR, 'valgrind-error-msg', + {'error': self.curr_error, + 'location': self.curr_location}, + 'TEST-UNEXPECTED-FAIL | valgrind-test | {error} at {location}') + for b in self.buffered_lines: + self.log(b) + self.curr_error = None + self.curr_location = None + self.buffered_lines = None + + if re.match(self.re_suppression, line): + self.suppression_count += 1 + diff --git a/build/valgrind/valgrind.sh b/build/valgrind/valgrind.sh new file mode 100755 index 000000000..c46f18876 --- /dev/null +++ b/build/valgrind/valgrind.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e +set -x + +srcdir=$PWD/src +objdir=${MOZ_OBJDIR-objdir} + +# If the objdir is a relative path, it is relative to the srcdir. +case "$objdir" in + /*) + ;; + *) + objdir="$srcdir/$objdir" + ;; +esac + +if [ ! -d $objdir ]; then + mkdir $objdir +fi +cd $objdir + +if [ "`uname -m`" = "x86_64" ]; then + _arch=64 +else + _arch=32 +fi + +TOOLTOOL_MANIFEST=browser/config/tooltool-manifests/linux${_arch}/releng.manifest +TOOLTOOL_SERVER=https://api.pub.build.mozilla.org/tooltool/ +(cd $srcdir; python /builds/tooltool.py --url $TOOLTOOL_SERVER --overwrite -m $TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c ${TOOLTOOL_CACHE}}) || exit 2 + +# Note: an exit code of 2 turns the job red on TBPL. +MOZCONFIG=$srcdir/browser/config/mozconfigs/linux${_arch}/valgrind make -f $srcdir/client.mk configure || exit 2 +make -j4 || exit 2 +make package || exit 2 + +# We need to set MOZBUILD_STATE_PATH so that |mach| skips its first-run +# initialization step and actually runs the |valgrind-test| command. +export MOZBUILD_STATE_PATH=. + +# |mach valgrind-test|'s exit code will be 1 (which turns the job orange on +# TBPL) if Valgrind finds errors, and 2 (which turns the job red) if something +# else goes wrong, such as Valgrind crashing. +python2.7 $srcdir/mach valgrind-test +exit $? diff --git a/build/valgrind/x86_64-redhat-linux-gnu.sup b/build/valgrind/x86_64-redhat-linux-gnu.sup new file mode 100644 index 000000000..d253a3a4e --- /dev/null +++ b/build/valgrind/x86_64-redhat-linux-gnu.sup @@ -0,0 +1,210 @@ +# Full list is tracked through meta bug 793882 + +#################################### +# Leaks in third party libraries # +#################################### + +{ + Bug 793537 + Memcheck:Leak + ... + obj:/usr/lib64/libpango-1.0.so.0.2800.1 + ... +} +{ + Bug 793598 + Memcheck:Leak + ... + obj:/lib64/libdbus-1.so.3.4.0 + ... +} +{ + Bug 793600 + Memcheck:Leak + fun:realloc + obj:/usr/lib64/libfontconfig.so.1.4.4 + ... + fun:FcDefaultSubstitute + fun:_ZN17gfxPangoFontGroup11MakeFontSetEP14_PangoLanguagedP9nsAutoRefI10_FcPatternE + ... +} +# Fontconfig is going fancy with its cache structure and that confuses valgrind. +# https://bugs.freedesktop.org/show_bug.cgi?id=8215 +# https://bugs.freedesktop.org/show_bug.cgi?id=8428 +{ + Bug 1187649 + Memcheck:Leak + match-leak-kinds: definite + fun:realloc + fun:FcPatternObjectInsertElt + ... +} +# With older versions of fontconfig (e.g. 2.8.0 on taskcluster systems), +# there's an uninitialized memory usage and leak when loading app fonts. +{ + Bug 1231701 + Memcheck:Param + write(buf) + ... + fun:FcDirCacheWrite + fun:FcDirCacheScan + fun:FcConfigAddDirList + fun:FcConfigAppFontAddDir + ... +} +{ + Bug 1231701 + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:FcDirScanConfig + fun:FcDirCacheScan + fun:FcConfigAddDirList + fun:FcConfigAppFontAddDir + ... +} +# Leaks due to either Gtk+3 or cairo, but Gecko is not directly involved with +# those cairo interactions. One suspected cause is Gecko not closing the +# display to work around a bug in old Gtk+3 versions. See also bug 1228724. +{ + Bug 1187649 + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:_cairo_freelist_alloc + fun:_cairo_xlib_display_queue_resource + fun:_cairo_xlib_surface_finish + ... +} +# The following leak is deep in Gtk+3, and it doesn't seem we're doing +# anything wrong on our end with the container objects. This suppression +# is purposefully verbose so as to avoid catching actual leaks due to +# Gecko code. +# Note: valgrind doesn't support more than 24 elements in a suppression stack. +{ + Bug 1187649 + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_slice_alloc + fun:g_list_prepend + fun:gtk_combo_box_get_path_for_child + fun:gtk_container_get_path_for_child + fun:gtk_widget_get_path + fun:_gtk_widget_update_path + fun:reset_style_recurse + fun:gtk_widget_reset_style + fun:gtk_widget_set_parent + fun:gtk_combo_box_add + fun:g_cclosure_marshal_VOID__OBJECTv + fun:_g_closure_invoke_va + fun:g_signal_emit_valist + fun:g_signal_emit + fun:gtk_combo_box_constructor + fun:g_object_newv + fun:g_object_new_valist + fun:g_object_new + fun:_ZN13nsLookAndFeel4InitEv + ... +} +# set_color() in gtkstyle.c of GTK version 3.4.4 only can leak GdkRGBA +# allocations when the theme has transparent colors: +# https://git.gnome.org/browse/gtk+/tree/gtk/deprecated/gtkstyle.c?h=3.4.4#n676 +{ + Bug 1250704 + Memcheck:Leak + match-leak-kinds: definite + fun:malloc + fun:g_malloc + fun:g_slice_alloc + fun:g_slice_copy + fun:boxed_proxy_lcopy_value + fun:gtk_style_context_get_valist + fun:gtk_style_context_get + fun:set_color + fun:gtk_style_update_from_context + fun:gtk_style_constructed + fun:g_object_newv + fun:g_object_new_valist + fun:g_object_new + ... +} +{ + Bug 794366 + Memcheck:Leak + ... + obj:/usr/lib64/libgtk-x11-2.0.so.0.1800.9 + ... +} +{ + Bug 794368 + Memcheck:Leak + ... + obj:/usr/lib64/libXrandr.so.2.2.0 + ... +} +{ + Bug 794373 + Memcheck:Leak + ... + obj:/lib64/libgobject-2.0.so.0.2200.5 + ... +} +{ + Bug 966673 + Memcheck:Leak + fun:malloc + obj:/lib64/libresolv-2.12.so + ... + fun:gaih_inet + fun:getaddrinfo + fun:PR_GetAddrInfoByName + ... +} +{ + Bug 979242 + Memcheck:Leak + fun:calloc + fun:xcb_connect_to_fd + fun:xcb_connect_to_display_with_auth_info + fun:_XConnectXCB + fun:XOpenDisplay + fun:gdk_display_open + ... +} + +################################### +# Leaks in short lived precesses # +################################### + +{ + Bug 984196 + Memcheck:Leak + ... + fun:glxtest + ... +} + +######################################### +# Uninitialised value false positives # +######################################### + +# This concerns a false positive pertaining to Memcheck's overly- +# conservative instrumentation of CPUID. See bug 1288618 comments +# 119 through 127. +{ + Bug 1288618 comments 119 through 127 + Memcheck:Cond + fun:_ZN6SkOptsL4initEv + fun:sk_once_no_arg_adaptor +} + +{ + Bug 1288618 comments 119 through 127 part 2 + Memcheck:Cond + fun:__get_cpuid + fun:cpuid + fun:_ZN6SkOptsL4initEv + fun:sk_once_no_arg_adaptor +} |