diff options
Diffstat (limited to 'testing/mozharness/external_tools/clobberer.py')
| -rwxr-xr-x | testing/mozharness/external_tools/clobberer.py | 280 |
1 files changed, 280 insertions, 0 deletions
diff --git a/testing/mozharness/external_tools/clobberer.py b/testing/mozharness/external_tools/clobberer.py new file mode 100755 index 000000000..a58b00402 --- /dev/null +++ b/testing/mozharness/external_tools/clobberer.py @@ -0,0 +1,280 @@ +#!/usr/bin/python +# vim:sts=2 sw=2 +import sys +import shutil +import urllib2 +import urllib +import os +import traceback +import time +if os.name == 'nt': + from win32file import RemoveDirectory, DeleteFile, \ + GetFileAttributesW, SetFileAttributesW, \ + FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY + from win32api import FindFiles + +clobber_suffix = '.deleteme' + + +def ts_to_str(ts): + if ts is None: + return None + return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts)) + + +def write_file(ts, fn): + assert isinstance(ts, int) + f = open(fn, "w") + f.write(str(ts)) + f.close() + + +def read_file(fn): + if not os.path.exists(fn): + return None + + data = open(fn).read().strip() + try: + return int(data) + except ValueError: + return None + +def rmdirRecursiveWindows(dir): + """Windows-specific version of rmdirRecursive that handles + path lengths longer than MAX_PATH. + """ + + dir = os.path.realpath(dir) + # Make sure directory is writable + SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL) + + for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'): + file_attr = ffrec[0] + name = ffrec[8] + if name == '.' or name == '..': + continue + full_name = os.path.join(dir, name) + + if file_attr & FILE_ATTRIBUTE_DIRECTORY: + rmdirRecursiveWindows(full_name) + else: + SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL) + DeleteFile('\\\\?\\' + full_name) + RemoveDirectory('\\\\?\\' + dir) + +def rmdirRecursive(dir): + """This is a replacement for shutil.rmtree that works better under + windows. Thanks to Bear at the OSAF for the code. + (Borrowed from buildbot.slave.commands)""" + if os.name == 'nt': + rmdirRecursiveWindows(dir) + return + + if not os.path.exists(dir): + # This handles broken links + if os.path.islink(dir): + os.remove(dir) + return + + if os.path.islink(dir): + os.remove(dir) + return + + # Verify the directory is read/write/execute for the current user + os.chmod(dir, 0700) + + for name in os.listdir(dir): + full_name = os.path.join(dir, name) + # on Windows, if we don't have write permission we can't remove + # the file/directory either, so turn that on + if os.name == 'nt': + if not os.access(full_name, os.W_OK): + # I think this is now redundant, but I don't have an NT + # machine to test on, so I'm going to leave it in place + # -warner + os.chmod(full_name, 0600) + + if os.path.isdir(full_name): + rmdirRecursive(full_name) + else: + # Don't try to chmod links + if not os.path.islink(full_name): + os.chmod(full_name, 0700) + os.remove(full_name) + os.rmdir(dir) + + +def do_clobber(dir, dryrun=False, skip=None): + try: + for f in os.listdir(dir): + if skip is not None and f in skip: + print "Skipping", f + continue + clobber_path = f + clobber_suffix + if os.path.isfile(f): + print "Removing", f + if not dryrun: + if os.path.exists(clobber_path): + os.unlink(clobber_path) + # Prevent repeated moving. + if f.endswith(clobber_suffix): + os.unlink(f) + else: + shutil.move(f, clobber_path) + os.unlink(clobber_path) + elif os.path.isdir(f): + print "Removing %s/" % f + if not dryrun: + if os.path.exists(clobber_path): + rmdirRecursive(clobber_path) + # Prevent repeated moving. + if f.endswith(clobber_suffix): + rmdirRecursive(f) + else: + shutil.move(f, clobber_path) + rmdirRecursive(clobber_path) + except: + print "Couldn't clobber properly, bailing out." + sys.exit(1) + + +def getClobberDates(clobberURL, branch, buildername, builddir, slave, master): + params = dict(branch=branch, buildername=buildername, + builddir=builddir, slave=slave, master=master) + url = "%s?%s" % (clobberURL, urllib.urlencode(params)) + print "Checking clobber URL: %s" % url + # The timeout arg was added to urlopen() at Python 2.6 + # Deprecate this test when esr17 reaches EOL + if sys.version_info[:2] < (2, 6): + data = urllib2.urlopen(url).read().strip() + else: + data = urllib2.urlopen(url, timeout=30).read().strip() + + retval = {} + try: + for line in data.split("\n"): + line = line.strip() + if not line: + continue + builddir, builder_time, who = line.split(":") + builder_time = int(builder_time) + retval[builddir] = (builder_time, who) + return retval + except ValueError: + print "Error parsing response from server" + print data + raise + +if __name__ == "__main__": + from optparse import OptionParser + parser = OptionParser( + "%prog [options] clobberURL branch buildername builddir slave master") + parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true", + default=False, help="don't actually delete anything") + parser.add_option("-t", "--periodic", dest="period", type="float", + default=None, help="hours between periodic clobbers") + parser.add_option('-s', '--skip', help='do not delete this file/directory', + action='append', dest='skip', default=['last-clobber']) + parser.add_option('-d', '--dir', help='clobber this directory', + dest='dir', default='.', type='string') + parser.add_option('-v', '--verbose', help='be more verbose', + dest='verbose', action='store_true', default=False) + + options, args = parser.parse_args() + if len(args) != 6: + parser.error("Incorrect number of arguments") + + if options.period: + periodicClobberTime = options.period * 3600 + else: + periodicClobberTime = None + + clobberURL, branch, builder, my_builddir, slave, master = args + + try: + server_clobber_dates = getClobberDates( + clobberURL, branch, builder, my_builddir, slave, master) + except: + if options.verbose: + traceback.print_exc() + print "Error contacting server" + sys.exit(1) + + if options.verbose: + print "Server gave us", server_clobber_dates + + now = int(time.time()) + + # Add ourself to the server_clobber_dates if it's not set + # This happens when this slave has never been clobbered + if my_builddir not in server_clobber_dates: + server_clobber_dates[my_builddir] = None, "" + + root_dir = os.path.abspath(options.dir) + + for builddir, (server_clobber_date, who) in server_clobber_dates.items(): + builder_dir = os.path.join(root_dir, builddir) + if not os.path.isdir(builder_dir): + print "%s doesn't exist, skipping" % builder_dir + continue + os.chdir(builder_dir) + + our_clobber_date = read_file("last-clobber") + + clobber = False + clobberType = None + + print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date) + print "%s:Server clobber date: " % builddir, ts_to_str(server_clobber_date) + + # If we don't have a last clobber date, then this is probably a fresh build. + # We should only do a forced server clobber if we know when our last clobber + # was, and if the server date is more recent than that. + if server_clobber_date is not None and our_clobber_date is not None: + # If the server is giving us a clobber date, compare the server's idea of + # the clobber date to our last clobber date + if server_clobber_date > our_clobber_date: + # If the server's clobber date is greater than our last clobber date, + # then we should clobber. + clobber = True + clobberType = "forced" + # We should also update our clobber date to match the server's + our_clobber_date = server_clobber_date + if who: + print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who) + else: + print "%s:Server is forcing a clobber" % builddir + + if not clobber: + # Disable periodic clobbers for builders that aren't my_builddir + if builddir != my_builddir: + continue + + # Next, check if more than the periodicClobberTime period has passed since + # our last clobber + if our_clobber_date is None: + # We've never been clobbered + # Set our last clobber time to now, so that we'll clobber + # properly after periodicClobberTime + clobberType = "purged" + our_clobber_date = now + write_file(our_clobber_date, "last-clobber") + elif periodicClobberTime and now > our_clobber_date + periodicClobberTime: + # periodicClobberTime has passed since our last clobber + clobber = True + clobberType = "periodic" + # Update our clobber date to now + our_clobber_date = now + print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime) + + if clobber: + # Finally, perform a clobber if we're supposed to + print "%s:Clobbering..." % builddir + do_clobber(builder_dir, options.dryrun, options.skip) + write_file(our_clobber_date, "last-clobber") + + # If this is the build dir for the current job, display the clobber type in TBPL. + # Note in the case of purged clobber, we output the clobber type even though no + # clobber was performed this time. + if clobberType and builddir == my_builddir: + print "TinderboxPrint: %s clobber" % clobberType |