path: root/testing/mozharness/external_tools/clobberer.py

#!/usr/bin/python
# vim:sts=2 sw=2
import sys
import shutil
import urllib2
import urllib
import os
import traceback
import time
if os.name == 'nt':
    from win32file import RemoveDirectory, DeleteFile, \
        GetFileAttributesW, SetFileAttributesW, \
        FILE_ATTRIBUTE_NORMAL, FILE_ATTRIBUTE_DIRECTORY
    from win32api import FindFiles

clobber_suffix = '.deleteme'


def ts_to_str(ts):
    if ts is None:
        return None
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ts))


def write_file(ts, fn):
    assert isinstance(ts, int)
    f = open(fn, "w")
    f.write(str(ts))
    f.close()


def read_file(fn):
    if not os.path.exists(fn):
        return None

    data = open(fn).read().strip()
    try:
        return int(data)
    except ValueError:
        return None

def rmdirRecursiveWindows(dir):
    """Windows-specific version of rmdirRecursive that handles
    path lengths longer than MAX_PATH.
    """

    dir = os.path.realpath(dir)
    # Make sure directory is writable
    SetFileAttributesW('\\\\?\\' + dir, FILE_ATTRIBUTE_NORMAL)

    for ffrec in FindFiles('\\\\?\\' + dir + '\\*.*'):
        file_attr = ffrec[0]
        name = ffrec[8]
        if name == '.' or name == '..':
            continue
        full_name = os.path.join(dir, name)

        if file_attr & FILE_ATTRIBUTE_DIRECTORY:
            rmdirRecursiveWindows(full_name)
        else:
            SetFileAttributesW('\\\\?\\' + full_name, FILE_ATTRIBUTE_NORMAL)
            DeleteFile('\\\\?\\' + full_name)
    RemoveDirectory('\\\\?\\' + dir)

def rmdirRecursive(dir):
    """This is a replacement for shutil.rmtree that works better under
    windows. Thanks to Bear at the OSAF for the code.
    (Borrowed from buildbot.slave.commands)"""
    if os.name == 'nt':
        rmdirRecursiveWindows(dir)
        return

    if not os.path.exists(dir):
        # This handles broken links
        if os.path.islink(dir):
            os.remove(dir)
        return

    if os.path.islink(dir):
        os.remove(dir)
        return

    # Verify the directory is read/write/execute for the current user
    os.chmod(dir, 0700)

    for name in os.listdir(dir):
        full_name = os.path.join(dir, name)
        # on Windows, if we don't have write permission we can't remove
        # the file/directory either, so turn that on
        if os.name == 'nt':
            if not os.access(full_name, os.W_OK):
                # I think this is now redundant, but I don't have an NT
                # machine to test on, so I'm going to leave it in place
                # -warner
                os.chmod(full_name, 0600)

        if os.path.isdir(full_name):
            rmdirRecursive(full_name)
        else:
            # Don't try to chmod links
            if not os.path.islink(full_name):
                os.chmod(full_name, 0700)
            os.remove(full_name)
    os.rmdir(dir)


def do_clobber(dir, dryrun=False, skip=None):
    try:
        for f in os.listdir(dir):
            if skip is not None and f in skip:
                print "Skipping", f
                continue
            clobber_path = f + clobber_suffix
            if os.path.isfile(f):
                print "Removing", f
                if not dryrun:
                    if os.path.exists(clobber_path):
                        os.unlink(clobber_path)
                    # Prevent repeated moving.
                    if f.endswith(clobber_suffix):
                        os.unlink(f)
                    else:
                        shutil.move(f, clobber_path)
                        os.unlink(clobber_path)
            elif os.path.isdir(f):
                print "Removing %s/" % f
                if not dryrun:
                    if os.path.exists(clobber_path):
                        rmdirRecursive(clobber_path)
                    # Prevent repeated moving.
                    if f.endswith(clobber_suffix):
                        rmdirRecursive(f)
                    else:
                        shutil.move(f, clobber_path)
                        rmdirRecursive(clobber_path)
    except:
        print "Couldn't clobber properly, bailing out."
        sys.exit(1)


def getClobberDates(clobberURL, branch, buildername, builddir, slave, master):
    params = dict(branch=branch, buildername=buildername,
                  builddir=builddir, slave=slave, master=master)
    url = "%s?%s" % (clobberURL, urllib.urlencode(params))
    print "Checking clobber URL: %s" % url
    # The timeout arg was added to urlopen() at Python 2.6
    # Deprecate this test when esr17 reaches EOL
    if sys.version_info[:2] < (2, 6):
        data = urllib2.urlopen(url).read().strip()
    else:
        data = urllib2.urlopen(url, timeout=30).read().strip()

    retval = {}
    try:
        for line in data.split("\n"):
            line = line.strip()
            if not line:
                continue
            builddir, builder_time, who = line.split(":")
            builder_time = int(builder_time)
            retval[builddir] = (builder_time, who)
        return retval
    except ValueError:
        print "Error parsing response from server"
        print data
        raise

if __name__ == "__main__":
    from optparse import OptionParser
    parser = OptionParser(
        "%prog [options] clobberURL branch buildername builddir slave master")
    parser.add_option("-n", "--dry-run", dest="dryrun", action="store_true",
                      default=False, help="don't actually delete anything")
    parser.add_option("-t", "--periodic", dest="period", type="float",
                      default=None, help="hours between periodic clobbers")
    parser.add_option('-s', '--skip', help='do not delete this file/directory',
                      action='append', dest='skip', default=['last-clobber'])
    parser.add_option('-d', '--dir', help='clobber this directory',
                      dest='dir', default='.', type='string')
    parser.add_option('-v', '--verbose', help='be more verbose',
                      dest='verbose', action='store_true', default=False)

    options, args = parser.parse_args()
    if len(args) != 6:
        parser.error("Incorrect number of arguments")

    if options.period:
        periodicClobberTime = options.period * 3600
    else:
        periodicClobberTime = None

    clobberURL, branch, builder, my_builddir, slave, master = args

    try:
        server_clobber_dates = getClobberDates(
            clobberURL, branch, builder, my_builddir, slave, master)
    except:
        if options.verbose:
            traceback.print_exc()
        print "Error contacting server"
        sys.exit(1)

    if options.verbose:
        print "Server gave us", server_clobber_dates

    now = int(time.time())

    # Add ourself to the server_clobber_dates if it's not set
    # This happens when this slave has never been clobbered
    if my_builddir not in server_clobber_dates:
        server_clobber_dates[my_builddir] = None, ""

    root_dir = os.path.abspath(options.dir)

    for builddir, (server_clobber_date, who) in server_clobber_dates.items():
        builder_dir = os.path.join(root_dir, builddir)
        if not os.path.isdir(builder_dir):
            print "%s doesn't exist, skipping" % builder_dir
            continue
        os.chdir(builder_dir)

        our_clobber_date = read_file("last-clobber")

        clobber = False
        clobberType = None

        print "%s:Our last clobber date: " % builddir, ts_to_str(our_clobber_date)
        print "%s:Server clobber date:   " % builddir, ts_to_str(server_clobber_date)

        # If we don't have a last clobber date, then this is probably a fresh build.
        # We should only do a forced server clobber if we know when our last clobber
        # was, and if the server date is more recent than that.
        if server_clobber_date is not None and our_clobber_date is not None:
            # If the server is giving us a clobber date, compare the server's idea of
            # the clobber date to our last clobber date
            if server_clobber_date > our_clobber_date:
                # If the server's clobber date is greater than our last clobber date,
                # then we should clobber.
                clobber = True
                clobberType = "forced"
                # We should also update our clobber date to match the server's
                our_clobber_date = server_clobber_date
                if who:
                    print "%s:Server is forcing a clobber, initiated by %s" % (builddir, who)
                else:
                    print "%s:Server is forcing a clobber" % builddir

        if not clobber:
            # Disable periodic clobbers for builders that aren't my_builddir
            if builddir != my_builddir:
                continue

            # Next, check if more than the periodicClobberTime period has passed since
            # our last clobber
            if our_clobber_date is None:
                # We've never been clobbered
                # Set our last clobber time to now, so that we'll clobber
                # properly after periodicClobberTime
                clobberType = "purged"
                our_clobber_date = now
                write_file(our_clobber_date, "last-clobber")
            elif periodicClobberTime and now > our_clobber_date + periodicClobberTime:
                # periodicClobberTime has passed since our last clobber
                clobber = True
                clobberType = "periodic"
                # Update our clobber date to now
                our_clobber_date = now
                print "%s:More than %s seconds have passed since our last clobber" % (builddir, periodicClobberTime)

        if clobber:
            # Finally, perform a clobber if we're supposed to
            print "%s:Clobbering..." % builddir
            do_clobber(builder_dir, options.dryrun, options.skip)
            write_file(our_clobber_date, "last-clobber")

        # If this is the build dir for the current job, display the clobber type in TBPL.
        # Note in the case of purged clobber, we output the clobber type even though no
        # clobber was performed this time.
        if clobberType and builddir == my_builddir:
            print "TinderboxPrint: %s clobber" % clobberType