diff options
Diffstat (limited to 'testing/web-platform/harness/wptrunner/update')
7 files changed, 1060 insertions, 0 deletions
diff --git a/testing/web-platform/harness/wptrunner/update/__init__.py b/testing/web-platform/harness/wptrunner/update/__init__.py new file mode 100644 index 000000000..84c19fd94 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/__init__.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys + +from mozlog.structured import structuredlog, commandline + +from .. import wptcommandline + +from update import WPTUpdate + +def remove_logging_args(args): + """Take logging args out of the dictionary of command line arguments so + they are not passed in as kwargs to the update code. This is particularly + necessary here because the arguments are often of type file, which cannot + be serialized. + + :param args: Dictionary of command line arguments. + """ + for name in args.keys(): + if name.startswith("log_"): + args.pop(name) + + +def setup_logging(args, defaults): + """Use the command line arguments to set up the logger. + + :param args: Dictionary of command line arguments. + :param defaults: Dictionary of {formatter_name: stream} to use if + no command line logging is specified""" + logger = commandline.setup_logging("web-platform-tests-update", args, defaults) + + remove_logging_args(args) + + return logger + + +def run_update(logger, **kwargs): + updater = WPTUpdate(logger, **kwargs) + return updater.run() + + +def main(): + args = wptcommandline.parse_args_update() + logger = setup_logging(args, {"mach": sys.stdout}) + assert structuredlog.get_default_logger() is not None + success = run_update(logger, **args) + sys.exit(0 if success else 1) + diff --git a/testing/web-platform/harness/wptrunner/update/base.py b/testing/web-platform/harness/wptrunner/update/base.py new file mode 100644 index 000000000..981ae2b70 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/base.py @@ -0,0 +1,69 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +exit_unclean = object() +exit_clean = object() + + +class Step(object): + provides = [] + + def __init__(self, logger): + self.logger = logger + + def run(self, step_index, state): + """Base class for state-creating steps. + + When a Step is run() the current state is checked to see + if the state from this step has already been created. If it + has the restore() method is invoked. Otherwise the create() + method is invoked with the state object. This is expected to + add items with all the keys in __class__.provides to the state + object. + """ + + name = self.__class__.__name__ + + try: + stored_step = state.steps[step_index] + except IndexError: + stored_step = None + + if stored_step == name: + self.restore(state) + elif stored_step is None: + self.create(state) + assert set(self.provides).issubset(set(state.keys())) + state.steps = state.steps + [name] + else: + raise ValueError("Expected a %s step, got a %s step" % (name, stored_step)) + + def create(self, data): + raise NotImplementedError + + def restore(self, state): + self.logger.debug("Step %s using stored state" % (self.__class__.__name__,)) + for key in self.provides: + assert key in state + + +class StepRunner(object): + steps = [] + + def __init__(self, logger, state): + """Class that runs a specified series of Steps with a common State""" + self.state = state + self.logger = logger + if "steps" not in state: + state.steps = [] + + def run(self): + rv = None + for step_index, step in enumerate(self.steps): + self.logger.debug("Starting step %s" % step.__name__) + rv = step(self.logger).run(step_index, self.state) + if rv in (exit_clean, exit_unclean): + break + + return rv diff --git a/testing/web-platform/harness/wptrunner/update/metadata.py b/testing/web-platform/harness/wptrunner/update/metadata.py new file mode 100644 index 000000000..c62dfec46 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/metadata.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +from .. import metadata, products + +from base import Step, StepRunner + +class GetUpdatePropertyList(Step): + provides = ["property_order", "boolean_properties"] + + + def create(self, state): + property_order, boolean_properties = products.load_product_update( + state.config, state.product) + state.property_order = property_order + state.boolean_properties = boolean_properties + + +class UpdateExpected(Step): + """Do the metadata update on the local checkout""" + + provides = ["needs_human"] + + def create(self, state): + if state.sync_tree is not None: + sync_root = state.sync_tree.root + else: + sync_root = None + + state.needs_human = metadata.update_expected(state.paths, + state.serve_root, + state.run_log, + rev_old=None, + ignore_existing=state.ignore_existing, + sync_root=sync_root, + property_order=state.property_order, + boolean_properties=state.boolean_properties) + + +class CreateMetadataPatch(Step): + """Create a patch/commit for the metadata checkout""" + + def create(self, state): + if state.no_patch: + return + + local_tree = state.local_tree + sync_tree = state.sync_tree + + if sync_tree is not None: + name = "web-platform-tests_update_%s_metadata" % sync_tree.rev + message = "Update %s expected data to revision %s" % (state.suite_name, sync_tree.rev) + else: + name = "web-platform-tests_update_metadata" + message = "Update %s expected data" % state.suite_name + + local_tree.create_patch(name, message) + + if not local_tree.is_clean: + metadata_paths = [manifest_path["metadata_path"] + for manifest_path in state.paths.itervalues()] + for path in metadata_paths: + local_tree.add_new(os.path.relpath(path, local_tree.root)) + local_tree.update_patch(include=metadata_paths) + local_tree.commit_patch() + + +class MetadataUpdateRunner(StepRunner): + """(Sub)Runner for updating metadata""" + steps = [GetUpdatePropertyList, + UpdateExpected, + CreateMetadataPatch] diff --git a/testing/web-platform/harness/wptrunner/update/state.py b/testing/web-platform/harness/wptrunner/update/state.py new file mode 100644 index 000000000..1d95cac5a --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/state.py @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import cPickle as pickle + +here = os.path.abspath(os.path.split(__file__)[0]) + +class State(object): + filename = os.path.join(here, ".wpt-update.lock") + + def __new__(cls, logger): + rv = cls.load(logger) + if rv is not None: + logger.debug("Existing state found") + return rv + + logger.debug("No existing state found") + return object.__new__(cls, logger) + + def __init__(self, logger): + """Object containing state variables created when running Steps. + + On write the state is serialized to disk, such that it can be restored in + the event that the program is interrupted before all steps are complete. + Note that this only works well if the values are immutable; mutating an + existing value will not cause the data to be serialized. + + Variables are set and get as attributes e.g. state_obj.spam = "eggs". + + :param parent: Parent State object or None if this is the root object. + """ + + if hasattr(self, "_data"): + return + + self._data = [{}] + self._logger = logger + self._index = 0 + + def __getstate__(self): + rv = self.__dict__.copy() + del rv["_logger"] + return rv + + @classmethod + def load(cls, logger): + """Load saved state from a file""" + try: + with open(cls.filename) as f: + try: + rv = pickle.load(f) + logger.debug("Loading data %r" % (rv._data,)) + rv._logger = logger + rv._index = 0 + return rv + except EOFError: + logger.warning("Found empty state file") + except IOError: + logger.debug("IOError loading stored state") + + def push(self, init_values): + """Push a new clean state dictionary + + :param init_values: List of variable names in the current state dict to copy + into the new state dict.""" + + return StateContext(self, init_values) + + def save(self): + """Write the state to disk""" + with open(self.filename, "w") as f: + pickle.dump(self, f) + + def is_empty(self): + return len(self._data) == 1 and self._data[0] == {} + + def clear(self): + """Remove all state and delete the stored copy.""" + try: + os.unlink(self.filename) + except OSError: + pass + self._data = [{}] + + + def __setattr__(self, key, value): + if key.startswith("_"): + object.__setattr__(self, key, value) + else: + self._data[self._index][key] = value + self.save() + + def __getattr__(self, key): + if key.startswith("_"): + raise AttributeError + try: + return self._data[self._index][key] + except KeyError: + raise AttributeError + + def __contains__(self, key): + return key in self._data[self._index] + + def update(self, items): + """Add a dictionary of {name: value} pairs to the state""" + self._data[self._index].update(items) + self.save() + + def keys(self): + return self._data[self._index].keys() + +class StateContext(object): + def __init__(self, state, init_values): + self.state = state + self.init_values = init_values + + def __enter__(self): + if len(self.state._data) == self.state._index + 1: + # This is the case where there is no stored state + new_state = {} + for key in self.init_values: + new_state[key] = self.state._data[self.state._index][key] + self.state._data.append(new_state) + self.state._index += 1 + self.state._logger.debug("Incremented index to %s" % self.state._index) + + def __exit__(self, *args, **kwargs): + if len(self.state._data) > 1: + assert self.state._index == len(self.state._data) - 1 + self.state._data.pop() + self.state._index -= 1 + self.state._logger.debug("Decremented index to %s" % self.state._index) + assert self.state._index >= 0 + else: + raise ValueError("Tried to pop the top state") diff --git a/testing/web-platform/harness/wptrunner/update/sync.py b/testing/web-platform/harness/wptrunner/update/sync.py new file mode 100644 index 000000000..28189eb38 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/sync.py @@ -0,0 +1,183 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import sys +import uuid + +from .. import testloader + +from base import Step, StepRunner +from tree import Commit + +here = os.path.abspath(os.path.split(__file__)[0]) + +bsd_license = """W3C 3-clause BSD License + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of works must retain the original copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the original copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +* Neither the name of the W3C nor the names of its contributors may be + used to endorse or promote products derived from this work without + specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +""" + + +def copy_wpt_tree(tree, dest): + """Copy the working copy of a Tree to a destination directory. + + :param tree: The Tree to copy. + :param dest: The destination directory""" + if os.path.exists(dest): + assert os.path.isdir(dest) + + shutil.rmtree(dest) + os.mkdir(dest) + + for tree_path in tree.paths(): + source_path = os.path.join(tree.root, tree_path) + dest_path = os.path.join(dest, tree_path) + + dest_dir = os.path.split(dest_path)[0] + if not os.path.isdir(source_path): + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + shutil.copy2(source_path, dest_path) + + for source, destination in [("testharness_runner.html", ""), + ("testharnessreport.js", "resources/")]: + source_path = os.path.join(here, os.pardir, source) + dest_path = os.path.join(dest, destination, os.path.split(source)[1]) + shutil.copy2(source_path, dest_path) + + add_license(dest) + + +def add_license(dest): + """Write the bsd license string to a LICENSE file. + + :param dest: Directory in which to place the LICENSE file.""" + with open(os.path.join(dest, "LICENSE"), "w") as f: + f.write(bsd_license) + +class UpdateCheckout(Step): + """Pull changes from upstream into the local sync tree.""" + + provides = ["local_branch"] + + def create(self, state): + sync_tree = state.sync_tree + state.local_branch = uuid.uuid4().hex + sync_tree.update(state.sync["remote_url"], + state.sync["branch"], + state.local_branch) + sync_path = os.path.abspath(sync_tree.root) + if not sync_path in sys.path: + from update import setup_paths + setup_paths(sync_path) + + def restore(self, state): + assert os.path.abspath(state.sync_tree.root) in sys.path + Step.restore(self, state) + + +class GetSyncTargetCommit(Step): + """Find the commit that we will sync to.""" + + provides = ["sync_commit"] + + def create(self, state): + if state.target_rev is None: + #Use upstream branch HEAD as the base commit + state.sync_commit = state.sync_tree.get_remote_sha1(state.sync["remote_url"], + state.sync["branch"]) + else: + state.sync_commit = Commit(state.sync_tree, state.rev) + + state.sync_tree.checkout(state.sync_commit.sha1, state.local_branch, force=True) + self.logger.debug("New base commit is %s" % state.sync_commit.sha1) + + +class LoadManifest(Step): + """Load the test manifest""" + + provides = ["manifest_path", "test_manifest", "old_manifest"] + + def create(self, state): + from manifest import manifest + state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json") + # Conservatively always rebuild the manifest when doing a sync + state.old_manifest = manifest.load(state.tests_path, state.manifest_path) + state.test_manifest = manifest.Manifest(None, "/") + + +class UpdateManifest(Step): + """Update the manifest to match the tests in the sync tree checkout""" + + def create(self, state): + from manifest import manifest, update + update.update(state.sync["path"], "/", state.test_manifest) + manifest.write(state.test_manifest, state.manifest_path) + + +class CopyWorkTree(Step): + """Copy the sync tree over to the destination in the local tree""" + + def create(self, state): + copy_wpt_tree(state.sync_tree, + state.tests_path) + + +class CreateSyncPatch(Step): + """Add the updated test files to a commit/patch in the local tree.""" + + def create(self, state): + if state.no_patch: + return + + local_tree = state.local_tree + sync_tree = state.sync_tree + + local_tree.create_patch("web-platform-tests_update_%s" % sync_tree.rev, + "Update %s to revision %s" % (state.suite_name, sync_tree.rev)) + local_tree.add_new(os.path.relpath(state.tests_path, + local_tree.root)) + updated = local_tree.update_patch(include=[state.tests_path, + state.metadata_path]) + local_tree.commit_patch() + + if not updated: + self.logger.info("Nothing to sync") + + +class SyncFromUpstreamRunner(StepRunner): + """(Sub)Runner for doing an upstream sync""" + steps = [UpdateCheckout, + GetSyncTargetCommit, + LoadManifest, + UpdateManifest, + CopyWorkTree, + CreateSyncPatch] diff --git a/testing/web-platform/harness/wptrunner/update/tree.py b/testing/web-platform/harness/wptrunner/update/tree.py new file mode 100644 index 000000000..05b35c246 --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/tree.py @@ -0,0 +1,387 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import subprocess + +from .. import vcs +from ..vcs import bind_to_repo, git, hg + + +def get_unique_name(existing, initial): + """Get a name either equal to initial or of the form initial_N, for some + integer N, that is not in the set existing. + + + :param existing: Set of names that must not be chosen. + :param initial: Name, or name prefix, to use""" + if initial not in existing: + return initial + for i in xrange(len(existing) + 1): + test = "%s_%s" % (initial, i + 1) + if test not in existing: + return test + assert False + +class NoVCSTree(object): + name = "non-vcs" + + def __init__(self, root=None): + if root is None: + root = os.path.abspath(os.curdir) + self.root = root + + @classmethod + def is_type(cls, path=None): + return True + + @property + def is_clean(self): + return True + + def add_new(self, prefix=None): + pass + + def create_patch(self, patch_name, message): + pass + + def update_patch(self, include=None): + pass + + def commit_patch(self): + pass + + +class HgTree(object): + name = "mercurial" + + def __init__(self, root=None): + if root is None: + root = hg("root").strip() + self.root = root + self.hg = vcs.bind_to_repo(hg, self.root) + + def __getstate__(self): + rv = self.__dict__.copy() + del rv['hg'] + return rv + + def __setstate__(self, dict): + self.__dict__.update(dict) + self.hg = vcs.bind_to_repo(vcs.hg, self.root) + + @classmethod + def is_type(cls, path=None): + kwargs = {"log_error": False} + if path is not None: + kwargs["repo"] = path + try: + hg("root", **kwargs) + except: + return False + return True + + @property + def is_clean(self): + return self.hg("status").strip() == "" + + def add_new(self, prefix=None): + if prefix is not None: + args = ("-I", prefix) + else: + args = () + self.hg("add", *args) + + def create_patch(self, patch_name, message): + try: + self.hg("qinit", log_error=False) + except subprocess.CalledProcessError: + pass + + patch_names = [item.strip() for item in self.hg("qseries").split("\n") if item.strip()] + + suffix = 0 + test_name = patch_name + while test_name in patch_names: + suffix += 1 + test_name = "%s-%i" % (patch_name, suffix) + + self.hg("qnew", test_name, "-X", self.root, "-m", message) + + def update_patch(self, include=None): + if include is not None: + args = [] + for item in include: + args.extend(["-I", item]) + else: + args = () + + self.hg("qrefresh", *args) + return True + + def commit_patch(self): + self.hg("qfinish") + + def contains_commit(self, commit): + try: + self.hg("identify", "-r", commit.sha1) + return True + except subprocess.CalledProcessError: + return False + + +class GitTree(object): + name = "git" + + def __init__(self, root=None): + if root is None: + root = git("rev-parse", "--show-toplevel").strip() + self.root = root + self.git = vcs.bind_to_repo(git, self.root) + self.message = None + self.commit_cls = Commit + + def __getstate__(self): + rv = self.__dict__.copy() + del rv['git'] + return rv + + def __setstate__(self, dict): + self.__dict__.update(dict) + self.git = vcs.bind_to_repo(vcs.git, self.root) + + @classmethod + def is_type(cls, path=None): + kwargs = {"log_error": False} + if path is not None: + kwargs["repo"] = path + try: + git("rev-parse", "--show-toplevel", **kwargs) + except: + return False + return True + + @property + def rev(self): + """Current HEAD revision""" + if vcs.is_git_root(self.root): + return self.git("rev-parse", "HEAD").strip() + else: + return None + + @property + def is_clean(self): + return self.git("status").strip() == "" + + def add_new(self, prefix=None): + """Add files to the staging area. + + :param prefix: None to include all files or a path prefix to + add all files under that path. + """ + if prefix is None: + args = ("-a",) + else: + args = ("--no-ignore-removal", prefix) + self.git("add", *args) + + def list_refs(self, ref_filter=None): + """Get a list of sha1, name tuples for references in a repository. + + :param ref_filter: Pattern that reference name must match (from the end, + matching whole /-delimited segments only + """ + args = [] + if ref_filter is not None: + args.append(ref_filter) + data = self.git("show-ref", *args) + rv = [] + for line in data.split("\n"): + if not line.strip(): + continue + sha1, ref = line.split() + rv.append((sha1, ref)) + return rv + + def list_remote(self, remote, ref_filter=None): + """Return a list of (sha1, name) tupes for references in a remote. + + :param remote: URL of the remote to list. + :param ref_filter: Pattern that the reference name must match. + """ + args = [] + if ref_filter is not None: + args.append(ref_filter) + data = self.git("ls-remote", remote, *args) + rv = [] + for line in data.split("\n"): + if not line.strip(): + continue + sha1, ref = line.split() + rv.append((sha1, ref)) + return rv + + def get_remote_sha1(self, remote, branch): + """Return the SHA1 of a particular branch in a remote. + + :param remote: the remote URL + :param branch: the branch name""" + for sha1, ref in self.list_remote(remote, branch): + if ref == "refs/heads/%s" % branch: + return self.commit_cls(self, sha1) + assert False + + def create_patch(self, patch_name, message): + # In git a patch is actually a commit + self.message = message + + def update_patch(self, include=None): + """Commit the staged changes, or changes to listed files. + + :param include: Either None, to commit staged changes, or a list + of filenames (which must already be in the repo) + to commit + """ + if include is not None: + args = tuple(include) + else: + args = () + + if self.git("status", "-uno", "-z", *args).strip(): + self.git("add", *args) + return True + return False + + def commit_patch(self): + assert self.message is not None + + if self.git("diff", "--name-only", "--staged", "-z").strip(): + self.git("commit", "-m", self.message) + return True + + return False + + def init(self): + self.git("init") + assert vcs.is_git_root(self.root) + + def checkout(self, rev, branch=None, force=False): + """Checkout a particular revision, optionally into a named branch. + + :param rev: Revision identifier (e.g. SHA1) to checkout + :param branch: Branch name to use + :param force: Force-checkout + """ + assert rev is not None + + args = [] + if branch: + branches = [ref[len("refs/heads/"):] for sha1, ref in self.list_refs() + if ref.startswith("refs/heads/")] + branch = get_unique_name(branches, branch) + + args += ["-b", branch] + + if force: + args.append("-f") + args.append(rev) + self.git("checkout", *args) + + def update(self, remote, remote_branch, local_branch): + """Fetch from the remote and checkout into a local branch. + + :param remote: URL to the remote repository + :param remote_branch: Branch on the remote repository to check out + :param local_branch: Local branch name to check out into + """ + if not vcs.is_git_root(self.root): + self.init() + self.git("clean", "-xdf") + self.git("fetch", remote, "%s:%s" % (remote_branch, local_branch)) + self.checkout(local_branch) + self.git("submodule", "update", "--init", "--recursive") + + def clean(self): + self.git("checkout", self.rev) + self.git("branch", "-D", self.local_branch) + + def paths(self): + """List paths in the tree""" + repo_paths = [self.root] + [os.path.join(self.root, path) + for path in self.submodules()] + + rv = [] + + for repo_path in repo_paths: + paths = vcs.git("ls-tree", "-r", "--name-only", "HEAD", repo=repo_path).split("\n") + rel_path = os.path.relpath(repo_path, self.root) + rv.extend(os.path.join(rel_path, item.strip()) for item in paths if item.strip()) + + return rv + + def submodules(self): + """List submodule directories""" + output = self.git("submodule", "status", "--recursive") + rv = [] + for line in output.split("\n"): + line = line.strip() + if not line: + continue + parts = line.split(" ") + rv.append(parts[1]) + return rv + + def contains_commit(self, commit): + try: + self.git("rev-parse", "--verify", commit.sha1) + return True + except subprocess.CalledProcessError: + return False + + +class CommitMessage(object): + def __init__(self, text): + self.text = text + self._parse_message() + + def __str__(self): + return self.text + + def _parse_message(self): + lines = self.text.splitlines() + self.full_summary = lines[0] + self.body = "\n".join(lines[1:]) + + +class Commit(object): + msg_cls = CommitMessage + + _sha1_re = re.compile("^[0-9a-f]{40}$") + + def __init__(self, tree, sha1): + """Object representing a commit in a specific GitTree. + + :param tree: GitTree to which this commit belongs. + :param sha1: Full sha1 string for the commit + """ + assert self._sha1_re.match(sha1) + + self.tree = tree + self.git = tree.git + self.sha1 = sha1 + self.author, self.email, self.message = self._get_meta() + + def __getstate__(self): + rv = self.__dict__.copy() + del rv['git'] + return rv + + def __setstate__(self, dict): + self.__dict__.update(dict) + self.git = self.tree.git + + def _get_meta(self): + author, email, message = self.git("show", "-s", "--format=format:%an\n%ae\n%B", self.sha1).split("\n", 2) + return author, email, self.msg_cls(message) diff --git a/testing/web-platform/harness/wptrunner/update/update.py b/testing/web-platform/harness/wptrunner/update/update.py new file mode 100644 index 000000000..213622c2a --- /dev/null +++ b/testing/web-platform/harness/wptrunner/update/update.py @@ -0,0 +1,158 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys + +from metadata import MetadataUpdateRunner +from sync import SyncFromUpstreamRunner +from tree import GitTree, HgTree, NoVCSTree + +from .. import environment as env +from base import Step, StepRunner, exit_clean, exit_unclean +from state import State + +def setup_paths(sync_path): + sys.path.insert(0, os.path.abspath(sync_path)) + from tools import localpaths + +class LoadConfig(Step): + """Step for loading configuration from the ini file and kwargs.""" + + provides = ["sync", "paths", "metadata_path", "tests_path"] + + def create(self, state): + state.sync = {"remote_url": state.kwargs["remote_url"], + "branch": state.kwargs["branch"], + "path": state.kwargs["sync_path"]} + + state.paths = state.kwargs["test_paths"] + state.tests_path = state.paths["/"]["tests_path"] + state.metadata_path = state.paths["/"]["metadata_path"] + + assert state.tests_path.startswith("/") + + +class LoadTrees(Step): + """Step for creating a Tree for the local copy and a GitTree for the + upstream sync.""" + + provides = ["local_tree", "sync_tree"] + + def create(self, state): + if os.path.exists(state.sync["path"]): + sync_tree = GitTree(root=state.sync["path"]) + else: + sync_tree = None + + if GitTree.is_type(): + local_tree = GitTree() + elif HgTree.is_type(): + local_tree = HgTree() + else: + local_tree = NoVCSTree() + + state.update({"local_tree": local_tree, + "sync_tree": sync_tree}) + + +class SyncFromUpstream(Step): + """Step that synchronises a local copy of the code with upstream.""" + + def create(self, state): + if not state.kwargs["sync"]: + return + + if not state.sync_tree: + os.mkdir(state.sync["path"]) + state.sync_tree = GitTree(root=state.sync["path"]) + + kwargs = state.kwargs + with state.push(["sync", "paths", "metadata_path", "tests_path", "local_tree", + "sync_tree"]): + state.target_rev = kwargs["rev"] + state.no_patch = kwargs["no_patch"] + state.suite_name = kwargs["suite_name"] + runner = SyncFromUpstreamRunner(self.logger, state) + runner.run() + + +class UpdateMetadata(Step): + """Update the expectation metadata from a set of run logs""" + + def create(self, state): + if not state.kwargs["run_log"]: + return + + kwargs = state.kwargs + with state.push(["local_tree", "sync_tree", "paths", "serve_root"]): + state.run_log = kwargs["run_log"] + state.ignore_existing = kwargs["ignore_existing"] + state.no_patch = kwargs["no_patch"] + state.suite_name = kwargs["suite_name"] + state.product = kwargs["product"] + state.config = kwargs["config"] + runner = MetadataUpdateRunner(self.logger, state) + runner.run() + + +class UpdateRunner(StepRunner): + """Runner for doing an overall update.""" + steps = [LoadConfig, + LoadTrees, + SyncFromUpstream, + UpdateMetadata] + + +class WPTUpdate(object): + def __init__(self, logger, runner_cls=UpdateRunner, **kwargs): + """Object that controls the running of a whole wptupdate. + + :param runner_cls: Runner subclass holding the overall list of + steps to run. + :param kwargs: Command line arguments + """ + self.runner_cls = runner_cls + self.serve_root = kwargs["test_paths"]["/"]["tests_path"] + + if not kwargs["sync"]: + setup_paths(self.serve_root) + else: + if os.path.exists(kwargs["sync_path"]): + # If the sync path doesn't exist we defer this until it does + setup_paths(kwargs["sync_path"]) + + self.state = State(logger) + self.kwargs = kwargs + self.logger = logger + + def run(self, **kwargs): + if self.kwargs["abort"]: + self.abort() + return exit_clean + + if not self.kwargs["continue"] and not self.state.is_empty(): + self.logger.error("Found existing state. Run with --continue to resume or --abort to clear state") + return exit_unclean + + if self.kwargs["continue"]: + if self.state.is_empty(): + self.logger.error("No sync in progress?") + return exit_clean + + self.kwargs = self.state.kwargs + else: + self.state.kwargs = self.kwargs + + self.state.serve_root = self.serve_root + + update_runner = self.runner_cls(self.logger, self.state) + rv = update_runner.run() + if rv in (exit_clean, None): + self.state.clear() + + return rv + + def abort(self): + self.state.clear() |