summaryrefslogtreecommitdiffstats
path: root/taskcluster/mach_commands.py
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /taskcluster/mach_commands.py
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'taskcluster/mach_commands.py')
-rw-r--r--taskcluster/mach_commands.py290
1 files changed, 290 insertions, 0 deletions
diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py
new file mode 100644
index 000000000..b5515db14
--- /dev/null
+++ b/taskcluster/mach_commands.py
@@ -0,0 +1,290 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import logging
+import sys
+import traceback
+
+from mach.decorators import (
+ CommandArgument,
+ CommandProvider,
+ Command,
+ SubCommand,
+)
+
+from mozbuild.base import MachCommandBase
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
+
+class ShowTaskGraphSubCommand(SubCommand):
+ """A SubCommand with TaskGraph-specific arguments"""
+
+ def __call__(self, func):
+ after = SubCommand.__call__(self, func)
+ args = [
+ CommandArgument('--root', '-r', default='taskcluster/ci',
+ help="root of the taskgraph definition relative to topsrcdir"),
+ CommandArgument('--quiet', '-q', action="store_true",
+ help="suppress all logging output"),
+ CommandArgument('--verbose', '-v', action="store_true",
+ help="include debug-level logging output"),
+ CommandArgument('--json', '-J', action="store_const",
+ dest="format", const="json",
+ help="Output task graph as a JSON object"),
+ CommandArgument('--labels', '-L', action="store_const",
+ dest="format", const="labels",
+ help="Output the label for each task in the task graph (default)"),
+ CommandArgument('--parameters', '-p', required=True,
+ help="parameters file (.yml or .json; see "
+ "`taskcluster/docs/parameters.rst`)`"),
+ CommandArgument('--no-optimize', dest="optimize", action="store_false",
+ default="true",
+ help="do not remove tasks from the graph that are found in the "
+ "index (a.k.a. optimize the graph)"),
+ ]
+ for arg in args:
+ after = arg(after)
+ return after
+
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+
+ @Command('taskgraph', category="ci",
+ description="Manipulate TaskCluster task graphs defined in-tree")
+ def taskgraph(self):
+ """The taskgraph subcommands all relate to the generation of task graphs
+ for Gecko continuous integration. A task graph is a set of tasks linked
+ by dependencies: for example, a binary must be built before it is tested,
+ and that build may further depend on various toolchains, libraries, etc.
+ """
+
+ @SubCommand('taskgraph', 'python-tests',
+ description='Run the taskgraph unit tests')
+ def taskgraph_python_tests(self, **options):
+ import unittest
+ import mozunit
+ suite = unittest.defaultTestLoader.discover('taskgraph.test')
+ runner = mozunit.MozTestRunner(verbosity=2)
+ result = runner.run(suite)
+ if not result.wasSuccessful():
+ sys.exit(1)
+
+ @ShowTaskGraphSubCommand('taskgraph', 'tasks',
+ description="Show all tasks in the taskgraph")
+ def taskgraph_tasks(self, **options):
+ return self.show_taskgraph('full_task_set', options)
+
+ @ShowTaskGraphSubCommand('taskgraph', 'full',
+ description="Show the full taskgraph")
+ def taskgraph_full(self, **options):
+ return self.show_taskgraph('full_task_graph', options)
+
+ @ShowTaskGraphSubCommand('taskgraph', 'target',
+ description="Show the target task set")
+ def taskgraph_target(self, **options):
+ return self.show_taskgraph('target_task_set', options)
+
+ @ShowTaskGraphSubCommand('taskgraph', 'target-graph',
+ description="Show the target taskgraph")
+ def taskgraph_target_taskgraph(self, **options):
+ return self.show_taskgraph('target_task_graph', options)
+
+ @ShowTaskGraphSubCommand('taskgraph', 'optimized',
+ description="Show the optimized taskgraph")
+ def taskgraph_optimized(self, **options):
+ return self.show_taskgraph('optimized_task_graph', options)
+
+ @SubCommand('taskgraph', 'decision',
+ description="Run the decision task")
+ @CommandArgument('--root', '-r',
+ default='taskcluster/ci',
+ help="root of the taskgraph definition relative to topsrcdir")
+ @CommandArgument('--base-repository',
+ required=True,
+ help='URL for "base" repository to clone')
+ @CommandArgument('--head-repository',
+ required=True,
+ help='URL for "head" repository to fetch revision from')
+ @CommandArgument('--head-ref',
+ required=True,
+ help='Reference (this is same as rev usually for hg)')
+ @CommandArgument('--head-rev',
+ required=True,
+ help='Commit revision to use from head repository')
+ @CommandArgument('--message',
+ required=True,
+ help='Commit message to be parsed. Example: "try: -b do -p all -u all"')
+ @CommandArgument('--revision-hash',
+ required=True,
+ help='Treeherder revision hash (long revision id) to attach results to')
+ @CommandArgument('--project',
+ required=True,
+ help='Project to use for creating task graph. Example: --project=try')
+ @CommandArgument('--pushlog-id',
+ dest='pushlog_id',
+ required=True,
+ default=0)
+ @CommandArgument('--pushdate',
+ dest='pushdate',
+ required=True,
+ type=int,
+ default=0)
+ @CommandArgument('--owner',
+ required=True,
+ help='email address of who owns this graph')
+ @CommandArgument('--level',
+ required=True,
+ help='SCM level of this repository')
+ @CommandArgument('--triggered-by',
+ choices=['nightly', 'push'],
+ default='push',
+ help='Source of execution of the decision graph')
+ @CommandArgument('--target-tasks-method',
+ help='method for selecting the target tasks to generate')
+ def taskgraph_decision(self, **options):
+ """Run the decision task: generate a task graph and submit to
+ TaskCluster. This is only meant to be called within decision tasks,
+ and requires a great many arguments. Commands like `mach taskgraph
+ optimized` are better suited to use on the command line, and can take
+ the parameters file generated by a decision task. """
+
+ import taskgraph.decision
+ try:
+ self.setup_logging()
+ return taskgraph.decision.taskgraph_decision(options)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ @SubCommand('taskgraph', 'action-task',
+ description="Run the action task")
+ @CommandArgument('--root', '-r',
+ default='taskcluster/ci',
+ help="root of the taskgraph definition relative to topsrcdir")
+ @CommandArgument('--decision-id',
+ required=True,
+ help="Decision Task ID of the reference decision task")
+ @CommandArgument('--task-labels',
+ required=True,
+ help='Comma separated list of task labels to be scheduled')
+ def taskgraph_action(self, **options):
+ """Run the action task: Generates a task graph using the set of labels
+ provided in the task-labels parameter. It uses the full-task file of
+ the gecko decision task."""
+
+ import taskgraph.action
+ try:
+ self.setup_logging()
+ return taskgraph.action.taskgraph_action(options)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ def setup_logging(self, quiet=False, verbose=True):
+ """
+ Set up Python logging for all loggers, sending results to stderr (so
+ that command output can be redirected easily) and adding the typical
+ mach timestamp.
+ """
+ # remove the old terminal handler
+ old = self.log_manager.replace_terminal_handler(None)
+
+ # re-add it, with level and fh set appropriately
+ if not quiet:
+ level = logging.DEBUG if verbose else logging.INFO
+ self.log_manager.add_terminal_logging(
+ fh=sys.stderr, level=level,
+ write_interval=old.formatter.write_interval,
+ write_times=old.formatter.write_times)
+
+ # all of the taskgraph logging is unstructured logging
+ self.log_manager.enable_unstructured()
+
+ def show_taskgraph(self, graph_attr, options):
+ import taskgraph.parameters
+ import taskgraph.target_tasks
+ import taskgraph.generator
+
+ try:
+ self.setup_logging(quiet=options['quiet'], verbose=options['verbose'])
+ parameters = taskgraph.parameters.load_parameters_file(options)
+ parameters.check()
+
+ target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
+ target_tasks_method = taskgraph.target_tasks.get_method(target_tasks_method)
+ tgg = taskgraph.generator.TaskGraphGenerator(
+ root_dir=options['root'],
+ parameters=parameters,
+ target_tasks_method=target_tasks_method)
+
+ tg = getattr(tgg, graph_attr)
+
+ show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels'))
+ show_method(tg)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ def show_taskgraph_labels(self, taskgraph):
+ for label in taskgraph.graph.visit_postorder():
+ print(label)
+
+ def show_taskgraph_json(self, taskgraph):
+ print(json.dumps(taskgraph.to_json(),
+ sort_keys=True, indent=2, separators=(',', ': ')))
+
+
+@CommandProvider
+class TaskClusterImagesProvider(object):
+ @Command('taskcluster-load-image', category="ci",
+ description="Load a pre-built Docker image")
+ @CommandArgument('--task-id',
+ help="Load the image at public/image.tar in this task,"
+ "rather than searching the index")
+ @CommandArgument('image_name', nargs='?',
+ help="Load the image of this name based on the current"
+ "contents of the tree (as built for mozilla-central"
+ "or mozilla-inbound)")
+ def load_image(self, image_name, task_id):
+ from taskgraph.docker import load_image_by_name, load_image_by_task_id
+ if not image_name and not task_id:
+ print("Specify either IMAGE-NAME or TASK-ID")
+ sys.exit(1)
+ try:
+ if task_id:
+ ok = load_image_by_task_id(task_id)
+ else:
+ ok = load_image_by_name(image_name)
+ if not ok:
+ sys.exit(1)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)
+
+ @Command('taskcluster-build-image', category='ci',
+ description='Build a Docker image')
+ @CommandArgument('image_name',
+ help='Name of the image to build')
+ @CommandArgument('--context-only',
+ help="File name the context tarball should be written to."
+ "with this option it will only build the context.tar.",
+ metavar='context.tar')
+ def build_image(self, image_name, context_only):
+ from taskgraph.docker import build_image, build_context
+ try:
+ if context_only is None:
+ build_image(image_name)
+ else:
+ build_context(image_name, context_only)
+ except Exception:
+ traceback.print_exc()
+ sys.exit(1)