Add m-esr52 at 52.6.0

76 files changed, 8556 insertions, 0 deletions

diff --git a/taskcluster/taskgraph/__init__.py b/taskcluster/taskgraph/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/__init__.py
diff --git a/taskcluster/taskgraph/action.py b/taskcluster/taskgraph/action.py
new file mode 100644
index 000000000..608fe3370
--- /dev/null
+++ b/taskcluster/taskgraph/action.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import logging
+import requests
+import yaml
+
+from .create import create_tasks
+from .decision import write_artifact
+from .optimize import optimize_task_graph
+from .taskgraph import TaskGraph
+
+logger = logging.getLogger(__name__)
+TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task/"
+
+
+def taskgraph_action(options):
+    """
+    Run the action task.  This function implements `mach taskgraph action-task`,
+    and is responsible for
+
+     * creating taskgraph of tasks asked for in parameters with respect to
+     a given gecko decision task and schedule these jobs.
+    """
+
+    decision_task_id = options['decision_id']
+    # read in the full graph for reference
+    full_task_json = get_artifact(decision_task_id, "public/full-task-graph.json")
+    decision_params = get_artifact(decision_task_id, "public/parameters.yml")
+    all_tasks, full_task_graph = TaskGraph.from_json(full_task_json)
+
+    target_tasks = set(options['task_labels'].split(','))
+    target_graph = full_task_graph.graph.transitive_closure(target_tasks)
+    target_task_graph = TaskGraph(
+        {l: all_tasks[l] for l in target_graph.nodes},
+        target_graph)
+
+    existing_tasks = get_artifact(decision_task_id, "public/label-to-taskid.json")
+
+    # We don't want to optimize target tasks since they have been requested by user
+    # Hence we put `target_tasks under` `do_not_optimize`
+    optimized_graph, label_to_taskid = optimize_task_graph(target_task_graph=target_task_graph,
+                                                           params=decision_params,
+                                                           do_not_optimize=target_tasks,
+                                                           existing_tasks=existing_tasks)
+
+    # write out the optimized task graph to describe what will actually happen,
+    # and the map of labels to taskids
+    write_artifact('task-graph.json', optimized_graph.to_json())
+    write_artifact('label-to-taskid.json', label_to_taskid)
+    # actually create the graph
+    create_tasks(optimized_graph, label_to_taskid, decision_params)
+
+
+def get_artifact(task_id, path):
+    url = TASKCLUSTER_QUEUE_URL + task_id + "/artifacts/" + path
+    resp = requests.get(url=url)
+    if path.endswith('.json'):
+        artifact = json.loads(resp.text)
+    elif path.endswith('.yml'):
+        artifact = yaml.load(resp.text)
+    return artifact
diff --git a/taskcluster/taskgraph/action.yml b/taskcluster/taskgraph/action.yml
new file mode 100644
index 000000000..c816f4d5c
--- /dev/null
+++ b/taskcluster/taskgraph/action.yml
@@ -0,0 +1,74 @@
+---
+created: '{{now}}'
+deadline: '{{#from_now}}1 day{{/from_now}}'
+expires: '{{#from_now}}14 day{{/from_now}}'
+metadata:
+  owner: mozilla-taskcluster-maintenance@mozilla.com
+  source: 'https://hg.mozilla.org/{{project}}/file/{{head_rev}}/taskcluster/taskgraph/action.yml'
+  name: "[tc] Action Task"
+  description: Helps schedule new jobs without new push
+
+workerType: "gecko-decision"
+provisionerId: "aws-provisioner-v1"
+schedulerId: "gecko-level-{{level}}"
+
+tags:
+  createdForUser: {{owner}}
+
+scopes:
+  # Bug 1269443: cache scopes, etc. must be listed explicitly
+  - "docker-worker:cache:level-1-*"
+  - "docker-worker:cache:tooltool-cache"
+  - "secrets:get:project/taskcluster/gecko/hgfingerprint"
+  - "assume:repo:hg.mozilla.org/try:*"
+
+routes:
+  - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
+  - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}"
+
+payload:
+  env:
+    GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
+    GECKO_HEAD_REPOSITORY: '{{{head_repository}}}'
+    GECKO_HEAD_REF: '{{head_ref}}'
+    GECKO_HEAD_REV: '{{head_rev}}'
+    HG_STORE_PATH: /home/worker/checkouts/hg-store
+
+  cache:
+    level-{{level}}-checkouts: /home/worker/checkouts
+
+  features:
+    taskclusterProxy: true
+
+  # Note: This task is built server side without the context or tooling that
+  # exist in tree so we must hard code the version
+  image: 'taskcluster/decision:0.1.7'
+
+  # Virtually no network or other potentially risky operations happen as part
+  # of the task timeout aside from the initial clone. We intentionally have
+  # set this to a lower value _all_ decision tasks should use a root
+  # repository which is cached.
+  maxRunTime: 1800
+
+  command:
+    - /home/worker/bin/run-task
+    - '--vcs-checkout=/home/worker/checkouts/gecko'
+    - '--'
+    - bash
+    - -cx
+    - >
+        cd /home/worker/checkouts/gecko &&
+        ln -s /home/worker/artifacts artifacts &&
+        ./mach --log-no-times taskgraph action-task
+        --decision-id='{{decision_task_id}}'
+        --task-label='{{task_labels}}'
+
+  artifacts:
+    'public':
+      type: 'directory'
+      path: '/home/worker/artifacts'
+      expires: '{{#from_now}}7 days{{/from_now}}'
+
+extra:
+  treeherder:
+    symbol: A
diff --git a/taskcluster/taskgraph/create.py b/taskcluster/taskgraph/create.py
new file mode 100644
index 000000000..f577f8873
--- /dev/null
+++ b/taskcluster/taskgraph/create.py
@@ -0,0 +1,122 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import concurrent.futures as futures
+import requests
+import requests.adapters
+import json
+import os
+import logging
+
+from slugid import nice as slugid
+from taskgraph.util.time import (
+    current_json_time,
+    json_time_from_now
+)
+
+logger = logging.getLogger(__name__)
+
+# the maximum number of parallel createTask calls to make
+CONCURRENCY = 50
+
+
+def create_tasks(taskgraph, label_to_taskid, params):
+    taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()}
+
+    session = requests.Session()
+
+    # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase
+    # that limit. Connections are established as needed, so using a large value
+    # should not negatively impact performance.
+    http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY,
+                                                 pool_maxsize=CONCURRENCY)
+    session.mount('https://', http_adapter)
+    session.mount('http://', http_adapter)
+
+    decision_task_id = os.environ.get('TASK_ID')
+
+    # when running as an actual decision task, we use the decision task's
+    # taskId as the taskGroupId.  The process that created the decision task
+    # helpfully placed it in this same taskGroup.  If there is no $TASK_ID,
+    # fall back to a slugid
+    task_group_id = decision_task_id or slugid()
+    scheduler_id = 'gecko-level-{}'.format(params['level'])
+
+    with futures.ThreadPoolExecutor(CONCURRENCY) as e:
+        fs = {}
+
+        # We can't submit a task until its dependencies have been submitted.
+        # So our strategy is to walk the graph and submit tasks once all
+        # their dependencies have been submitted.
+        #
+        # Using visit_postorder() here isn't the most efficient: we'll
+        # block waiting for dependencies of task N to submit even though
+        # dependencies for task N+1 may be finished. If we need to optimize
+        # this further, we can build a graph of task dependencies and walk
+        # that.
+        for task_id in taskgraph.graph.visit_postorder():
+            task_def = taskgraph.tasks[task_id].task
+            attributes = taskgraph.tasks[task_id].attributes
+            # if this task has no dependencies, make it depend on this decision
+            # task so that it does not start immediately; and so that if this loop
+            # fails halfway through, none of the already-created tasks run.
+            if decision_task_id and not task_def.get('dependencies'):
+                task_def['dependencies'] = [decision_task_id]
+
+            task_def['taskGroupId'] = task_group_id
+            task_def['schedulerId'] = scheduler_id
+
+            # Wait for dependencies before submitting this.
+            deps_fs = [fs[dep] for dep in task_def.get('dependencies', [])
+                       if dep in fs]
+            for f in futures.as_completed(deps_fs):
+                f.result()
+
+            fs[task_id] = e.submit(_create_task, session, task_id,
+                                   taskid_to_label[task_id], task_def)
+
+            # Schedule tasks as many times as task_duplicates indicates
+            for i in range(1, attributes.get('task_duplicates', 1)):
+                # We use slugid() since we want a distinct task id
+                fs[task_id] = e.submit(_create_task, session, slugid(),
+                                       taskid_to_label[task_id], task_def)
+
+        # Wait for all futures to complete.
+        for f in futures.as_completed(fs.values()):
+            f.result()
+
+
+def _create_task(session, task_id, label, task_def):
+    # create the task using 'http://taskcluster/queue', which is proxied to the queue service
+    # with credentials appropriate to this job.
+
+    # Resolve timestamps
+    now = current_json_time(datetime_format=True)
+    task_def = resolve_timestamps(now, task_def)
+
+    logger.debug("Creating task with taskId {} for {}".format(task_id, label))
+    res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id),
+                      data=json.dumps(task_def))
+    if res.status_code != 200:
+        try:
+            logger.error(res.json()['message'])
+        except:
+            logger.error(res.text)
+        res.raise_for_status()
+
+
+def resolve_timestamps(now, task_def):
+    def recurse(val):
+        if isinstance(val, list):
+            return [recurse(v) for v in val]
+        elif isinstance(val, dict):
+            if val.keys() == ['relative-datestamp']:
+                return json_time_from_now(val['relative-datestamp'], now)
+            else:
+                return {k: recurse(v) for k, v in val.iteritems()}
+        else:
+            return val
+    return recurse(task_def)
diff --git a/taskcluster/taskgraph/decision.py b/taskcluster/taskgraph/decision.py
new file mode 100644
index 000000000..d6460390f
--- /dev/null
+++ b/taskcluster/taskgraph/decision.py
@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import json
+import logging
+
+import time
+import yaml
+
+from .generator import TaskGraphGenerator
+from .create import create_tasks
+from .parameters import Parameters
+from .target_tasks import get_method
+from .taskgraph import TaskGraph
+
+from taskgraph.util.templates import Templates
+from taskgraph.util.time import (
+    json_time_from_now,
+    current_json_time,
+)
+
+logger = logging.getLogger(__name__)
+
+ARTIFACTS_DIR = 'artifacts'
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+
+# For each project, this gives a set of parameters specific to the project.
+# See `taskcluster/docs/parameters.rst` for information on parameters.
+PER_PROJECT_PARAMETERS = {
+    'try': {
+        'target_tasks_method': 'try_option_syntax',
+        # Always perform optimization.  This makes it difficult to use try
+        # pushes to run a task that would otherwise be optimized, but is a
+        # compromise to avoid essentially disabling optimization in try.
+        'optimize_target_tasks': True,
+    },
+
+    'ash': {
+        'target_tasks_method': 'ash_tasks',
+        'optimize_target_tasks': True,
+    },
+
+    'cedar': {
+        'target_tasks_method': 'cedar_tasks',
+        'optimize_target_tasks': True,
+    },
+
+    # the default parameters are used for projects that do not match above.
+    'default': {
+        'target_tasks_method': 'default',
+        'optimize_target_tasks': True,
+    }
+}
+
+
+def taskgraph_decision(options):
+    """
+    Run the decision task.  This function implements `mach taskgraph decision`,
+    and is responsible for
+
+     * processing decision task command-line options into parameters
+     * running task-graph generation exactly the same way the other `mach
+       taskgraph` commands do
+     * generating a set of artifacts to memorialize the graph
+     * calling TaskCluster APIs to create the graph
+    """
+
+    parameters = get_decision_parameters(options)
+
+    # create a TaskGraphGenerator instance
+    target_tasks_method = parameters.get('target_tasks_method', 'all_tasks')
+    target_tasks_method = get_method(target_tasks_method)
+    tgg = TaskGraphGenerator(
+        root_dir=options['root'],
+        parameters=parameters,
+        target_tasks_method=target_tasks_method)
+
+    # write out the parameters used to generate this graph
+    write_artifact('parameters.yml', dict(**parameters))
+
+    # write out the yml file for action tasks
+    write_artifact('action.yml', get_action_yml(parameters))
+
+    # write out the full graph for reference
+    full_task_json = tgg.full_task_graph.to_json()
+    write_artifact('full-task-graph.json', full_task_json)
+
+    # this is just a test to check whether the from_json() function is working
+    _, _ = TaskGraph.from_json(full_task_json)
+
+    # write out the target task set to allow reproducing this as input
+    write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys())
+
+    # write out the optimized task graph to describe what will actually happen,
+    # and the map of labels to taskids
+    write_artifact('task-graph.json', tgg.optimized_task_graph.to_json())
+    write_artifact('label-to-taskid.json', tgg.label_to_taskid)
+
+    # actually create the graph
+    create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid, parameters)
+
+
+def get_decision_parameters(options):
+    """
+    Load parameters from the command-line options for 'taskgraph decision'.
+    This also applies per-project parameters, based on the given project.
+
+    """
+    parameters = {n: options[n] for n in [
+        'base_repository',
+        'head_repository',
+        'head_rev',
+        'head_ref',
+        'message',
+        'project',
+        'pushlog_id',
+        'pushdate',
+        'owner',
+        'level',
+        'triggered_by',
+        'target_tasks_method',
+    ] if n in options}
+
+    # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which
+    # case, fake it
+    if '@' not in parameters['owner']:
+        parameters['owner'] += '@noreply.mozilla.org'
+
+    # use the pushdate as build_date if given, else use current time
+    parameters['build_date'] = parameters['pushdate'] or int(time.time())
+    # moz_build_date is the build identifier based on build_date
+    parameters['moz_build_date'] = time.strftime("%Y%m%d%H%M%S",
+                                                 time.gmtime(parameters['build_date']))
+
+    project = parameters['project']
+    try:
+        parameters.update(PER_PROJECT_PARAMETERS[project])
+    except KeyError:
+        logger.warning("using default project parameters; add {} to "
+                       "PER_PROJECT_PARAMETERS in {} to customize behavior "
+                       "for this project".format(project, __file__))
+        parameters.update(PER_PROJECT_PARAMETERS['default'])
+
+    # `target_tasks_method` has higher precedence than `project` parameters
+    if options.get('target_tasks_method'):
+        parameters['target_tasks_method'] = options['target_tasks_method']
+
+    return Parameters(parameters)
+
+
+def write_artifact(filename, data):
+    logger.info('writing artifact file `{}`'.format(filename))
+    if not os.path.isdir(ARTIFACTS_DIR):
+        os.mkdir(ARTIFACTS_DIR)
+    path = os.path.join(ARTIFACTS_DIR, filename)
+    if filename.endswith('.yml'):
+        with open(path, 'w') as f:
+            yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
+    elif filename.endswith('.json'):
+        with open(path, 'w') as f:
+            json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': '))
+    else:
+        raise TypeError("Don't know how to write to {}".format(filename))
+
+
+def get_action_yml(parameters):
+    templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph"))
+    action_parameters = parameters.copy()
+    action_parameters.update({
+        "decision_task_id": "{{decision_task_id}}",
+        "task_labels": "{{task_labels}}",
+        "from_now": json_time_from_now,
+        "now": current_json_time()
+    })
+    return templates.load('action.yml', action_parameters)
diff --git a/taskcluster/taskgraph/docker.py b/taskcluster/taskgraph/docker.py
new file mode 100644
index 000000000..8159fd5a4
--- /dev/null
+++ b/taskcluster/taskgraph/docker.py
@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import subprocess
+import tarfile
+import tempfile
+import urllib2
+import which
+
+from taskgraph.util import docker
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+IMAGE_DIR = os.path.join(GECKO, 'testing', 'docker')
+INDEX_URL = 'https://index.taskcluster.net/v1/task/' + docker.INDEX_PREFIX + '.{}.{}.hash.{}'
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
+
+def load_image_by_name(image_name):
+    context_path = os.path.join(GECKO, 'testing', 'docker', image_name)
+    context_hash = docker.generate_context_hash(GECKO, context_path, image_name)
+
+    image_index_url = INDEX_URL.format('mozilla-central', image_name, context_hash)
+    print("Fetching", image_index_url)
+    task = json.load(urllib2.urlopen(image_index_url))
+
+    return load_image_by_task_id(task['taskId'])
+
+
+def load_image_by_task_id(task_id):
+    # because we need to read this file twice (and one read is not all the way
+    # through), it is difficult to stream it.  So we download to disk and then
+    # read it back.
+    filename = 'temp-docker-image.tar'
+
+    artifact_url = ARTIFACT_URL.format(task_id, 'public/image.tar.zst')
+    print("Downloading", artifact_url)
+    tempfilename = 'temp-docker-image.tar.zst'
+    subprocess.check_call(['curl', '-#', '-L', '-o', tempfilename, artifact_url])
+    print("Decompressing")
+    subprocess.check_call(['zstd', '-d', tempfilename, '-o', filename])
+    print("Deleting temporary file")
+    os.unlink(tempfilename)
+
+    print("Determining image name")
+    tf = tarfile.open(filename)
+    repositories = json.load(tf.extractfile('repositories'))
+    name = repositories.keys()[0]
+    tag = repositories[name].keys()[0]
+    name = '{}:{}'.format(name, tag)
+    print("Image name:", name)
+
+    print("Loading image into docker")
+    try:
+        subprocess.check_call(['docker', 'load', '-i', filename])
+    except subprocess.CalledProcessError:
+        print("*** `docker load` failed.  You may avoid re-downloading that tarball by fixing the")
+        print("*** problem and running `docker load < {}`.".format(filename))
+        raise
+
+    print("Deleting temporary file")
+    os.unlink(filename)
+
+    print("The requested docker image is now available as", name)
+    print("Try: docker run -ti --rm {} bash".format(name))
+
+
+def build_context(name, outputFile):
+    """Build a context.tar for image with specified name.
+    """
+    if not name:
+        raise ValueError('must provide a Docker image name')
+    if not outputFile:
+        raise ValueError('must provide a outputFile')
+
+    image_dir = os.path.join(IMAGE_DIR, name)
+    if not os.path.isdir(image_dir):
+        raise Exception('image directory does not exist: %s' % image_dir)
+
+    docker.create_context_tar(GECKO, image_dir, outputFile, "")
+
+
+def build_image(name):
+    """Build a Docker image of specified name.
+
+    Output from image building process will be printed to stdout.
+    """
+    if not name:
+        raise ValueError('must provide a Docker image name')
+
+    image_dir = os.path.join(IMAGE_DIR, name)
+    if not os.path.isdir(image_dir):
+        raise Exception('image directory does not exist: %s' % image_dir)
+
+    tag = docker.docker_image(name, default_version='latest')
+
+    docker_bin = which.which('docker')
+
+    # Verify that Docker is working.
+    try:
+        subprocess.check_output([docker_bin, '--version'])
+    except subprocess.CalledProcessError:
+        raise Exception('Docker server is unresponsive. Run `docker ps` and '
+                        'check that Docker is running')
+
+    # We obtain a context archive and build from that. Going through the
+    # archive creation is important: it normalizes things like file owners
+    # and mtimes to increase the chances that image generation is
+    # deterministic.
+    fd, context_path = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        docker.create_context_tar(GECKO, image_dir, context_path, name)
+        docker.build_from_context(docker_bin, context_path, name, tag)
+    finally:
+        os.unlink(context_path)
+
+    print('Successfully built %s and tagged with %s' % (name, tag))
+
+    if tag.endswith(':latest'):
+        print('*' * 50)
+        print('WARNING: no VERSION file found in image directory.')
+        print('Image is not suitable for deploying/pushing.')
+        print('Create an image suitable for deploying/pushing by creating')
+        print('a VERSION file in the image directory.')
+        print('*' * 50)
diff --git a/taskcluster/taskgraph/files_changed.py b/taskcluster/taskgraph/files_changed.py
new file mode 100644
index 000000000..973a62bb8
--- /dev/null
+++ b/taskcluster/taskgraph/files_changed.py
@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Support for optimizing tasks based on the set of files that have changed.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import requests
+from redo import retry
+from mozpack.path import match as mozpackmatch
+
+logger = logging.getLogger(__name__)
+_cache = {}
+
+
+def get_changed_files(repository, revision):
+    """
+    Get the set of files changed in the push headed by the given revision.
+    Responses are cached, so multiple calls with the same arguments are OK.
+    """
+    key = repository, revision
+    if key not in _cache:
+        url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
+        logger.debug("Querying version control for metadata: %s", url)
+
+        def get_automationrelevance():
+            response = requests.get(url, timeout=5)
+            return response.json()
+        contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
+
+        logger.debug('{} commits influencing task scheduling:'
+                     .format(len(contents['changesets'])))
+        changed_files = set()
+        for c in contents['changesets']:
+            logger.debug(" {cset} {desc}".format(
+                cset=c['node'][0:12],
+                desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
+            changed_files |= set(c['files'])
+
+        _cache[key] = changed_files
+    return _cache[key]
+
+
+def check(params, file_patterns):
+    """Determine whether any of the files changed in the indicated push to
+    https://hg.mozilla.org match any of the given file patterns."""
+    repository = params.get('head_repository')
+    revision = params.get('head_rev')
+    if not repository or not revision:
+        logger.warning("Missing `head_repository` or `head_rev` parameters; "
+                       "assuming all files have changed")
+        return True
+
+    changed_files = get_changed_files(repository, revision)
+
+    for pattern in file_patterns:
+        for path in changed_files:
+            if mozpackmatch(path, pattern):
+                return True
+
+    return False
diff --git a/taskcluster/taskgraph/generator.py b/taskcluster/taskgraph/generator.py
new file mode 100644
index 000000000..809ed1f5c
--- /dev/null
+++ b/taskcluster/taskgraph/generator.py
@@ -0,0 +1,218 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+import logging
+import os
+import yaml
+
+from .graph import Graph
+from .taskgraph import TaskGraph
+from .optimize import optimize_task_graph
+from .util.python_path import find_object
+
+logger = logging.getLogger(__name__)
+
+
+class Kind(object):
+
+    def __init__(self, name, path, config):
+        self.name = name
+        self.path = path
+        self.config = config
+
+    def _get_impl_class(self):
+        # load the class defined by implementation
+        try:
+            impl = self.config['implementation']
+        except KeyError:
+            raise KeyError("{!r} does not define implementation".format(self.path))
+        return find_object(impl)
+
+    def load_tasks(self, parameters, loaded_tasks):
+        impl_class = self._get_impl_class()
+        return impl_class.load_tasks(self.name, self.path, self.config,
+                                     parameters, loaded_tasks)
+
+
+class TaskGraphGenerator(object):
+    """
+    The central controller for taskgraph.  This handles all phases of graph
+    generation.  The task is generated from all of the kinds defined in
+    subdirectories of the generator's root directory.
+
+    Access to the results of this generation, as well as intermediate values at
+    various phases of generation, is available via properties.  This encourages
+    the provision of all generation inputs at instance construction time.
+    """
+
+    # Task-graph generation is implemented as a Python generator that yields
+    # each "phase" of generation.  This allows some mach subcommands to short-
+    # circuit generation of the entire graph by never completing the generator.
+
+    def __init__(self, root_dir, parameters,
+                 target_tasks_method):
+        """
+        @param root_dir: root directory, with subdirectories for each kind
+        @param parameters: parameters for this task-graph generation
+        @type parameters: dict
+        @param target_tasks_method: function to determine the target_task_set;
+                see `./target_tasks.py`.
+        @type target_tasks_method: function
+        """
+
+        self.root_dir = root_dir
+        self.parameters = parameters
+        self.target_tasks_method = target_tasks_method
+
+        # this can be set up until the time the target task set is generated;
+        # it defaults to parameters['target_tasks']
+        self._target_tasks = parameters.get('target_tasks')
+
+        # start the generator
+        self._run = self._run()
+        self._run_results = {}
+
+    @property
+    def full_task_set(self):
+        """
+        The full task set: all tasks defined by any kind (a graph without edges)
+
+        @type: TaskGraph
+        """
+        return self._run_until('full_task_set')
+
+    @property
+    def full_task_graph(self):
+        """
+        The full task graph: the full task set, with edges representing
+        dependencies.
+
+        @type: TaskGraph
+        """
+        return self._run_until('full_task_graph')
+
+    @property
+    def target_task_set(self):
+        """
+        The set of targetted tasks (a graph without edges)
+
+        @type: TaskGraph
+        """
+        return self._run_until('target_task_set')
+
+    @property
+    def target_task_graph(self):
+        """
+        The set of targetted tasks and all of their dependencies
+
+        @type: TaskGraph
+        """
+        return self._run_until('target_task_graph')
+
+    @property
+    def optimized_task_graph(self):
+        """
+        The set of targetted tasks and all of their dependencies; tasks that
+        have been optimized out are either omitted or replaced with a Task
+        instance containing only a task_id.
+
+        @type: TaskGraph
+        """
+        return self._run_until('optimized_task_graph')
+
+    @property
+    def label_to_taskid(self):
+        """
+        A dictionary mapping task label to assigned taskId.  This property helps
+        in interpreting `optimized_task_graph`.
+
+        @type: dictionary
+        """
+        return self._run_until('label_to_taskid')
+
+    def _load_kinds(self):
+        for path in os.listdir(self.root_dir):
+            path = os.path.join(self.root_dir, path)
+            if not os.path.isdir(path):
+                continue
+            kind_name = os.path.basename(path)
+
+            kind_yml = os.path.join(path, 'kind.yml')
+            if not os.path.exists(kind_yml):
+                continue
+
+            logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
+            with open(kind_yml) as f:
+                config = yaml.load(f)
+
+            yield Kind(kind_name, path, config)
+
+    def _run(self):
+        logger.info("Loading kinds")
+        # put the kinds into a graph and sort topologically so that kinds are loaded
+        # in post-order
+        kinds = {kind.name: kind for kind in self._load_kinds()}
+        edges = set()
+        for kind in kinds.itervalues():
+            for dep in kind.config.get('kind-dependencies', []):
+                edges.add((kind.name, dep, 'kind-dependency'))
+        kind_graph = Graph(set(kinds), edges)
+
+        logger.info("Generating full task set")
+        all_tasks = {}
+        for kind_name in kind_graph.visit_postorder():
+            logger.debug("Loading tasks for kind {}".format(kind_name))
+            kind = kinds[kind_name]
+            new_tasks = kind.load_tasks(self.parameters, list(all_tasks.values()))
+            for task in new_tasks:
+                if task.label in all_tasks:
+                    raise Exception("duplicate tasks with label " + task.label)
+                all_tasks[task.label] = task
+            logger.info("Generated {} tasks for kind {}".format(len(new_tasks), kind_name))
+        full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
+        yield 'full_task_set', full_task_set
+
+        logger.info("Generating full task graph")
+        edges = set()
+        for t in full_task_set:
+            for dep, depname in t.get_dependencies(full_task_set):
+                edges.add((t.label, dep, depname))
+
+        full_task_graph = TaskGraph(all_tasks,
+                                    Graph(full_task_set.graph.nodes, edges))
+        yield 'full_task_graph', full_task_graph
+
+        logger.info("Generating target task set")
+        target_tasks = set(self.target_tasks_method(full_task_graph, self.parameters))
+        target_task_set = TaskGraph(
+            {l: all_tasks[l] for l in target_tasks},
+            Graph(target_tasks, set()))
+        yield 'target_task_set', target_task_set
+
+        logger.info("Generating target task graph")
+        target_graph = full_task_graph.graph.transitive_closure(target_tasks)
+        target_task_graph = TaskGraph(
+            {l: all_tasks[l] for l in target_graph.nodes},
+            target_graph)
+        yield 'target_task_graph', target_task_graph
+
+        logger.info("Generating optimized task graph")
+        do_not_optimize = set()
+        if not self.parameters.get('optimize_target_tasks', True):
+            do_not_optimize = target_task_set.graph.nodes
+        optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
+                                                                    self.parameters,
+                                                                    do_not_optimize)
+        yield 'label_to_taskid', label_to_taskid
+        yield 'optimized_task_graph', optimized_task_graph
+
+    def _run_until(self, name):
+        while name not in self._run_results:
+            try:
+                k, v = self._run.next()
+            except StopIteration:
+                raise AttributeError("No such run result {}".format(name))
+            self._run_results[k] = v
+        return self._run_results[name]
diff --git a/taskcluster/taskgraph/graph.py b/taskcluster/taskgraph/graph.py
new file mode 100644
index 000000000..731341c51
--- /dev/null
+++ b/taskcluster/taskgraph/graph.py
@@ -0,0 +1,117 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import collections
+
+
+class Graph(object):
+    """
+    Generic representation of a directed acyclic graph with labeled edges
+    connecting the nodes.  Graph operations are implemented in a functional
+    manner, so the data structure is immutable.
+
+    It permits at most one edge of a given name between any set of nodes.  The
+    graph is not checked for cycles, and methods may hang or otherwise fail if
+    given a cyclic graph.
+
+    The `nodes` and `edges` attributes may be accessed in a read-only fashion.
+    The `nodes` attribute is a set of node names, while `edges` is a set of
+    `(left, right, name)` tuples representing an edge named `name` going from
+    node `left` to node `right..
+    """
+
+    def __init__(self, nodes, edges):
+        """
+        Create a graph.  Nodes and edges are both as described in the class
+        documentation.  Both values are used by reference, and should not be
+        modified after building a graph.
+        """
+        assert isinstance(nodes, set)
+        assert isinstance(edges, set)
+        self.nodes = nodes
+        self.edges = edges
+
+    def __eq__(self, other):
+        return self.nodes == other.nodes and self.edges == other.edges
+
+    def __repr__(self):
+        return "<Graph nodes={!r} edges={!r}>".format(self.nodes, self.edges)
+
+    def transitive_closure(self, nodes):
+        """
+        Return the transitive closure of <nodes>: the graph containing all
+        specified nodes as well as any nodes reachable from them, and any
+        intervening edges.
+        """
+        assert isinstance(nodes, set)
+        assert nodes <= self.nodes
+
+        # generate a new graph by expanding along edges until reaching a fixed
+        # point
+        new_nodes, new_edges = nodes, set()
+        nodes, edges = set(), set()
+        while (new_nodes, new_edges) != (nodes, edges):
+            nodes, edges = new_nodes, new_edges
+            add_edges = set((left, right, name)
+                            for (left, right, name) in self.edges
+                            if left in nodes)
+            add_nodes = set(right for (_, right, _) in add_edges)
+            new_nodes = nodes | add_nodes
+            new_edges = edges | add_edges
+        return Graph(new_nodes, new_edges)
+
+    def visit_postorder(self):
+        """
+        Generate a sequence of nodes in postorder, such that every node is
+        visited *after* any nodes it links to.
+
+        Behavior is undefined (read: it will hang) if the graph contains a
+        cycle.
+        """
+        queue = collections.deque(sorted(self.nodes))
+        links_by_node = self.links_dict()
+        seen = set()
+        while queue:
+            node = queue.popleft()
+            if node in seen:
+                continue
+            links = links_by_node[node]
+            if all((n in seen) for n in links):
+                seen.add(node)
+                yield node
+            else:
+                queue.extend(n for n in links if n not in seen)
+                queue.append(node)
+
+    def links_dict(self):
+        """
+        Return a dictionary mapping each node to a set of the nodes it links to
+        (omitting edge names)
+        """
+        links = collections.defaultdict(set)
+        for left, right, _ in self.edges:
+            links[left].add(right)
+        return links
+
+    def named_links_dict(self):
+        """
+        Return a two-level dictionary mapping each node to a dictionary mapping
+        edge names to labels.
+        """
+        links = collections.defaultdict(dict)
+        for left, right, name in self.edges:
+            links[left][name] = right
+        return links
+
+    def reverse_links_dict(self):
+        """
+        Return a dictionary mapping each node to a set of the nodes linking to
+        it (omitting edge names)
+        """
+        links = collections.defaultdict(set)
+        for left, right, _ in self.edges:
+            links[right].add(left)
+        return links
diff --git a/taskcluster/taskgraph/optimize.py b/taskcluster/taskgraph/optimize.py
new file mode 100644
index 000000000..120e6807b
--- /dev/null
+++ b/taskcluster/taskgraph/optimize.py
@@ -0,0 +1,156 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+import logging
+import re
+
+from .graph import Graph
+from .taskgraph import TaskGraph
+from slugid import nice as slugid
+
+logger = logging.getLogger(__name__)
+TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
+
+
+def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
+    """
+    Perform task optimization, without optimizing tasks named in
+    do_not_optimize.
+    """
+    named_links_dict = target_task_graph.graph.named_links_dict()
+    label_to_taskid = {}
+
+    # This proceeds in two phases.  First, mark all optimized tasks (those
+    # which will be removed from the graph) as such, including a replacement
+    # taskId where applicable.  Second, generate a new task graph containing
+    # only the non-optimized tasks, with all task labels resolved to taskIds
+    # and with task['dependencies'] populated.
+    annotate_task_graph(target_task_graph=target_task_graph,
+                        params=params,
+                        do_not_optimize=do_not_optimize,
+                        named_links_dict=named_links_dict,
+                        label_to_taskid=label_to_taskid,
+                        existing_tasks=existing_tasks)
+    return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid
+
+
+def resolve_task_references(label, task_def, taskid_for_edge_name):
+    def repl(match):
+        key = match.group(1)
+        try:
+            return taskid_for_edge_name[key]
+        except KeyError:
+            # handle escaping '<'
+            if key == '<':
+                return key
+            raise KeyError("task '{}' has no dependency named '{}'".format(label, key))
+
+    def recurse(val):
+        if isinstance(val, list):
+            return [recurse(v) for v in val]
+        elif isinstance(val, dict):
+            if val.keys() == ['task-reference']:
+                return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference'])
+            else:
+                return {k: recurse(v) for k, v in val.iteritems()}
+        else:
+            return val
+    return recurse(task_def)
+
+
+def annotate_task_graph(target_task_graph, params, do_not_optimize,
+                        named_links_dict, label_to_taskid, existing_tasks):
+    """
+    Annotate each task in the graph with .optimized (boolean) and .task_id
+    (possibly None), following the rules for optimization and calling the task
+    kinds' `optimize_task` method.
+
+    As a side effect, label_to_taskid is updated with labels for all optimized
+    tasks that are replaced with existing tasks.
+    """
+
+    # set .optimized for all tasks, and .task_id for optimized tasks
+    # with replacements
+    for label in target_task_graph.graph.visit_postorder():
+        task = target_task_graph.tasks[label]
+        named_task_dependencies = named_links_dict.get(label, {})
+
+        # check whether any dependencies have been optimized away
+        dependencies = [target_task_graph.tasks[l] for l in named_task_dependencies.itervalues()]
+        for t in dependencies:
+            if t.optimized and not t.task_id:
+                raise Exception(
+                    "task {} was optimized away, but {} depends on it".format(
+                        t.label, label))
+
+        # if this task is blacklisted, don't even consider optimizing
+        replacement_task_id = None
+        if label in do_not_optimize:
+            optimized = False
+        # Let's check whether this task has been created before
+        elif existing_tasks is not None and label in existing_tasks:
+            optimized = True
+            replacement_task_id = existing_tasks[label]
+        # otherwise, examine the task itself (which may be an expensive operation)
+        else:
+            optimized, replacement_task_id = task.optimize(params)
+
+        task.optimized = optimized
+        task.task_id = replacement_task_id
+        if replacement_task_id:
+            label_to_taskid[label] = replacement_task_id
+
+        if optimized:
+            if replacement_task_id:
+                logger.debug("optimizing `{}`, replacing with task `{}`"
+                             .format(label, replacement_task_id))
+            else:
+                logger.debug("optimizing `{}` away".format(label))
+                # note: any dependent tasks will fail when they see this
+        else:
+            if replacement_task_id:
+                raise Exception("{}: optimize_task returned False with a taskId".format(label))
+
+
+def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid):
+    """
+    Return the subgraph of annotated_task_graph consisting only of
+    non-optimized tasks and edges between them.
+
+    To avoid losing track of taskIds for tasks optimized away, this method
+    simultaneously substitutes real taskIds for task labels in the graph, and
+    populates each task definition's `dependencies` key with the appropriate
+    taskIds.  Task references are resolved in the process.
+    """
+
+    # resolve labels to taskIds and populate task['dependencies']
+    tasks_by_taskid = {}
+    for label in annotated_task_graph.graph.visit_postorder():
+        task = annotated_task_graph.tasks[label]
+        if task.optimized:
+            continue
+        task.task_id = label_to_taskid[label] = slugid()
+        named_task_dependencies = {
+                name: label_to_taskid[label]
+                for name, label in named_links_dict.get(label, {}).iteritems()}
+        task.task = resolve_task_references(task.label, task.task, named_task_dependencies)
+        task.task.setdefault('dependencies', []).extend(named_task_dependencies.itervalues())
+        tasks_by_taskid[task.task_id] = task
+
+    # resolve edges to taskIds
+    edges_by_taskid = (
+        (label_to_taskid.get(left), label_to_taskid.get(right), name)
+        for (left, right, name) in annotated_task_graph.graph.edges
+        )
+    # ..and drop edges that are no longer in the task graph
+    edges_by_taskid = set(
+        (left, right, name)
+        for (left, right, name) in edges_by_taskid
+        if left in tasks_by_taskid and right in tasks_by_taskid
+        )
+
+    return TaskGraph(
+        tasks_by_taskid,
+        Graph(set(tasks_by_taskid), edges_by_taskid))
diff --git a/taskcluster/taskgraph/parameters.py b/taskcluster/taskgraph/parameters.py
new file mode 100644
index 000000000..51b9e77c7
--- /dev/null
+++ b/taskcluster/taskgraph/parameters.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import yaml
+from mozbuild.util import ReadOnlyDict
+
+# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst
+PARAMETER_NAMES = set([
+    'base_repository',
+    'build_date',
+    'head_ref',
+    'head_repository',
+    'head_rev',
+    'level',
+    'message',
+    'moz_build_date',
+    'optimize_target_tasks',
+    'owner',
+    'project',
+    'pushdate',
+    'pushlog_id',
+    'target_tasks_method',
+    'triggered_by',
+])
+
+
+class Parameters(ReadOnlyDict):
+    """An immutable dictionary with nicer KeyError messages on failure"""
+    def check(self):
+        names = set(self)
+        msg = []
+
+        missing = PARAMETER_NAMES - names
+        if missing:
+            msg.append("missing parameters: " + ", ".join(missing))
+
+        extra = names - PARAMETER_NAMES
+        if extra:
+            msg.append("extra parameters: " + ", ".join(extra))
+
+        if msg:
+            raise Exception("; ".join(msg))
+
+    def __getitem__(self, k):
+        if k not in PARAMETER_NAMES:
+            raise KeyError("no such parameter {!r}".format(k))
+        try:
+            return super(Parameters, self).__getitem__(k)
+        except KeyError:
+            raise KeyError("taskgraph parameter {!r} not found".format(k))
+
+
+def load_parameters_file(options):
+    """
+    Load parameters from the --parameters option
+    """
+    filename = options['parameters']
+    if not filename:
+        return Parameters()
+    with open(filename) as f:
+        if filename.endswith('.yml'):
+            return Parameters(**yaml.safe_load(f))
+        elif filename.endswith('.json'):
+            return Parameters(**json.load(f))
+        else:
+            raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename))
diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py
new file mode 100644
index 000000000..d2b3f5a7f
--- /dev/null
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph import try_option_syntax
+
+INTEGRATION_PROJECTS = set([
+    'mozilla-inbound',
+    'autoland',
+])
+
+RELEASE_PROJECTS = set([
+    'mozilla-central',
+    'mozilla-aurora',
+    'mozilla-beta',
+    'mozilla-release',
+    'mozilla-esr52',
+])
+
+_target_task_methods = {}
+
+
+def _target_task(name):
+    def wrap(func):
+        _target_task_methods[name] = func
+        return func
+    return wrap
+
+
+def get_method(method):
+    """Get a target_task_method to pass to a TaskGraphGenerator."""
+    return _target_task_methods[method]
+
+
+@_target_task('try_option_syntax')
+def target_tasks_try_option_syntax(full_task_graph, parameters):
+    """Generate a list of target tasks based on try syntax in
+    parameters['message'] and, for context, the full task graph."""
+    options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph)
+    target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues()
+                           if options.task_matches(t.attributes)]
+
+    # If the developer wants test jobs to be rebuilt N times we add that value here
+    if int(options.trigger_tests) > 1:
+        for l in target_tasks_labels:
+            task = full_task_graph[l]
+            if 'unittest_suite' in task.attributes:
+                task.attributes['task_duplicates'] = options.trigger_tests
+
+    # Add notifications here as well
+    if options.notifications:
+        for task in full_task_graph:
+            owner = parameters.get('owner')
+            routes = task.task.setdefault('routes', [])
+            if options.notifications == 'all':
+                routes.append("notify.email.{}.on-any".format(owner))
+            elif options.notifications == 'failure':
+                routes.append("notify.email.{}.on-failed".format(owner))
+                routes.append("notify.email.{}.on-exception".format(owner))
+
+    return target_tasks_labels
+
+
+@_target_task('default')
+def target_tasks_default(full_task_graph, parameters):
+    """Target the tasks which have indicated they should be run on this project
+    via the `run_on_projects` attributes."""
+    def filter(task):
+        run_on_projects = set(t.attributes.get('run_on_projects', []))
+        if 'all' in run_on_projects:
+            return True
+        project = parameters['project']
+        if 'integration' in run_on_projects:
+            if project in INTEGRATION_PROJECTS:
+                return True
+        if 'release' in run_on_projects:
+            if project in RELEASE_PROJECTS:
+                return True
+        return project in run_on_projects
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('ash_tasks')
+def target_tasks_ash(full_task_graph, parameters):
+    """Target tasks that only run on the ash branch."""
+    def filter(task):
+        platform = task.attributes.get('build_platform')
+        # only select platforms
+        if platform not in ('linux64', 'linux64-asan', 'linux64-pgo'):
+            return False
+        # and none of this linux64-asan/debug stuff
+        if platform == 'linux64-asan' and task.attributes['build_type'] == 'debug':
+            return False
+        # no non-et10s tests
+        if task.attributes.get('unittest_suite') or task.attributes.get('talos_siute'):
+            if not task.attributes.get('e10s'):
+                return False
+        # don't upload symbols
+        if task.attributes['kind'] == 'upload-symbols':
+            return False
+        return True
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+@_target_task('cedar_tasks')
+def target_tasks_cedar(full_task_graph, parameters):
+    """Target tasks that only run on the cedar branch."""
+    def filter(task):
+        platform = task.attributes.get('build_platform')
+        # only select platforms
+        if platform not in ['linux64']:
+            return False
+        if task.attributes.get('unittest_suite'):
+            if not (task.attributes['unittest_suite'].startswith('mochitest')
+                    or 'xpcshell' in task.attributes['unittest_suite']):
+                return False
+        return True
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
diff --git a/taskcluster/taskgraph/task/__init__.py b/taskcluster/taskgraph/task/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/task/__init__.py
diff --git a/taskcluster/taskgraph/task/base.py b/taskcluster/taskgraph/task/base.py
new file mode 100644
index 000000000..2d9cbf5d9
--- /dev/null
+++ b/taskcluster/taskgraph/task/base.py
@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import abc
+
+
+class Task(object):
+    """
+    Representation of a task in a TaskGraph.  Each Task has, at creation:
+
+    - kind: the name of the task kind
+    - label; the label for this task
+    - attributes: a dictionary of attributes for this task (used for filtering)
+    - task: the task definition (JSON-able dictionary)
+
+    And later, as the task-graph processing proceeds:
+
+    - task_id -- TaskCluster taskId under which this task will be created
+    - optimized -- true if this task need not be performed
+
+    A kind represents a collection of tasks that share common characteristics.
+    For example, all build jobs.  Each instance of a kind is intialized with a
+    path from which it draws its task configuration.  The instance is free to
+    store as much local state as it needs.
+    """
+    __metaclass__ = abc.ABCMeta
+
+    def __init__(self, kind, label, attributes, task):
+        self.kind = kind
+        self.label = label
+        self.attributes = attributes
+        self.task = task
+
+        self.task_id = None
+        self.optimized = False
+
+        self.attributes['kind'] = kind
+
+    def __eq__(self, other):
+        return self.kind == other.kind and \
+            self.label == other.label and \
+            self.attributes == other.attributes and \
+            self.task == other.task and \
+            self.task_id == other.task_id
+
+    @classmethod
+    @abc.abstractmethod
+    def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
+        """
+        Load the tasks for a given kind.
+
+        The `kind` is the name of the kind; the configuration for that kind
+        named this class.
+
+        The `path` is the path to the configuration directory for the kind.  This
+        can be used to load extra data, templates, etc.
+
+        The `parameters` give details on which to base the task generation.
+        See `taskcluster/docs/parameters.rst` for details.
+
+        At the time this method is called, all kinds on which this kind depends
+        (that is, specified in the `kind-dependencies` key in `self.config`
+        have already loaded their tasks, and those tasks are available in
+        the list `loaded_tasks`.
+
+        The return value is a list of Task instances.
+        """
+
+    @abc.abstractmethod
+    def get_dependencies(self, taskgraph):
+        """
+        Get the set of task labels this task depends on, by querying the full
+        task set, given as `taskgraph`.
+
+        Returns a list of (task_label, dependency_name) pairs describing the
+        dependencies.
+        """
+
+    def optimize(self, params):
+        """
+        Determine whether this task can be optimized, and if it can, what taskId
+        it should be replaced with.
+
+        The return value is a tuple `(optimized, taskId)`.  If `optimized` is
+        true, then the task will be optimized (in other words, not included in
+        the task graph).  If the second argument is a taskid, then any
+        dependencies on this task will isntead depend on that taskId.  It is an
+        error to return no taskId for a task on which other tasks depend.
+
+        The default never optimizes.
+        """
+        return False, None
+
+    @classmethod
+    def from_json(cls, task_dict):
+        """
+        Given a data structure as produced by taskgraph.to_json, re-construct
+        the original Task object.  This is used to "resume" the task-graph
+        generation process, for example in Action tasks.
+        """
+        return cls(
+            kind=task_dict['attributes']['kind'],
+            label=task_dict['label'],
+            attributes=task_dict['attributes'],
+            task=task_dict['task'])
diff --git a/taskcluster/taskgraph/task/docker_image.py b/taskcluster/taskgraph/task/docker_image.py
new file mode 100644
index 000000000..fd67c4832
--- /dev/null
+++ b/taskcluster/taskgraph/task/docker_image.py
@@ -0,0 +1,130 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import json
+import os
+import urllib2
+
+from . import base
+from taskgraph.util.docker import (
+    docker_image,
+    generate_context_hash,
+    INDEX_PREFIX,
+)
+from taskgraph.util.templates import Templates
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+
+class DockerImageTask(base.Task):
+
+    def __init__(self, *args, **kwargs):
+        self.index_paths = kwargs.pop('index_paths')
+        super(DockerImageTask, self).__init__(*args, **kwargs)
+
+    def __eq__(self, other):
+        return super(DockerImageTask, self).__eq__(other) and \
+               self.index_paths == other.index_paths
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params, loaded_tasks):
+        parameters = {
+            'pushlog_id': params.get('pushlog_id', 0),
+            'pushdate': params['moz_build_date'],
+            'pushtime': params['moz_build_date'][8:],
+            'year': params['moz_build_date'][0:4],
+            'month': params['moz_build_date'][4:6],
+            'day': params['moz_build_date'][6:8],
+            'project': params['project'],
+            'docker_image': docker_image,
+            'base_repository': params['base_repository'] or params['head_repository'],
+            'head_repository': params['head_repository'],
+            'head_ref': params['head_ref'] or params['head_rev'],
+            'head_rev': params['head_rev'],
+            'owner': params['owner'],
+            'level': params['level'],
+            'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
+                      .format(repo=params['head_repository'], rev=params['head_rev']),
+            'index_image_prefix': INDEX_PREFIX,
+            'artifact_path': 'public/image.tar.zst',
+        }
+
+        tasks = []
+        templates = Templates(path)
+        for image_name, image_symbol in config['images'].iteritems():
+            context_path = os.path.join('testing', 'docker', image_name)
+            context_hash = generate_context_hash(GECKO, context_path, image_name)
+
+            image_parameters = dict(parameters)
+            image_parameters['image_name'] = image_name
+            image_parameters['context_hash'] = context_hash
+
+            image_task = templates.load('image.yml', image_parameters)
+            attributes = {'image_name': image_name}
+
+            # unique symbol for different docker image
+            if 'extra' in image_task['task']:
+                image_task['task']['extra']['treeherder']['symbol'] = image_symbol
+
+            # As an optimization, if the context hash exists for a high level, that image
+            # task ID will be used.  The reasoning behind this is that eventually everything ends
+            # up on level 3 at some point if most tasks use this as a common image
+            # for a given context hash, a worker within Taskcluster does not need to contain
+            # the same image per branch.
+            index_paths = ['{}.level-{}.{}.hash.{}'.format(
+                                INDEX_PREFIX, level, image_name, context_hash)
+                           for level in range(int(params['level']), 4)]
+
+            tasks.append(cls(kind, 'build-docker-image-' + image_name,
+                             task=image_task['task'], attributes=attributes,
+                             index_paths=index_paths))
+
+        return tasks
+
+    def get_dependencies(self, taskgraph):
+        return []
+
+    def optimize(self, params):
+        for index_path in self.index_paths:
+            try:
+                url = INDEX_URL.format(index_path)
+                existing_task = json.load(urllib2.urlopen(url))
+                # Only return the task ID if the artifact exists for the indexed
+                # task.  Otherwise, continue on looking at each of the branches.  Method
+                # continues trying other branches in case mozilla-central has an expired
+                # artifact, but 'project' might not. Only return no task ID if all
+                # branches have been tried
+                request = urllib2.Request(
+                    ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))
+                request.get_method = lambda: 'HEAD'
+                urllib2.urlopen(request)
+
+                # HEAD success on the artifact is enough
+                return True, existing_task['taskId']
+            except urllib2.HTTPError:
+                pass
+
+        return False, None
+
+    @classmethod
+    def from_json(cls, task_dict):
+        # Generating index_paths for optimization
+        imgMeta = task_dict['task']['extra']['imageMeta']
+        image_name = imgMeta['imageName']
+        context_hash = imgMeta['contextHash']
+        index_paths = ['{}.level-{}.{}.hash.{}'.format(
+                            INDEX_PREFIX, level, image_name, context_hash)
+                       for level in range(int(imgMeta['level']), 4)]
+        docker_image_task = cls(kind='docker-image',
+                                label=task_dict['label'],
+                                attributes=task_dict['attributes'],
+                                task=task_dict['task'],
+                                index_paths=index_paths)
+        return docker_image_task
diff --git a/taskcluster/taskgraph/task/post_build.py b/taskcluster/taskgraph/task/post_build.py
new file mode 100644
index 000000000..09c76c44a
--- /dev/null
+++ b/taskcluster/taskgraph/task/post_build.py
@@ -0,0 +1,53 @@
+# This Source Code Form is subject to the terms of the Mozilla Public License,
+# v. 2.0. If a copy of the MPL was not distributed with this file, You can
+# obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import logging
+
+from . import transform
+from ..util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+class PostBuildTask(transform.TransformTask):
+    """
+    A task implementing a post-build job.  These depend on jobs and perform
+    various followup tasks after a build has completed.
+
+    The `only-for-build-platforms` kind configuration, if specified, will limit
+    the build platforms for which a post-build task will be created.
+
+    The `job-template' kind configuration points to a yaml file which will
+    be used to create the input to the transforms.  It will have added to it
+    keys `build-label`, the label for the build task, and `build-platform`, its
+    platform.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        if config.get('kind-dependencies', []) != ["build"]:
+            raise Exception("PostBuildTask kinds must depend on builds")
+
+        only_platforms = config.get('only-for-build-platforms')
+        prototype = load_yaml(path, config.get('job-template'))
+
+        for task in loaded_tasks:
+            if task.kind != 'build':
+                continue
+
+            build_platform = task.attributes.get('build_platform')
+            build_type = task.attributes.get('build_type')
+            if not build_platform or not build_type:
+                continue
+            platform = "{}/{}".format(build_platform, build_type)
+            if only_platforms and platform not in only_platforms:
+                continue
+
+            post_task = copy.deepcopy(prototype)
+            post_task['build-label'] = task.label
+            post_task['build-platform'] = platform
+            yield post_task
diff --git a/taskcluster/taskgraph/task/signing.py b/taskcluster/taskgraph/task/signing.py
new file mode 100644
index 000000000..a2a9ae3d6
--- /dev/null
+++ b/taskcluster/taskgraph/task/signing.py
@@ -0,0 +1,64 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import os
+
+from . import base
+from taskgraph.util.templates import Templates
+
+
+logger = logging.getLogger(__name__)
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
+
+
+class SigningTask(base.Task):
+
+    def __init__(self, kind, name, task, attributes):
+        self.unsigned_artifact_label = task['unsigned-task']['label']
+        super(SigningTask, self).__init__(kind, name, task=task['task'],
+                                          attributes=attributes)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params, loaded_tasks):
+        root = os.path.abspath(path)
+
+        tasks = []
+        for filename in config.get('jobs-from', []):
+            templates = Templates(root)
+            jobs = templates.load(filename, {})
+
+            for name, job in jobs.iteritems():
+                for artifact in job['unsigned-task']['artifacts']:
+                    url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
+                    job['task']['payload']['unsignedArtifacts'].append({
+                        'task-reference': url
+                    })
+                attributes = job.setdefault('attributes', {})
+                attributes.update({'kind': 'signing'})
+                tasks.append(cls(kind, name, job, attributes=attributes))
+
+        return tasks
+
+    def get_dependencies(self, taskgraph):
+        return [(self.unsigned_artifact_label, 'unsigned-artifact')]
+
+    def optimize(self, params):
+        return False, None
+
+    @classmethod
+    def from_json(cls, task_dict):
+        unsigned_task_label = task_dict['dependencies']['unsigned-artifact']
+        task_dict['unsigned-task'] = {
+            'label': unsigned_task_label
+        }
+        signing_task = cls(kind='build-signing',
+                           name=task_dict['label'],
+                           attributes=task_dict['attributes'],
+                           task=task_dict)
+        return signing_task
diff --git a/taskcluster/taskgraph/task/test.py b/taskcluster/taskgraph/task/test.py
new file mode 100644
index 000000000..928f32a5a
--- /dev/null
+++ b/taskcluster/taskgraph/task/test.py
@@ -0,0 +1,112 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import logging
+
+from . import transform
+from ..util.yaml import load_yaml
+
+logger = logging.getLogger(__name__)
+
+
+class TestTask(transform.TransformTask):
+    """
+    A task implementing a Gecko test.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+
+        # the kind on which this one depends
+        if len(config.get('kind-dependencies', [])) != 1:
+            raise Exception("TestTask kinds must have exactly one item in kind-dependencies")
+        dep_kind = config['kind-dependencies'][0]
+
+        # get build tasks, keyed by build platform
+        builds_by_platform = cls.get_builds_by_platform(dep_kind, loaded_tasks)
+
+        # get the test platforms for those build tasks
+        test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
+        test_platforms = cls.get_test_platforms(test_platforms_cfg, builds_by_platform)
+
+        # expand the test sets for each of those platforms
+        test_sets_cfg = load_yaml(path, 'test-sets.yml')
+        test_platforms = cls.expand_tests(test_sets_cfg, test_platforms)
+
+        # load the test descriptions
+        test_descriptions = load_yaml(path, 'tests.yml')
+
+        # generate all tests for all test platforms
+        for test_platform_name, test_platform in test_platforms.iteritems():
+            for test_name in test_platform['test-names']:
+                test = copy.deepcopy(test_descriptions[test_name])
+                test['build-platform'] = test_platform['build-platform']
+                test['test-platform'] = test_platform_name
+                test['build-label'] = test_platform['build-label']
+                test['test-name'] = test_name
+
+                logger.debug("Generating tasks for {} test {} on platform {}".format(
+                    kind, test_name, test['test-platform']))
+                yield test
+
+    @classmethod
+    def get_builds_by_platform(cls, dep_kind, loaded_tasks):
+        """Find the build tasks on which tests will depend, keyed by
+        platform/type.  Returns a dictionary mapping build platform to task
+        label."""
+        builds_by_platform = {}
+        for task in loaded_tasks:
+            if task.kind != dep_kind:
+                continue
+
+            build_platform = task.attributes.get('build_platform')
+            build_type = task.attributes.get('build_type')
+            if not build_platform or not build_type:
+                continue
+            platform = "{}/{}".format(build_platform, build_type)
+            if platform in builds_by_platform:
+                raise Exception("multiple build jobs for " + platform)
+            builds_by_platform[platform] = task.label
+        return builds_by_platform
+
+    @classmethod
+    def get_test_platforms(cls, test_platforms_cfg, builds_by_platform):
+        """Get the test platforms for which test tasks should be generated,
+        based on the available build platforms.  Returns a dictionary mapping
+        test platform to {test-set, build-platform, build-label}."""
+        test_platforms = {}
+        for test_platform, cfg in test_platforms_cfg.iteritems():
+            build_platform = cfg['build-platform']
+            if build_platform not in builds_by_platform:
+                logger.warning(
+                    "No build task with platform {}; ignoring test platform {}".format(
+                        build_platform, test_platform))
+                continue
+            test_platforms[test_platform] = {
+                'test-set': cfg['test-set'],
+                'build-platform': build_platform,
+                'build-label': builds_by_platform[build_platform],
+            }
+        return test_platforms
+
+    @classmethod
+    def expand_tests(cls, test_sets_cfg, test_platforms):
+        """Expand the test sets in `test_platforms` out to sets of test names.
+        Returns a dictionary like `get_test_platforms`, with an additional
+        `test-names` key for each test platform, containing a set of test
+        names."""
+        rv = {}
+        for test_platform, cfg in test_platforms.iteritems():
+            test_set = cfg['test-set']
+            if test_set not in test_sets_cfg:
+                raise Exception(
+                    "Test set '{}' for test platform {} is not defined".format(
+                        test_set, test_platform))
+            test_names = test_sets_cfg[test_set]
+            rv[test_platform] = cfg.copy()
+            rv[test_platform]['test-names'] = test_names
+        return rv
diff --git a/taskcluster/taskgraph/task/transform.py b/taskcluster/taskgraph/task/transform.py
new file mode 100644
index 000000000..8183254a0
--- /dev/null
+++ b/taskcluster/taskgraph/task/transform.py
@@ -0,0 +1,109 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import logging
+import itertools
+
+from . import base
+from .. import files_changed
+from ..util.python_path import find_object
+from ..util.templates import merge
+from ..util.yaml import load_yaml
+from ..util.seta import is_low_value_task
+
+from ..transforms.base import TransformSequence, TransformConfig
+
+logger = logging.getLogger(__name__)
+
+
+class TransformTask(base.Task):
+    """
+    Tasks of this class are generated by applying transformations to a sequence
+    of input entities.  By default, it gets those inputs from YAML data in the
+    kind directory, but subclasses may override `get_inputs` to produce them in
+    some other way.
+    """
+
+    @classmethod
+    def get_inputs(cls, kind, path, config, params, loaded_tasks):
+        """
+        Get the input elements that will be transformed into tasks.  The
+        elements themselves are free-form, and become the input to the first
+        transform.
+
+        By default, this reads jobs from the `jobs` key, or from yaml files
+        named by `jobs-from`.  The entities are read from mappings, and the
+        keys to those mappings are added in the `name` key of each entity.
+
+        If there is a `job-defaults` config, then every job is merged with it.
+        This provides a simple way to set default values for all jobs of a
+        kind.  More complex defaults should be implemented with custom
+        transforms.
+
+        This method can be overridden in subclasses that need to perform more
+        complex calculations to generate the list of inputs.
+        """
+        def jobs():
+            defaults = config.get('job-defaults')
+            jobs = config.get('jobs', {}).iteritems()
+            jobs_from = itertools.chain.from_iterable(
+                load_yaml(path, filename).iteritems()
+                for filename in config.get('jobs-from', {}))
+            for name, job in itertools.chain(jobs, jobs_from):
+                if defaults:
+                    job = merge(defaults, job)
+                yield name, job
+
+        for name, job in jobs():
+            job['name'] = name
+            logger.debug("Generating tasks for {} {}".format(kind, name))
+            yield job
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, params, loaded_tasks):
+        inputs = cls.get_inputs(kind, path, config, params, loaded_tasks)
+
+        transforms = TransformSequence()
+        for xform_path in config['transforms']:
+            transform = find_object(xform_path)
+            transforms.add(transform)
+
+        # perform the transformations
+        trans_config = TransformConfig(kind, path, config, params)
+        tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
+        return tasks
+
+    def __init__(self, kind, task):
+        self.dependencies = task['dependencies']
+        self.when = task['when']
+        super(TransformTask, self).__init__(kind, task['label'],
+                                            task['attributes'], task['task'])
+
+    def get_dependencies(self, taskgraph):
+        return [(label, name) for name, label in self.dependencies.items()]
+
+    def optimize(self, params):
+        if 'files-changed' in self.when:
+            changed = files_changed.check(
+                params, self.when['files-changed'])
+            if not changed:
+                logger.debug('no files found matching a pattern in `when.files-changed` for ' +
+                             self.label)
+                return True, None
+
+        # we would like to return 'False, None' while it's high_value_task
+        # and we wouldn't optimize it. Otherwise, it will return 'True, None'
+        if is_low_value_task(self.label, params.get('project')):
+            # Always optimize away low-value tasks
+            return True, None
+        else:
+            return False, None
+
+    @classmethod
+    def from_json(cls, task_dict):
+        # when reading back from JSON, we lose the "when" information
+        task_dict['when'] = {}
+        return cls(task_dict['attributes']['kind'], task_dict)
diff --git a/taskcluster/taskgraph/taskgraph.py b/taskcluster/taskgraph/taskgraph.py
new file mode 100644
index 000000000..7736745ef
--- /dev/null
+++ b/taskcluster/taskgraph/taskgraph.py
@@ -0,0 +1,82 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+from .graph import Graph
+from .util.python_path import find_object
+
+TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task/"
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
+
+
+class TaskGraph(object):
+    """
+    Representation of a task graph.
+
+    A task graph is a combination of a Graph and a dictionary of tasks indexed
+    by label.  TaskGraph instances should be treated as immutable.
+    """
+
+    def __init__(self, tasks, graph):
+        assert set(tasks) == graph.nodes
+        self.tasks = tasks
+        self.graph = graph
+
+    def to_json(self):
+        "Return a JSON-able object representing the task graph, as documented"
+        named_links_dict = self.graph.named_links_dict()
+        # this dictionary may be keyed by label or by taskid, so let's just call it 'key'
+        tasks = {}
+        for key in self.graph.visit_postorder():
+            task = self.tasks[key]
+            implementation = task.__class__.__module__ + ":" + task.__class__.__name__
+            task_json = {
+                'label': task.label,
+                'attributes': task.attributes,
+                'dependencies': named_links_dict.get(key, {}),
+                'task': task.task,
+                'kind_implementation': implementation
+            }
+            if task.task_id:
+                task_json['task_id'] = task.task_id
+            tasks[key] = task_json
+        return tasks
+
+    def __getitem__(self, label):
+        "Get a task by label"
+        return self.tasks[label]
+
+    def __iter__(self):
+        "Iterate over tasks in undefined order"
+        return self.tasks.itervalues()
+
+    def __repr__(self):
+        return "<TaskGraph graph={!r} tasks={!r}>".format(self.graph, self.tasks)
+
+    def __eq__(self, other):
+        return self.tasks == other.tasks and self.graph == other.graph
+
+    @classmethod
+    def from_json(cls, tasks_dict):
+        """
+        This code is used to generate the a TaskGraph using a dictionary
+        which is representative of the TaskGraph.
+        """
+        tasks = {}
+        edges = set()
+        for key, value in tasks_dict.iteritems():
+            # We get the implementation from JSON
+            implementation = value['kind_implementation']
+            # Loading the module and creating a Task from a dictionary
+            task_kind = find_object(implementation)
+            tasks[key] = task_kind.from_json(value)
+            if 'task_id' in value:
+                tasks[key].task_id = value['task_id']
+            for depname, dep in value['dependencies'].iteritems():
+                edges.add((key, dep, depname))
+        task_graph = cls(tasks, Graph(set(tasks), edges))
+        return tasks, task_graph
diff --git a/taskcluster/taskgraph/test/__init__.py b/taskcluster/taskgraph/test/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/test/__init__.py
diff --git a/taskcluster/taskgraph/test/automationrelevance.json b/taskcluster/taskgraph/test/automationrelevance.json
new file mode 100644
index 000000000..8adfc446d
--- /dev/null
+++ b/taskcluster/taskgraph/test/automationrelevance.json
@@ -0,0 +1,425 @@
+{
+    "changesets": [
+        {
+            "author": "James Long <longster@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300866",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
+                }
+            ],
+            "date": [
+                1473196655.0,
+                14400
+            ],
+            "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/debugger/new/index.html"
+            ],
+            "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "parents": [
+                "37c9349b4e8167a61b08b7e119c21ea177b98942"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312890,
+            "reviewers": [
+                {
+                    "name": "jlast",
+                    "revset": "reviewer(jlast)"
+                },
+                {
+                    "name": "bgrins",
+                    "revset": "reviewer(bgrins)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Wes Kocher <wkocher@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473208638.0,
+                25200
+            ],
+            "desc": "Merge m-c to fx-team, a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "taskcluster/scripts/builder/build-l10n.sh"
+            ],
+            "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+            "parents": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312891,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Towkir Ahmed <towkir17@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1296648",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
+                }
+            ],
+            "date": [
+                1472957580.0,
+                14400
+            ],
+            "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/themes/rules.css"
+            ],
+            "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+            "parents": [
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312892,
+            "reviewers": [
+                {
+                    "name": "ntim",
+                    "revset": "reviewer(ntim)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Oriol <oriol-bugzilla@hotmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1300336",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
+                }
+            ],
+            "date": [
+                1472921160.0,
+                14400
+            ],
+            "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/webconsole/test/browser_webconsole_output_06.js",
+                "devtools/server/actors/object.js"
+            ],
+            "node": "99c542fa43a72ee863c813b5624048d1b443549b",
+            "parents": [
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312893,
+            "reviewers": [
+                {
+                    "name": "fitzgen",
+                    "revset": "reviewer(fitzgen)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Ruturaj Vartak <ruturaj@gmail.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1472854020.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
+            "extra": {
+                "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
+                "branch": "default",
+                "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
+                "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js",
+                "devtools/server/actors/highlighters/eye-dropper.js"
+            ],
+            "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+            "parents": [
+                "99c542fa43a72ee863c813b5624048d1b443549b"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312894,
+            "reviewers": [
+                {
+                    "name": "pbro",
+                    "revset": "reviewer(pbro)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Matteo Ferretti <mferretti@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1299154",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
+                }
+            ],
+            "date": [
+                1472629906.0,
+                -7200
+            ],
+            "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
+            "extra": {
+                "branch": "default",
+                "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
+            },
+            "files": [
+                "docshell/base/nsDocShell.cpp",
+                "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
+            ],
+            "node": "541c9086c0f27fba60beecc9bc94543103895c86",
+            "parents": [
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312895,
+            "reviewers": [
+                {
+                    "name": "mstange",
+                    "revset": "reviewer(mstange)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Patrick Brosset <pbrosset@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [
+                {
+                    "no": "1295010",
+                    "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
+                }
+            ],
+            "date": [
+                1473239449.0,
+                -7200
+            ],
+            "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [
+                "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
+                "devtools/client/inspector/test/head.js"
+            ],
+            "node": "041a925171e431bf51fb50193ab19d156088c89a",
+            "parents": [
+                "541c9086c0f27fba60beecc9bc94543103895c86"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312896,
+            "reviewers": [
+                {
+                    "name": "me",
+                    "revset": "reviewer(me)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        },
+        {
+            "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
+            "backsoutnodes": [],
+            "bugs": [],
+            "date": [
+                1473261233.0,
+                -7200
+            ],
+            "desc": "merge fx-team to mozilla-central a=merge",
+            "extra": {
+                "branch": "default"
+            },
+            "files": [],
+            "node": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "parents": [
+                "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
+                "041a925171e431bf51fb50193ab19d156088c89a"
+            ],
+            "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+            "pushdate": [
+                1473261248,
+                0
+            ],
+            "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
+            "pushid": 30664,
+            "pushnodes": [
+                "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
+                "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
+                "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
+                "99c542fa43a72ee863c813b5624048d1b443549b",
+                "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
+                "541c9086c0f27fba60beecc9bc94543103895c86",
+                "041a925171e431bf51fb50193ab19d156088c89a",
+                "a14f88a9af7a59e677478694bafd9375ac53683e"
+            ],
+            "pushuser": "cbook@mozilla.com",
+            "rev": 312897,
+            "reviewers": [
+                {
+                    "name": "merge",
+                    "revset": "reviewer(merge)"
+                }
+            ],
+            "treeherderrepo": "mozilla-central",
+            "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
+        }
+    ],
+    "visible": true
+}
+
diff --git a/taskcluster/taskgraph/test/test_create.py b/taskcluster/taskgraph/test/test_create.py
new file mode 100644
index 000000000..b8da3aec0
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_create.py
@@ -0,0 +1,76 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import os
+
+from .. import create
+from ..graph import Graph
+from ..taskgraph import TaskGraph
+from .util import TestTask
+
+from mozunit import main
+
+
+class TestCreate(unittest.TestCase):
+
+    def setUp(self):
+        self.old_task_id = os.environ.get('TASK_ID')
+        if 'TASK_ID' in os.environ:
+            del os.environ['TASK_ID']
+        self.created_tasks = {}
+        self.old_create_task = create._create_task
+        create._create_task = self.fake_create_task
+
+    def tearDown(self):
+        create._create_task = self.old_create_task
+        if self.old_task_id:
+            os.environ['TASK_ID'] = self.old_task_id
+        elif 'TASK_ID' in os.environ:
+            del os.environ['TASK_ID']
+
+    def fake_create_task(self, session, task_id, label, task_def):
+        self.created_tasks[task_id] = task_def
+
+    def test_create_tasks(self):
+        tasks = {
+            'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
+            'tid-b': TestTask(label='b', task={'payload': 'hello world'}),
+        }
+        label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
+        graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')})
+        taskgraph = TaskGraph(tasks, graph)
+
+        create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
+
+        for tid, task in self.created_tasks.iteritems():
+            self.assertEqual(task['payload'], 'hello world')
+            self.assertEqual(task['schedulerId'], 'gecko-level-4')
+            # make sure the dependencies exist, at least
+            for depid in task.get('dependencies', []):
+                if depid is 'decisiontask':
+                    # Don't look for decisiontask here
+                    continue
+                self.assertIn(depid, self.created_tasks)
+
+    def test_create_task_without_dependencies(self):
+        "a task with no dependencies depends on the decision task"
+        os.environ['TASK_ID'] = 'decisiontask'
+        tasks = {
+            'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
+        }
+        label_to_taskid = {'a': 'tid-a'}
+        graph = Graph(nodes={'tid-a'}, edges=set())
+        taskgraph = TaskGraph(tasks, graph)
+
+        create.create_tasks(taskgraph, label_to_taskid, {'level': '4'})
+
+        for tid, task in self.created_tasks.iteritems():
+            self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
+
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_decision.py b/taskcluster/taskgraph/test/test_decision.py
new file mode 100644
index 000000000..364965206
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_decision.py
@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import json
+import yaml
+import shutil
+import unittest
+import tempfile
+
+from .. import decision
+from ..graph import Graph
+from ..taskgraph import TaskGraph
+from .util import TestTask
+from mozunit import main
+
+
+class TestDecision(unittest.TestCase):
+
+    def test_taskgraph_to_json(self):
+        tasks = {
+            'a': TestTask(label='a', attributes={'attr': 'a-task'}),
+            'b': TestTask(label='b', task={'task': 'def'}),
+        }
+        graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
+        taskgraph = TaskGraph(tasks, graph)
+
+        res = taskgraph.to_json()
+
+        self.assertEqual(res, {
+            'a': {
+                'label': 'a',
+                'attributes': {'attr': 'a-task', 'kind': 'test'},
+                'task': {},
+                'dependencies': {'edgelabel': 'b'},
+                'kind_implementation': 'taskgraph.test.util:TestTask',
+            },
+            'b': {
+                'label': 'b',
+                'attributes': {'kind': 'test'},
+                'task': {'task': 'def'},
+                'dependencies': {},
+                'kind_implementation': 'taskgraph.test.util:TestTask',
+            }
+        })
+
+    def test_write_artifact_json(self):
+        data = [{'some': 'data'}]
+        tmpdir = tempfile.mkdtemp()
+        try:
+            decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+            decision.write_artifact("artifact.json", data)
+            with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f:
+                self.assertEqual(json.load(f), data)
+        finally:
+            if os.path.exists(tmpdir):
+                shutil.rmtree(tmpdir)
+            decision.ARTIFACTS_DIR = 'artifacts'
+
+    def test_write_artifact_yml(self):
+        data = [{'some': 'data'}]
+        tmpdir = tempfile.mkdtemp()
+        try:
+            decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts")
+            decision.write_artifact("artifact.yml", data)
+            with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f:
+                self.assertEqual(yaml.safe_load(f), data)
+        finally:
+            if os.path.exists(tmpdir):
+                shutil.rmtree(tmpdir)
+            decision.ARTIFACTS_DIR = 'artifacts'
+
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_files_changed.py b/taskcluster/taskgraph/test/test_files_changed.py
new file mode 100644
index 000000000..0e3366b3c
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_files_changed.py
@@ -0,0 +1,73 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import os
+import unittest
+
+from .. import files_changed
+
+PARAMS = {
+    'head_repository': 'https://hg.mozilla.org/mozilla-central',
+    'head_rev': 'a14f88a9af7a',
+}
+
+FILES_CHANGED = [
+    'devtools/client/debugger/new/index.html',
+    'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js',
+    'devtools/client/inspector/test/head.js',
+    'devtools/client/themes/rules.css',
+    'devtools/client/webconsole/test/browser_webconsole_output_06.js',
+    'devtools/server/actors/highlighters/eye-dropper.js',
+    'devtools/server/actors/object.js',
+    'docshell/base/nsDocShell.cpp',
+    'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html',
+    'taskcluster/scripts/builder/build-l10n.sh',
+]
+
+
+class FakeResponse:
+
+    def json(self):
+        with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f:
+            return json.load(f)
+
+
+class TestGetChangedFiles(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache.clear()
+        self.old_get = files_changed.requests.get
+
+        def fake_get(url, **kwargs):
+            return FakeResponse()
+        files_changed.requests.get = fake_get
+
+    def tearDown(self):
+        files_changed.requests.get = self.old_get
+
+    def test_get_changed_files(self):
+        """Get_changed_files correctly gets the list of changed files in a push.
+        This tests against the production hg.mozilla.org so that it will detect
+        any changes in the format of the returned data."""
+        self.assertEqual(
+            sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])),
+            FILES_CHANGED)
+
+
+class TestCheck(unittest.TestCase):
+
+    def setUp(self):
+        files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED
+
+    def test_check_no_params(self):
+        self.assertTrue(files_changed.check({}, ["ignored"]))
+
+    def test_check_no_match(self):
+        self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
+
+    def test_check_match(self):
+        self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))
diff --git a/taskcluster/taskgraph/test/test_generator.py b/taskcluster/taskgraph/test/test_generator.py
new file mode 100644
index 000000000..f1b466e4d
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_generator.py
@@ -0,0 +1,129 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..generator import TaskGraphGenerator, Kind
+from .. import graph
+from ..task import base
+from mozunit import main
+
+
+class FakeTask(base.Task):
+
+    def __init__(self, **kwargs):
+        self.i = kwargs.pop('i')
+        super(FakeTask, self).__init__(**kwargs)
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
+        return [cls(kind=kind,
+                    label='{}-t-{}'.format(kind, i),
+                    attributes={'tasknum': str(i)},
+                    task={},
+                    i=i)
+                for i in range(3)]
+
+    def get_dependencies(self, full_task_set):
+        i = self.i
+        if i > 0:
+            return [('{}-t-{}'.format(self.kind, i - 1), 'prev')]
+        else:
+            return []
+
+    def optimize(self, params):
+        return False, None
+
+
+class FakeKind(Kind):
+
+    def _get_impl_class(self):
+        return FakeTask
+
+    def load_tasks(self, parameters, loaded_tasks):
+        FakeKind.loaded_kinds.append(self.name)
+        return super(FakeKind, self).load_tasks(parameters, loaded_tasks)
+
+
+class WithFakeKind(TaskGraphGenerator):
+
+    def _load_kinds(self):
+        for kind_name, deps in self.parameters['kinds']:
+            yield FakeKind(
+                kind_name, '/fake',
+                {'kind-dependencies': deps} if deps else {})
+
+
+class TestGenerator(unittest.TestCase):
+
+    def maketgg(self, target_tasks=None, kinds=[('fake', [])]):
+        FakeKind.loaded_kinds = []
+        self.target_tasks = target_tasks or []
+
+        def target_tasks_method(full_task_graph, parameters):
+            return self.target_tasks
+        return WithFakeKind('/root', {'kinds': kinds}, target_tasks_method)
+
+    def test_kind_ordering(self):
+        "When task kinds depend on each other, they are loaded in postorder"
+        self.tgg = self.maketgg(kinds=[
+            ('fake3', ['fake2', 'fake1']),
+            ('fake2', ['fake1']),
+            ('fake1', []),
+        ])
+        self.tgg._run_until('full_task_set')
+        self.assertEqual(FakeKind.loaded_kinds, ['fake1', 'fake2', 'fake3'])
+
+    def test_full_task_set(self):
+        "The full_task_set property has all tasks"
+        self.tgg = self.maketgg()
+        self.assertEqual(self.tgg.full_task_set.graph,
+                         graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'}, set()))
+        self.assertEqual(sorted(self.tgg.full_task_set.tasks.keys()),
+                         sorted(['fake-t-0', 'fake-t-1', 'fake-t-2']))
+
+    def test_full_task_graph(self):
+        "The full_task_graph property has all tasks, and links"
+        self.tgg = self.maketgg()
+        self.assertEqual(self.tgg.full_task_graph.graph,
+                         graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'},
+                                     {
+                                         ('fake-t-1', 'fake-t-0', 'prev'),
+                                         ('fake-t-2', 'fake-t-1', 'prev'),
+                         }))
+        self.assertEqual(sorted(self.tgg.full_task_graph.tasks.keys()),
+                         sorted(['fake-t-0', 'fake-t-1', 'fake-t-2']))
+
+    def test_target_task_set(self):
+        "The target_task_set property has the targeted tasks"
+        self.tgg = self.maketgg(['fake-t-1'])
+        self.assertEqual(self.tgg.target_task_set.graph,
+                         graph.Graph({'fake-t-1'}, set()))
+        self.assertEqual(self.tgg.target_task_set.tasks.keys(),
+                         ['fake-t-1'])
+
+    def test_target_task_graph(self):
+        "The target_task_graph property has the targeted tasks and deps"
+        self.tgg = self.maketgg(['fake-t-1'])
+        self.assertEqual(self.tgg.target_task_graph.graph,
+                         graph.Graph({'fake-t-0', 'fake-t-1'},
+                                     {('fake-t-1', 'fake-t-0', 'prev')}))
+        self.assertEqual(sorted(self.tgg.target_task_graph.tasks.keys()),
+                         sorted(['fake-t-0', 'fake-t-1']))
+
+    def test_optimized_task_graph(self):
+        "The optimized task graph contains task ids"
+        self.tgg = self.maketgg(['fake-t-2'])
+        tid = self.tgg.label_to_taskid
+        self.assertEqual(
+            self.tgg.optimized_task_graph.graph,
+            graph.Graph({tid['fake-t-0'], tid['fake-t-1'], tid['fake-t-2']}, {
+                (tid['fake-t-1'], tid['fake-t-0'], 'prev'),
+                (tid['fake-t-2'], tid['fake-t-1'], 'prev'),
+            }))
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_graph.py b/taskcluster/taskgraph/test/test_graph.py
new file mode 100644
index 000000000..5c4c950a7
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_graph.py
@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..graph import Graph
+from mozunit import main
+
+
+class TestGraph(unittest.TestCase):
+
+    tree = Graph(set(['a', 'b', 'c', 'd', 'e', 'f', 'g']), {
+        ('a', 'b', 'L'),
+        ('a', 'c', 'L'),
+        ('b', 'd', 'K'),
+        ('b', 'e', 'K'),
+        ('c', 'f', 'N'),
+        ('c', 'g', 'N'),
+    })
+
+    linear = Graph(set(['1', '2', '3', '4']), {
+        ('1', '2', 'L'),
+        ('2', '3', 'L'),
+        ('3', '4', 'L'),
+    })
+
+    diamonds = Graph(set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']),
+                     set(tuple(x) for x in
+                         'AFL ADL BDL BEL CEL CHL DFL DGL EGL EHL FIL GIL GJL HJL'.split()
+                         ))
+
+    multi_edges = Graph(set(['1', '2', '3', '4']), {
+        ('2', '1', 'red'),
+        ('2', '1', 'blue'),
+        ('3', '1', 'red'),
+        ('3', '2', 'blue'),
+        ('3', '2', 'green'),
+        ('4', '3', 'green'),
+    })
+
+    disjoint = Graph(set(['1', '2', '3', '4', 'α', 'β', 'γ']), {
+        ('2', '1', 'red'),
+        ('3', '1', 'red'),
+        ('3', '2', 'green'),
+        ('4', '3', 'green'),
+        ('α', 'β', 'πράσινο'),
+        ('β', 'γ', 'κόκκινο'),
+        ('α', 'γ', 'μπλε'),
+    })
+
+    def test_transitive_closure_empty(self):
+        "transitive closure of an empty set is an empty graph"
+        g = Graph(set(['a', 'b', 'c']), {('a', 'b', 'L'), ('a', 'c', 'L')})
+        self.assertEqual(g.transitive_closure(set()),
+                         Graph(set(), set()))
+
+    def test_transitive_closure_disjoint(self):
+        "transitive closure of a disjoint set is a subset"
+        g = Graph(set(['a', 'b', 'c']), set())
+        self.assertEqual(g.transitive_closure(set(['a', 'c'])),
+                         Graph(set(['a', 'c']), set()))
+
+    def test_transitive_closure_trees(self):
+        "transitive closure of a tree, at two non-root nodes, is the two subtrees"
+        self.assertEqual(self.tree.transitive_closure(set(['b', 'c'])),
+                         Graph(set(['b', 'c', 'd', 'e', 'f', 'g']), {
+                             ('b', 'd', 'K'),
+                             ('b', 'e', 'K'),
+                             ('c', 'f', 'N'),
+                             ('c', 'g', 'N'),
+                         }))
+
+    def test_transitive_closure_multi_edges(self):
+        "transitive closure of a tree with multiple edges between nodes keeps those edges"
+        self.assertEqual(self.multi_edges.transitive_closure(set(['3'])),
+                         Graph(set(['1', '2', '3']), {
+                             ('2', '1', 'red'),
+                             ('2', '1', 'blue'),
+                             ('3', '1', 'red'),
+                             ('3', '2', 'blue'),
+                             ('3', '2', 'green'),
+                         }))
+
+    def test_transitive_closure_disjoint_edges(self):
+        "transitive closure of a disjoint graph keeps those edges"
+        self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])),
+                         Graph(set(['1', '2', '3', 'β', 'γ']), {
+                             ('2', '1', 'red'),
+                             ('3', '1', 'red'),
+                             ('3', '2', 'green'),
+                             ('β', 'γ', 'κόκκινο'),
+                         }))
+
+    def test_transitive_closure_linear(self):
+        "transitive closure of a linear graph includes all nodes in the line"
+        self.assertEqual(self.linear.transitive_closure(set(['1'])), self.linear)
+
+    def test_visit_postorder_empty(self):
+        "postorder visit of an empty graph is empty"
+        self.assertEqual(list(Graph(set(), set()).visit_postorder()), [])
+
+    def assert_postorder(self, seq, all_nodes):
+        seen = set()
+        for e in seq:
+            for l, r, n in self.tree.edges:
+                if l == e:
+                    self.failUnless(r in seen)
+            seen.add(e)
+        self.assertEqual(seen, all_nodes)
+
+    def test_visit_postorder_tree(self):
+        "postorder visit of a tree satisfies invariant"
+        self.assert_postorder(self.tree.visit_postorder(), self.tree.nodes)
+
+    def test_visit_postorder_diamonds(self):
+        "postorder visit of a graph full of diamonds satisfies invariant"
+        self.assert_postorder(self.diamonds.visit_postorder(), self.diamonds.nodes)
+
+    def test_visit_postorder_multi_edges(self):
+        "postorder visit of a graph with duplicate edges satisfies invariant"
+        self.assert_postorder(self.multi_edges.visit_postorder(), self.multi_edges.nodes)
+
+    def test_visit_postorder_disjoint(self):
+        "postorder visit of a disjoint graph satisfies invariant"
+        self.assert_postorder(self.disjoint.visit_postorder(), self.disjoint.nodes)
+
+    def test_links_dict(self):
+        "link dict for a graph with multiple edges is correct"
+        self.assertEqual(self.multi_edges.links_dict(), {
+            '2': set(['1']),
+            '3': set(['1', '2']),
+            '4': set(['3']),
+        })
+
+    def test_named_links_dict(self):
+        "named link dict for a graph with multiple edges is correct"
+        self.assertEqual(self.multi_edges.named_links_dict(), {
+            '2': dict(red='1', blue='1'),
+            '3': dict(red='1', blue='2', green='2'),
+            '4': dict(green='3'),
+        })
+
+    def test_reverse_links_dict(self):
+        "reverse link dict for a graph with multiple edges is correct"
+        self.assertEqual(self.multi_edges.reverse_links_dict(), {
+            '1': set(['2', '3']),
+            '2': set(['3']),
+            '3': set(['4']),
+        })
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_optimize.py b/taskcluster/taskgraph/test/test_optimize.py
new file mode 100644
index 000000000..8d2ddf247
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_optimize.py
@@ -0,0 +1,256 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..optimize import optimize_task_graph, resolve_task_references
+from ..optimize import annotate_task_graph, get_subgraph
+from ..taskgraph import TaskGraph
+from .. import graph
+from .util import TestTask
+
+
+class TestResolveTaskReferences(unittest.TestCase):
+
+    def do(self, input, output):
+        taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)}
+        self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), output)
+
+    def test_in_list(self):
+        "resolve_task_references resolves task references in a list"
+        self.do({'in-a-list': ['stuff', {'task-reference': '<edge1>'}]},
+                {'in-a-list': ['stuff', 'tid1']})
+
+    def test_in_dict(self):
+        "resolve_task_references resolves task references in a dict"
+        self.do({'in-a-dict': {'stuff': {'task-reference': '<edge2>'}}},
+                {'in-a-dict': {'stuff': 'tid2'}})
+
+    def test_multiple(self):
+        "resolve_task_references resolves multiple references in the same string"
+        self.do({'multiple': {'task-reference': 'stuff <edge1> stuff <edge2> after'}},
+                {'multiple': 'stuff tid1 stuff tid2 after'})
+
+    def test_embedded(self):
+        "resolve_task_references resolves ebmedded references"
+        self.do({'embedded': {'task-reference': 'stuff before <edge3> stuff after'}},
+                {'embedded': 'stuff before tid3 stuff after'})
+
+    def test_escaping(self):
+        "resolve_task_references resolves escapes in task references"
+        self.do({'escape': {'task-reference': '<<><edge3>>'}},
+                {'escape': '<tid3>'})
+
+    def test_invalid(self):
+        "resolve_task_references raises a KeyError on reference to an invalid task"
+        self.assertRaisesRegexp(
+            KeyError,
+            "task 'subject' has no dependency named 'no-such'",
+            lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {})
+        )
+
+
+class OptimizingTask(TestTask):
+    # the `optimize` method on this class is overridden direclty in the tests
+    # below.
+    pass
+
+
+class TestOptimize(unittest.TestCase):
+
+    kind = None
+
+    def make_task(self, label, task_def=None, optimized=None, task_id=None):
+        task_def = task_def or {'sample': 'task-def'}
+        task = OptimizingTask(label=label, task=task_def)
+        task.optimized = optimized
+        task.task_id = task_id
+        return task
+
+    def make_graph(self, *tasks_and_edges):
+        tasks = {t.label: t for t in tasks_and_edges if isinstance(t, OptimizingTask)}
+        edges = {e for e in tasks_and_edges if not isinstance(e, OptimizingTask)}
+        return TaskGraph(tasks, graph.Graph(set(tasks), edges))
+
+    def assert_annotations(self, graph, **annotations):
+        def repl(task_id):
+            return 'SLUGID' if task_id and len(task_id) == 22 else task_id
+        got_annotations = {
+            t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
+        }
+        self.assertEqual(got_annotations, annotations)
+
+    def test_annotate_task_graph_no_optimize(self):
+        "annotating marks everything as un-optimized if the kind returns that"
+        OptimizingTask.optimize = lambda self, params: (False, None)
+        graph = self.make_graph(
+            self.make_task('task1'),
+            self.make_task('task2'),
+            self.make_task('task3'),
+            ('task2', 'task1', 'build'),
+            ('task2', 'task3', 'image'),
+        )
+        annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
+        self.assert_annotations(
+            graph,
+            task1=(False, None),
+            task2=(False, None),
+            task3=(False, None)
+        )
+
+    def test_annotate_task_graph_taskid_without_optimize(self):
+        "raises exception if kind returns a taskid without optimizing"
+        OptimizingTask.optimize = lambda self, params: (False, 'some-taskid')
+        graph = self.make_graph(self.make_task('task1'))
+        self.assertRaises(
+            Exception,
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
+        )
+
+    def test_annotate_task_graph_optimize_away_dependency(self):
+        "raises exception if kind optimizes away a task on which another depends"
+        OptimizingTask.optimize = \
+            lambda self, params: (True, None) if self.label == 'task1' else (False, None)
+        graph = self.make_graph(
+            self.make_task('task1'),
+            self.make_task('task2'),
+            ('task2', 'task1', 'build'),
+        )
+        self.assertRaises(
+            Exception,
+            lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
+        )
+
+    def test_annotate_task_graph_do_not_optimize(self):
+        "annotating marks everything as un-optimized if in do_not_optimize"
+        OptimizingTask.optimize = lambda self, params: (True, 'taskid')
+        graph = self.make_graph(
+            self.make_task('task1'),
+            self.make_task('task2'),
+            ('task2', 'task1', 'build'),
+        )
+        label_to_taskid = {}
+        annotate_task_graph(graph, {}, {'task1', 'task2'},
+                            graph.graph.named_links_dict(), label_to_taskid, None)
+        self.assert_annotations(
+            graph,
+            task1=(False, None),
+            task2=(False, None)
+        )
+        self.assertEqual
+
+    def test_annotate_task_graph_nos_do_not_propagate(self):
+        "a task with a non-optimized dependency can be optimized"
+        OptimizingTask.optimize = \
+            lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid')
+        graph = self.make_graph(
+            self.make_task('task1'),
+            self.make_task('task2'),
+            self.make_task('task3'),
+            ('task2', 'task1', 'build'),
+            ('task2', 'task3', 'image'),
+        )
+        annotate_task_graph(graph, {}, set(),
+                            graph.graph.named_links_dict(), {}, None)
+        self.assert_annotations(
+            graph,
+            task1=(False, None),
+            task2=(True, 'taskid'),
+            task3=(True, 'taskid')
+        )
+
+    def test_get_subgraph_single_dep(self):
+        "when a single dependency is optimized, it is omitted from the graph"
+        graph = self.make_graph(
+            self.make_task('task1', optimized=True, task_id='dep1'),
+            self.make_task('task2', optimized=False),
+            self.make_task('task3', optimized=False),
+            ('task2', 'task1', 'build'),
+            ('task2', 'task3', 'image'),
+        )
+        label_to_taskid = {'task1': 'dep1'}
+        sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid)
+        task2 = label_to_taskid['task2']
+        task3 = label_to_taskid['task3']
+        self.assertEqual(sub.graph.nodes, {task2, task3})
+        self.assertEqual(sub.graph.edges, {(task2, task3, 'image')})
+        self.assertEqual(sub.tasks[task2].task_id, task2)
+        self.assertEqual(sorted(sub.tasks[task2].task['dependencies']),
+                         sorted([task3, 'dep1']))
+        self.assertEqual(sub.tasks[task3].task_id, task3)
+        self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), [])
+
+    def test_get_subgraph_dep_chain(self):
+        "when a dependency chain is optimized, it is omitted from the graph"
+        graph = self.make_graph(
+            self.make_task('task1', optimized=True, task_id='dep1'),
+            self.make_task('task2', optimized=True, task_id='dep2'),
+            self.make_task('task3', optimized=False),
+            ('task2', 'task1', 'build'),
+            ('task3', 'task2', 'image'),
+        )
+        label_to_taskid = {'task1': 'dep1', 'task2': 'dep2'}
+        sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid)
+        task3 = label_to_taskid['task3']
+        self.assertEqual(sub.graph.nodes, {task3})
+        self.assertEqual(sub.graph.edges, set())
+        self.assertEqual(sub.tasks[task3].task_id, task3)
+        self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), ['dep2'])
+
+    def test_get_subgraph_opt_away(self):
+        "when a leaf task is optimized away, it is omitted from the graph"
+        graph = self.make_graph(
+            self.make_task('task1', optimized=False),
+            self.make_task('task2', optimized=True),
+            ('task2', 'task1', 'build'),
+        )
+        label_to_taskid = {'task2': 'dep2'}
+        sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid)
+        task1 = label_to_taskid['task1']
+        self.assertEqual(sub.graph.nodes, {task1})
+        self.assertEqual(sub.graph.edges, set())
+        self.assertEqual(sub.tasks[task1].task_id, task1)
+        self.assertEqual(sorted(sub.tasks[task1].task['dependencies']), [])
+
+    def test_get_subgraph_refs_resolved(self):
+        "get_subgraph resolves task references"
+        graph = self.make_graph(
+            self.make_task('task1', optimized=True, task_id='dep1'),
+            self.make_task(
+                'task2',
+                optimized=False,
+                task_def={'payload': {'task-reference': 'http://<build>/<test>'}}
+            ),
+            ('task2', 'task1', 'build'),
+            ('task2', 'task3', 'test'),
+            self.make_task('task3', optimized=False),
+        )
+        label_to_taskid = {'task1': 'dep1'}
+        sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid)
+        task2 = label_to_taskid['task2']
+        task3 = label_to_taskid['task3']
+        self.assertEqual(sub.graph.nodes, {task2, task3})
+        self.assertEqual(sub.graph.edges, {(task2, task3, 'test')})
+        self.assertEqual(sub.tasks[task2].task_id, task2)
+        self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1']))
+        self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3)
+        self.assertEqual(sub.tasks[task3].task_id, task3)
+
+    def test_optimize(self):
+        "optimize_task_graph annotates and extracts the subgraph from a simple graph"
+        OptimizingTask.optimize = \
+            lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None)
+        input = self.make_graph(
+            self.make_task('task1'),
+            self.make_task('task2'),
+            self.make_task('task3'),
+            ('task2', 'task1', 'build'),
+            ('task2', 'task3', 'image'),
+        )
+        opt, label_to_taskid = optimize_task_graph(input, {}, set())
+        self.assertEqual(opt.graph, graph.Graph(
+            {label_to_taskid['task2'], label_to_taskid['task3']},
+            {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))
diff --git a/taskcluster/taskgraph/test/test_parameters.py b/taskcluster/taskgraph/test/test_parameters.py
new file mode 100644
index 000000000..43d853d7b
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_parameters.py
@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..parameters import Parameters, load_parameters_file, PARAMETER_NAMES
+from mozunit import main, MockedOpen
+
+
+class TestParameters(unittest.TestCase):
+
+    vals = {n: n for n in PARAMETER_NAMES}
+
+    def test_Parameters_immutable(self):
+        p = Parameters(**self.vals)
+
+        def assign():
+            p['head_ref'] = 20
+        self.assertRaises(Exception, assign)
+
+    def test_Parameters_missing_KeyError(self):
+        p = Parameters(**self.vals)
+        self.assertRaises(KeyError, lambda: p['z'])
+
+    def test_Parameters_invalid_KeyError(self):
+        """even if the value is present, if it's not a valid property, raise KeyError"""
+        p = Parameters(xyz=10, **self.vals)
+        self.assertRaises(KeyError, lambda: p['xyz'])
+
+    def test_Parameters_get(self):
+        p = Parameters(head_ref=10, level=20)
+        self.assertEqual(p['head_ref'], 10)
+
+    def test_Parameters_check(self):
+        p = Parameters(**self.vals)
+        p.check()  # should not raise
+
+    def test_Parameters_check_missing(self):
+        p = Parameters()
+        self.assertRaises(Exception, lambda: p.check())
+
+    def test_Parameters_check_extra(self):
+        p = Parameters(xyz=10, **self.vals)
+        self.assertRaises(Exception, lambda: p.check())
+
+    def test_load_parameters_file_yaml(self):
+        with MockedOpen({"params.yml": "some: data\n"}):
+            self.assertEqual(
+                    load_parameters_file({'parameters': 'params.yml'}),
+                    {'some': 'data'})
+
+    def test_load_parameters_file_json(self):
+        with MockedOpen({"params.json": '{"some": "data"}'}):
+            self.assertEqual(
+                    load_parameters_file({'parameters': 'params.json'}),
+                    {'some': 'data'})
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_target_tasks.py b/taskcluster/taskgraph/test/test_target_tasks.py
new file mode 100644
index 000000000..035ccefd8
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_target_tasks.py
@@ -0,0 +1,81 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from .. import target_tasks
+from .. import try_option_syntax
+from ..graph import Graph
+from ..taskgraph import TaskGraph
+from .util import TestTask
+from mozunit import main
+
+
+class FakeTryOptionSyntax(object):
+
+    def __init__(self, message, task_graph):
+        self.trigger_tests = 0
+        self.notifications = None
+
+    def task_matches(self, attributes):
+        return 'at-at' in attributes
+
+
+class TestTargetTasks(unittest.TestCase):
+
+    def default_matches(self, run_on_projects, project):
+        method = target_tasks.get_method('default')
+        graph = TaskGraph(tasks={
+            'a': TestTask(kind='build', label='a',
+                          attributes={'run_on_projects': run_on_projects}),
+        }, graph=Graph(nodes={'a'}, edges=set()))
+        parameters = {'project': project}
+        return 'a' in method(graph, parameters)
+
+    def test_default_all(self):
+        """run_on_projects=[all] includes release, integration, and other projects"""
+        self.assertTrue(self.default_matches(['all'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-inbound'))
+        self.assertTrue(self.default_matches(['all'], 'mozilla-aurora'))
+        self.assertTrue(self.default_matches(['all'], 'baobab'))
+
+    def test_default_integration(self):
+        """run_on_projects=[integration] includes integration projects"""
+        self.assertFalse(self.default_matches(['integration'], 'mozilla-central'))
+        self.assertTrue(self.default_matches(['integration'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['integration'], 'baobab'))
+
+    def test_default_relesae(self):
+        """run_on_projects=[release] includes release projects"""
+        self.assertTrue(self.default_matches(['release'], 'mozilla-central'))
+        self.assertFalse(self.default_matches(['release'], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches(['release'], 'baobab'))
+
+    def test_default_nothing(self):
+        """run_on_projects=[] includes nothing"""
+        self.assertFalse(self.default_matches([], 'mozilla-central'))
+        self.assertFalse(self.default_matches([], 'mozilla-inbound'))
+        self.assertFalse(self.default_matches([], 'baobab'))
+
+    def test_try_option_syntax(self):
+        tasks = {
+            'a': TestTask(kind=None, label='a'),
+            'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}),
+        }
+        graph = Graph(nodes=set('ab'), edges=set())
+        tg = TaskGraph(tasks, graph)
+        params = {'message': 'try me'}
+
+        orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax
+        try:
+            try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax
+            method = target_tasks.get_method('try_option_syntax')
+            self.assertEqual(method(tg, params), ['b'])
+        finally:
+            try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_task_docker_image.py b/taskcluster/taskgraph/test/test_task_docker_image.py
new file mode 100644
index 000000000..8f247db3e
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_task_docker_image.py
@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import os
+
+from ..task import docker_image
+from mozunit import main
+
+
+KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image')
+
+
+class TestDockerImageKind(unittest.TestCase):
+
+    def setUp(self):
+        self.task = docker_image.DockerImageTask(
+            'docker-image',
+            KIND_PATH,
+            {},
+            {},
+            index_paths=[])
+
+    def test_get_task_dependencies(self):
+        # this one's easy!
+        self.assertEqual(self.task.get_dependencies(None), [])
+
+    # TODO: optimize_task
+
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_taskgraph.py b/taskcluster/taskgraph/test/test_taskgraph.py
new file mode 100644
index 000000000..f8f09bce9
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_taskgraph.py
@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..graph import Graph
+from ..task.docker_image import DockerImageTask
+from ..task.transform import TransformTask
+from ..taskgraph import TaskGraph
+from mozunit import main
+from taskgraph.util.docker import INDEX_PREFIX
+
+
+class TestTargetTasks(unittest.TestCase):
+
+    def test_from_json(self):
+        task = {
+            "routes": [],
+            "extra": {
+                "imageMeta": {
+                    "contextHash": "<hash>",
+                    "imageName": "<image>",
+                    "level": "1"
+                }
+            }
+        }
+        index_paths = ["{}.level-{}.<image>.hash.<hash>".format(INDEX_PREFIX, level)
+                       for level in range(1, 4)]
+        graph = TaskGraph(tasks={
+            'a': TransformTask(
+                kind='fancy',
+                task={
+                    'label': 'a',
+                    'attributes': {},
+                    'dependencies': {},
+                    'when': {},
+                    'task': {'task': 'def'},
+                }),
+            'b': DockerImageTask(kind='docker-image',
+                                 label='b',
+                                 attributes={},
+                                 task=task,
+                                 index_paths=index_paths),
+        }, graph=Graph(nodes={'a', 'b'}, edges=set()))
+
+        tasks, new_graph = TaskGraph.from_json(graph.to_json())
+        self.assertEqual(graph.tasks['a'], new_graph.tasks['a'])
+        self.assertEqual(graph, new_graph)
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_transforms_base.py b/taskcluster/taskgraph/test/test_transforms_base.py
new file mode 100644
index 000000000..0a0dfcaf2
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_transforms_base.py
@@ -0,0 +1,143 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+from mozunit import main
+from taskgraph.transforms.base import (
+    validate_schema,
+    get_keyed_by,
+    TransformSequence
+)
+from voluptuous import Schema
+
+schema = Schema({
+    'x': int,
+    'y': basestring,
+})
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def trans1(config, tests):
+    for test in tests:
+        test['one'] = 1
+        yield test
+
+
+@transforms.add
+def trans2(config, tests):
+    for test in tests:
+        test['two'] = 2
+        yield test
+
+
+class TestTransformSequence(unittest.TestCase):
+
+    def test_sequence(self):
+        tests = [{}, {'two': 1, 'second': True}]
+        res = list(transforms({}, tests))
+        self.assertEqual(res, [
+            {u'two': 2, u'one': 1},
+            {u'second': True, u'two': 2, u'one': 1},
+        ])
+
+
+class TestValidateSchema(unittest.TestCase):
+
+    def test_valid(self):
+        validate_schema(schema, {'x': 10, 'y': 'foo'}, "pfx")
+
+    def test_invalid(self):
+        try:
+            validate_schema(schema, {'x': 'not-int'}, "pfx")
+            self.fail("no exception raised")
+        except Exception, e:
+            self.failUnless(str(e).startswith("pfx\n"))
+
+
+class TestKeyedBy(unittest.TestCase):
+
+    def test_simple_value(self):
+        test = {
+            'test-name': 'tname',
+            'option': 10,
+        }
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), 10)
+
+    def test_by_value(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'by-other-value': {
+                    'a': 10,
+                    'b': 20,
+                },
+            },
+            'other-value': 'b',
+        }
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), 20)
+
+    def test_by_value_regex(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'by-test-platform': {
+                    'macosx64/.*': 10,
+                    'linux64/debug': 20,
+                    'default': 5,
+                },
+            },
+            'test-platform': 'macosx64/debug',
+        }
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), 10)
+
+    def test_by_value_default(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'by-other-value': {
+                    'a': 10,
+                    'default': 30,
+                },
+            },
+            'other-value': 'xxx',
+        }
+        self.assertEqual(get_keyed_by(test, 'option', 'x'), 30)
+
+    def test_by_value_invalid_dict(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'by-something-else': {},
+                'by-other-value': {},
+            },
+        }
+        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+
+    def test_by_value_invalid_no_default(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'by-other-value': {
+                    'a': 10,
+                },
+            },
+            'other-value': 'b',
+        }
+        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+
+    def test_by_value_invalid_no_by(self):
+        test = {
+            'test-name': 'tname',
+            'option': {
+                'other-value': {},
+            },
+        }
+        self.assertRaises(Exception, get_keyed_by, test, 'option', 'x')
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_try_option_syntax.py b/taskcluster/taskgraph/test/test_try_option_syntax.py
new file mode 100644
index 000000000..29aa2d5a9
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_try_option_syntax.py
@@ -0,0 +1,274 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import itertools
+
+from ..try_option_syntax import TryOptionSyntax
+from ..try_option_syntax import RIDEALONG_BUILDS
+from ..graph import Graph
+from ..taskgraph import TaskGraph
+from .util import TestTask
+from mozunit import main
+
+# an empty graph, for things that don't look at it
+empty_graph = TaskGraph({}, Graph(set(), set()))
+
+
+def unittest_task(n, tp):
+    return (n, TestTask('test', n, {
+        'unittest_try_name': n,
+        'test_platform': tp,
+    }))
+
+
+def talos_task(n, tp):
+    return (n, TestTask('test', n, {
+        'talos_try_name': n,
+        'test_platform': tp,
+    }))
+
+tasks = {k: v for k, v in [
+    unittest_task('mochitest-browser-chrome', 'linux'),
+    unittest_task('mochitest-e10s-browser-chrome', 'linux64'),
+    unittest_task('mochitest-chrome', 'linux'),
+    unittest_task('mochitest-webgl', 'linux'),
+    unittest_task('crashtest-e10s', 'linux'),
+    unittest_task('gtest', 'linux64'),
+    talos_task('dromaeojs', 'linux64'),
+]}
+unittest_tasks = {k: v for k, v in tasks.iteritems()
+                  if 'unittest_try_name' in v.attributes}
+talos_tasks = {k: v for k, v in tasks.iteritems()
+               if 'talos_try_name' in v.attributes}
+graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set()))
+
+
+class TestTryOptionSyntax(unittest.TestCase):
+
+    def test_empty_message(self):
+        "Given an empty message, it should return an empty value"
+        tos = TryOptionSyntax('', empty_graph)
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, [])
+        self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.talos, [])
+        self.assertEqual(tos.platforms, [])
+
+    def test_message_without_try(self):
+        "Given a non-try message, it should return an empty value"
+        tos = TryOptionSyntax('Bug 1234: frobnicte the foo', empty_graph)
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, [])
+        self.assertEqual(tos.unittests, [])
+        self.assertEqual(tos.talos, [])
+        self.assertEqual(tos.platforms, [])
+
+    def test_unknown_args(self):
+        "unknown arguments are ignored"
+        tos = TryOptionSyntax('try: --doubledash -z extra', empty_graph)
+        # equilvant to "try:"..
+        self.assertEqual(tos.build_types, [])
+        self.assertEqual(tos.jobs, None)
+
+    def test_b_do(self):
+        "-b do should produce both build_types"
+        tos = TryOptionSyntax('try: -b do', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug', 'opt'])
+
+    def test_b_d(self):
+        "-b d should produce build_types=['debug']"
+        tos = TryOptionSyntax('try: -b d', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug'])
+
+    def test_b_o(self):
+        "-b o should produce build_types=['opt']"
+        tos = TryOptionSyntax('try: -b o', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['opt'])
+
+    def test_build_o(self):
+        "--build o should produce build_types=['opt']"
+        tos = TryOptionSyntax('try: --build o', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['opt'])
+
+    def test_b_dx(self):
+        "-b dx should produce build_types=['debug'], silently ignoring the x"
+        tos = TryOptionSyntax('try: -b dx', empty_graph)
+        self.assertEqual(sorted(tos.build_types), ['debug'])
+
+    def test_j_job(self):
+        "-j somejob sets jobs=['somejob']"
+        tos = TryOptionSyntax('try: -j somejob', empty_graph)
+        self.assertEqual(sorted(tos.jobs), ['somejob'])
+
+    def test_j_jobs(self):
+        "-j job1,job2 sets jobs=['job1', 'job2']"
+        tos = TryOptionSyntax('try: -j job1,job2', empty_graph)
+        self.assertEqual(sorted(tos.jobs), ['job1', 'job2'])
+
+    def test_j_all(self):
+        "-j all sets jobs=None"
+        tos = TryOptionSyntax('try: -j all', empty_graph)
+        self.assertEqual(tos.jobs, None)
+
+    def test_j_twice(self):
+        "-j job1 -j job2 sets jobs=job1, job2"
+        tos = TryOptionSyntax('try: -j job1 -j job2', empty_graph)
+        self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2']))
+
+    def test_p_all(self):
+        "-p all sets platforms=None"
+        tos = TryOptionSyntax('try: -p all', empty_graph)
+        self.assertEqual(tos.platforms, None)
+
+    def test_p_linux(self):
+        "-p linux sets platforms=['linux', 'linux-l10n']"
+        tos = TryOptionSyntax('try: -p linux', empty_graph)
+        self.assertEqual(tos.platforms, ['linux', 'linux-l10n'])
+
+    def test_p_linux_win32(self):
+        "-p linux,win32 sets platforms=['linux', 'linux-l10n', 'win32']"
+        tos = TryOptionSyntax('try: -p linux,win32', empty_graph)
+        self.assertEqual(sorted(tos.platforms), ['linux', 'linux-l10n', 'win32'])
+
+    def test_p_expands_ridealongs(self):
+        "-p linux,linux64 includes the RIDEALONG_BUILDS"
+        tos = TryOptionSyntax('try: -p linux,linux64', empty_graph)
+        ridealongs = list(task
+                          for task in itertools.chain.from_iterable(
+                                RIDEALONG_BUILDS.itervalues()
+                          )
+                          if 'android' not in task)  # Don't include android-l10n
+        self.assertEqual(sorted(tos.platforms), sorted(['linux', 'linux64'] + ridealongs))
+
+    def test_u_none(self):
+        "-u none sets unittests=[]"
+        tos = TryOptionSyntax('try: -u none', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), [])
+
+    def test_u_all(self):
+        "-u all sets unittests=[..whole list..]"
+        tos = TryOptionSyntax('try: -u all', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks]))
+
+    def test_u_single(self):
+        "-u mochitest-webgl sets unittests=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
+
+    def test_u_alias(self):
+        "-u mochitest-gl sets unittests=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}]))
+
+    def test_u_multi_alias(self):
+        "-u e10s sets unittests=[all e10s unittests]"
+        tos = TryOptionSyntax('try: -u e10s', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': t} for t in unittest_tasks if 'e10s' in t
+        ]))
+
+    def test_u_commas(self):
+        "-u mochitest-webgl,gtest sets unittests=both"
+        tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'mochitest-webgl'},
+            {'test': 'gtest'},
+        ]))
+
+    def test_u_chunks(self):
+        "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest"
+        tos = TryOptionSyntax('try: -u gtest-3,gtest-4', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'only_chunks': set('34')},
+        ]))
+
+    def test_u_platform(self):
+        "-u gtest[linux] selects the linux platform for gtest"
+        tos = TryOptionSyntax('try: -u gtest[linux]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux']},
+        ]))
+
+    def test_u_platforms(self):
+        "-u gtest[linux,win32] selects the linux and win32 platforms for gtest"
+        tos = TryOptionSyntax('try: -u gtest[linux,win32]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux', 'win32']},
+        ]))
+
+    def test_u_platforms_pretty(self):
+        "-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan platforms for gtest"
+        tos = TryOptionSyntax('try: -u gtest[Ubuntu]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux', 'linux64', 'linux64-asan']},
+        ]))
+
+    def test_u_platforms_negated(self):
+        "-u gtest[-linux] selects all platforms but linux for gtest"
+        tos = TryOptionSyntax('try: -u gtest[-linux]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux64']},
+        ]))
+
+    def test_u_platforms_negated_pretty(self):
+        "-u gtest[Ubuntu,-x64] selects just linux for gtest"
+        tos = TryOptionSyntax('try: -u gtest[Ubuntu,-x64]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux']},
+        ]))
+
+    def test_u_chunks_platforms(self):
+        "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest"
+        tos = TryOptionSyntax('try: -u gtest-1[linux,win32]', graph_with_jobs)
+        self.assertEqual(sorted(tos.unittests), sorted([
+            {'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')},
+        ]))
+
+    def test_t_none(self):
+        "-t none sets talos=[]"
+        tos = TryOptionSyntax('try: -t none', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), [])
+
+    def test_t_all(self):
+        "-t all sets talos=[..whole list..]"
+        tos = TryOptionSyntax('try: -t all', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks]))
+
+    def test_t_single(self):
+        "-t mochitest-webgl sets talos=[mochitest-webgl]"
+        tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs)
+        self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}]))
+
+    # -t shares an implementation with -u, so it's not tested heavily
+
+    def test_trigger_tests(self):
+        "--rebuild 10 sets trigger_tests"
+        tos = TryOptionSyntax('try: --rebuild 10', empty_graph)
+        self.assertEqual(tos.trigger_tests, 10)
+
+    def test_interactive(self):
+        "--interactive sets interactive"
+        tos = TryOptionSyntax('try: --interactive', empty_graph)
+        self.assertEqual(tos.interactive, True)
+
+    def test_all_email(self):
+        "--all-emails sets notifications"
+        tos = TryOptionSyntax('try: --all-emails', empty_graph)
+        self.assertEqual(tos.notifications, 'all')
+
+    def test_fail_email(self):
+        "--failure-emails sets notifications"
+        tos = TryOptionSyntax('try: --failure-emails', empty_graph)
+        self.assertEqual(tos.notifications, 'failure')
+
+    def test_no_email(self):
+        "no email settings don't set notifications"
+        tos = TryOptionSyntax('try:', empty_graph)
+        self.assertEqual(tos.notifications, None)
+
+if __name__ == '__main__':
+    main()
diff --git a/taskcluster/taskgraph/test/test_util_attributes.py b/taskcluster/taskgraph/test/test_util_attributes.py
new file mode 100644
index 000000000..c3575e917
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_attributes.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+from taskgraph.util.attributes import attrmatch
+
+
+class Attrmatch(unittest.TestCase):
+
+    def test_trivial_match(self):
+        """Given no conditions, anything matches"""
+        self.assertTrue(attrmatch({}))
+
+    def test_missing_attribute(self):
+        """If a filtering attribute is not present, no match"""
+        self.assertFalse(attrmatch({}, someattr=10))
+
+    def test_literal_attribute(self):
+        """Literal attributes must match exactly"""
+        self.assertTrue(attrmatch({'att': 10}, att=10))
+        self.assertFalse(attrmatch({'att': 10}, att=20))
+
+    def test_set_attribute(self):
+        """Set attributes require set membership"""
+        self.assertTrue(attrmatch({'att': 10}, att=set([9, 10])))
+        self.assertFalse(attrmatch({'att': 10}, att=set([19, 20])))
+
+    def test_callable_attribute(self):
+        """Callable attributes are called and any False causes the match to fail"""
+        self.assertTrue(attrmatch({'att': 10}, att=lambda val: True))
+        self.assertFalse(attrmatch({'att': 10}, att=lambda val: False))
+
+        def even(val):
+            return val % 2 == 0
+        self.assertTrue(attrmatch({'att': 10}, att=even))
+        self.assertFalse(attrmatch({'att': 11}, att=even))
+
+    def test_all_matches_required(self):
+        """If only one attribute does not match, the result is False"""
+        self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3))
+        self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3))
+        self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3))
diff --git a/taskcluster/taskgraph/test/test_util_docker.py b/taskcluster/taskgraph/test/test_util_docker.py
new file mode 100644
index 000000000..3bb7fe8f0
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_docker.py
@@ -0,0 +1,194 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import shutil
+import stat
+import tarfile
+import tempfile
+import unittest
+
+from ..util import docker
+from mozunit import MockedOpen
+
+
+MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+
+class TestDocker(unittest.TestCase):
+
+    def test_generate_context_hash(self):
+        tmpdir = tempfile.mkdtemp()
+        old_GECKO = docker.GECKO
+        docker.GECKO = tmpdir
+        try:
+            os.makedirs(os.path.join(tmpdir, 'docker', 'my-image'))
+            p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile')
+            with open(p, 'w') as f:
+                f.write("FROM node\nADD a-file\n")
+            os.chmod(p, MODE_STANDARD)
+            p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file')
+            with open(p, 'w') as f:
+                f.write("data\n")
+            os.chmod(p, MODE_STANDARD)
+            self.assertEqual(
+                docker.generate_context_hash(docker.GECKO,
+                                             os.path.join(docker.GECKO, 'docker/my-image'),
+                                             'my-image'),
+                'e61e675ce05e8c11424437db3f1004079374c1a5fe6ad6800346cebe137b0797'
+            )
+        finally:
+            docker.GECKO = old_GECKO
+            shutil.rmtree(tmpdir)
+
+    def test_docker_image_explicit_registry(self):
+        files = {}
+        files["{}/myimage/REGISTRY".format(docker.DOCKER_ROOT)] = "cool-images"
+        files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3"
+        with MockedOpen(files):
+            self.assertEqual(docker.docker_image('myimage'), "cool-images/myimage:1.2.3")
+
+    def test_docker_image_default_registry(self):
+        files = {}
+        files["{}/REGISTRY".format(docker.DOCKER_ROOT)] = "mozilla"
+        files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3"
+        with MockedOpen(files):
+            self.assertEqual(docker.docker_image('myimage'), "mozilla/myimage:1.2.3")
+
+    def test_create_context_tar_basic(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test_image')
+            os.mkdir(d)
+            with open(os.path.join(d, 'Dockerfile'), 'a'):
+                pass
+            os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD)
+
+            with open(os.path.join(d, 'extra'), 'a'):
+                pass
+            os.chmod(os.path.join(d, 'extra'), MODE_STANDARD)
+
+            tp = os.path.join(tmp, 'tar')
+            h = docker.create_context_tar(tmp, d, tp, 'my_image')
+            self.assertEqual(h, '2a6d7f1627eba60daf85402418e041d728827d309143c6bc1c6bb3035bde6717')
+
+            # File prefix should be "my_image"
+            with tarfile.open(tp, 'r:gz') as tf:
+                self.assertEqual(tf.getnames(), [
+                    'my_image/Dockerfile',
+                    'my_image/extra',
+                ])
+        finally:
+            shutil.rmtree(tmp)
+
+    def test_create_context_topsrcdir_files(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test-image')
+            os.mkdir(d)
+            with open(os.path.join(d, 'Dockerfile'), 'wb') as fh:
+                fh.write(b'# %include extra/file0\n')
+            os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD)
+
+            extra = os.path.join(tmp, 'extra')
+            os.mkdir(extra)
+            with open(os.path.join(extra, 'file0'), 'a'):
+                pass
+            os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD)
+
+            tp = os.path.join(tmp, 'tar')
+            h = docker.create_context_tar(tmp, d, tp, 'test_image')
+            self.assertEqual(h, '20faeb7c134f21187b142b5fadba94ae58865dc929c6c293d8cbc0a087269338')
+
+            with tarfile.open(tp, 'r:gz') as tf:
+                self.assertEqual(tf.getnames(), [
+                    'test_image/Dockerfile',
+                    'test_image/topsrcdir/extra/file0',
+                ])
+        finally:
+            shutil.rmtree(tmp)
+
+    def test_create_context_absolute_path(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test-image')
+            os.mkdir(d)
+
+            # Absolute paths in %include syntax are not allowed.
+            with open(os.path.join(d, 'Dockerfile'), 'wb') as fh:
+                fh.write(b'# %include /etc/shadow\n')
+
+            with self.assertRaisesRegexp(Exception, 'cannot be absolute'):
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+        finally:
+            shutil.rmtree(tmp)
+
+    def test_create_context_outside_topsrcdir(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test-image')
+            os.mkdir(d)
+
+            with open(os.path.join(d, 'Dockerfile'), 'wb') as fh:
+                fh.write(b'# %include foo/../../../etc/shadow\n')
+
+            with self.assertRaisesRegexp(Exception, 'path outside topsrcdir'):
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+        finally:
+            shutil.rmtree(tmp)
+
+    def test_create_context_missing_extra(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test-image')
+            os.mkdir(d)
+
+            with open(os.path.join(d, 'Dockerfile'), 'wb') as fh:
+                fh.write(b'# %include does/not/exist\n')
+
+            with self.assertRaisesRegexp(Exception, 'path does not exist'):
+                docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test')
+        finally:
+            shutil.rmtree(tmp)
+
+    def test_create_context_extra_directory(self):
+        tmp = tempfile.mkdtemp()
+        try:
+            d = os.path.join(tmp, 'test-image')
+            os.mkdir(d)
+
+            with open(os.path.join(d, 'Dockerfile'), 'wb') as fh:
+                fh.write(b'# %include extra\n')
+                fh.write(b'# %include file0\n')
+            os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD)
+
+            extra = os.path.join(tmp, 'extra')
+            os.mkdir(extra)
+            for i in range(3):
+                p = os.path.join(extra, 'file%d' % i)
+                with open(p, 'wb') as fh:
+                    fh.write(b'file%d' % i)
+                os.chmod(p, MODE_STANDARD)
+
+            with open(os.path.join(tmp, 'file0'), 'a'):
+                pass
+            os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD)
+
+            tp = os.path.join(tmp, 'tar')
+            h = docker.create_context_tar(tmp, d, tp, 'my_image')
+
+            self.assertEqual(h, 'e5440513ab46ae4c1d056269e1c6715d5da7d4bd673719d360411e35e5b87205')
+
+            with tarfile.open(tp, 'r:gz') as tf:
+                self.assertEqual(tf.getnames(), [
+                    'my_image/Dockerfile',
+                    'my_image/topsrcdir/extra/file0',
+                    'my_image/topsrcdir/extra/file1',
+                    'my_image/topsrcdir/extra/file2',
+                    'my_image/topsrcdir/file0',
+                ])
+        finally:
+            shutil.rmtree(tmp)
diff --git a/taskcluster/taskgraph/test/test_util_python_path.py b/taskcluster/taskgraph/test/test_util_python_path.py
new file mode 100644
index 000000000..9615d1347
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_python_path.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+from ..util import python_path
+
+
+class TestObject(object):
+
+    testClassProperty = object()
+
+
+class TestPythonPath(unittest.TestCase):
+
+    def test_find_object_no_such_module(self):
+        """find_object raises ImportError for a nonexistent module"""
+        self.assertRaises(ImportError, python_path.find_object, "no_such_module:someobj")
+
+    def test_find_object_no_such_object(self):
+        """find_object raises AttributeError for a nonexistent object"""
+        self.assertRaises(AttributeError, python_path.find_object,
+                          "taskgraph.test.test_util_python_path:NoSuchObject")
+
+    def test_find_object_exists(self):
+        """find_object finds an existing object"""
+        obj = python_path.find_object(
+            "taskgraph.test.test_util_python_path:TestObject.testClassProperty")
+        self.assertIs(obj, TestObject.testClassProperty)
diff --git a/taskcluster/taskgraph/test/test_util_templates.py b/taskcluster/taskgraph/test/test_util_templates.py
new file mode 100755
index 000000000..47f7494a0
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_templates.py
@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+import mozunit
+import textwrap
+from taskgraph.util.templates import (
+    merge_to,
+    merge,
+    Templates,
+    TemplatesException
+)
+
+files = {}
+files['/fixtures/circular.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular_ref.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/inherit.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'templates.yml'
+      variables:
+        woot: 'inherit'
+    """)
+
+files['/fixtures/extend_child.yml'] = textwrap.dedent("""\
+    list: ['1', '2', '3']
+    was_list: ['1']
+    obj:
+      level: 1
+      deeper:
+        woot: 'bar'
+        list: ['baz']
+    """)
+
+files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'circular.yml'
+    """)
+
+files['/fixtures/child_pass.yml'] = textwrap.dedent("""\
+    values:
+      - {{a}}
+      - {{b}}
+      - {{c}}
+    """)
+
+files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'child_pass.yml'
+      variables:
+        a: 'a'
+        b: 'b'
+        c: 'c'
+    """)
+
+files['/fixtures/deep/2.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/1.yml
+
+    """)
+
+files['/fixtures/deep/3.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/2.yml
+
+    """)
+
+files['/fixtures/deep/4.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: deep/3.yml
+    """)
+
+files['/fixtures/deep/1.yml'] = textwrap.dedent("""\
+    variable: {{value}}
+    """)
+
+files['/fixtures/simple.yml'] = textwrap.dedent("""\
+    is_simple: true
+    """)
+
+files['/fixtures/templates.yml'] = textwrap.dedent("""\
+    content: 'content'
+    variable: '{{woot}}'
+    """)
+
+files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\
+    $inherits:
+      from: 'extend_child.yml'
+
+    list: ['4']
+    was_list:
+      replaced: true
+    obj:
+      level: 2
+      from_parent: true
+      deeper:
+        list: ['bar']
+    """)
+
+
+class TemplatesTest(unittest.TestCase):
+
+    def setUp(self):
+        self.mocked_open = mozunit.MockedOpen(files)
+        self.mocked_open.__enter__()
+        self.subject = Templates('/fixtures')
+
+    def tearDown(self):
+        self.mocked_open.__exit__(None, None, None)
+
+    def test_invalid_path(self):
+        with self.assertRaisesRegexp(TemplatesException, 'must be a directory'):
+            Templates('/zomg/not/a/dir')
+
+    def test_no_templates(self):
+        content = self.subject.load('simple.yml', {})
+        self.assertEquals(content, {
+            'is_simple': True
+        })
+
+    def test_with_templates(self):
+        content = self.subject.load('templates.yml', {
+            'woot': 'bar'
+        })
+
+        self.assertEquals(content, {
+            'content': 'content',
+            'variable': 'bar'
+        })
+
+    def test_inheritance(self):
+        '''
+        The simple single pass inheritance case.
+        '''
+        content = self.subject.load('inherit.yml', {})
+        self.assertEqual(content, {
+            'content': 'content',
+            'variable': 'inherit'
+        })
+
+    def test_inheritance_implicat_pass(self):
+        '''
+        Implicitly pass parameters from the child to the ancestor.
+        '''
+        content = self.subject.load('inherit_pass.yml', {
+            'a': 'overriden'
+        })
+
+        self.assertEqual(content, {'values': ['overriden', 'b', 'c']})
+
+    def test_inheritance_circular(self):
+        '''
+        Circular reference handling.
+        '''
+        with self.assertRaisesRegexp(TemplatesException, 'circular'):
+            self.subject.load('circular.yml', {})
+
+    def test_deep_inheritance(self):
+        content = self.subject.load('deep/4.yml', {
+            'value': 'myvalue'
+        })
+        self.assertEqual(content, {'variable': 'myvalue'})
+
+    def test_inheritance_with_simple_extensions(self):
+        content = self.subject.load('extend_parent.yml', {})
+        self.assertEquals(content, {
+            'list': ['1', '2', '3', '4'],
+            'obj': {
+                'from_parent': True,
+                'deeper': {
+                    'woot': 'bar',
+                    'list': ['baz', 'bar']
+                },
+                'level': 2,
+            },
+            'was_list': {'replaced': True}
+        })
+
+
+class MergeTest(unittest.TestCase):
+
+    def test_merge_to_dicts(self):
+        source = {'a': 1, 'b': 2}
+        dest = {'b': '20', 'c': 30}
+        expected = {
+            'a': 1,   # source only
+            'b': 2,   # source overrides dest
+            'c': 30,  # dest only
+        }
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge_to_lists(self):
+        source = {'x': [3, 4]}
+        dest = {'x': [1, 2]}
+        expected = {'x': [1, 2, 3, 4]}  # dest first
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge_diff_types(self):
+        source = {'x': [1, 2]}
+        dest = {'x': 'abc'}
+        expected = {'x': [1, 2]}  # source wins
+        self.assertEqual(merge_to(source, dest), expected)
+        self.assertEqual(dest, expected)
+
+    def test_merge(self):
+        first = {'a': 1, 'b': 2, 'd': 11}
+        second = {'b': 20, 'c': 30}
+        third = {'c': 300, 'd': 400}
+        expected = {
+            'a': 1,
+            'b': 20,
+            'c': 300,
+            'd': 400,
+        }
+        self.assertEqual(merge(first, second, third), expected)
+
+        # inputs haven't changed..
+        self.assertEqual(first, {'a': 1, 'b': 2, 'd': 11})
+        self.assertEqual(second, {'b': 20, 'c': 30})
+        self.assertEqual(third, {'c': 300, 'd': 400})
+
+if __name__ == '__main__':
+    mozunit.main()
diff --git a/taskcluster/taskgraph/test/test_util_time.py b/taskcluster/taskgraph/test/test_util_time.py
new file mode 100755
index 000000000..f001c9d9c
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_time.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+from datetime import datetime
+from taskgraph.util.time import (
+    InvalidString,
+    UnknownTimeMeasurement,
+    value_of,
+    json_time_from_now
+)
+
+
+class FromNowTest(unittest.TestCase):
+
+    def test_invalid_str(self):
+        with self.assertRaises(InvalidString):
+            value_of('wtfs')
+
+    def test_missing_unit(self):
+        with self.assertRaises(InvalidString):
+            value_of('1')
+
+    def test_missing_unknown_unit(self):
+        with self.assertRaises(UnknownTimeMeasurement):
+            value_of('1z')
+
+    def test_value_of(self):
+        self.assertEqual(value_of('1s').total_seconds(), 1)
+        self.assertEqual(value_of('1 second').total_seconds(), 1)
+        self.assertEqual(value_of('1min').total_seconds(), 60)
+        self.assertEqual(value_of('1h').total_seconds(), 3600)
+        self.assertEqual(value_of('1d').total_seconds(), 86400)
+        self.assertEqual(value_of('1mo').total_seconds(), 2592000)
+        self.assertEqual(value_of('1 month').total_seconds(), 2592000)
+        self.assertEqual(value_of('1y').total_seconds(), 31536000)
+
+        with self.assertRaises(UnknownTimeMeasurement):
+            value_of('1m').total_seconds()  # ambiguous between minute and month
+
+    def test_json_from_now_utc_now(self):
+        # Just here to ensure we don't raise.
+        json_time_from_now('1 years')
+
+    def test_json_from_now(self):
+        now = datetime(2014, 1, 1)
+        self.assertEqual(json_time_from_now('1 years', now),
+                         '2015-01-01T00:00:00Z')
+        self.assertEqual(json_time_from_now('6 days', now),
+                         '2014-01-07T00:00:00Z')
+
+if __name__ == '__main__':
+    mozunit.main()
diff --git a/taskcluster/taskgraph/test/test_util_treeherder.py b/taskcluster/taskgraph/test/test_util_treeherder.py
new file mode 100644
index 000000000..cf7513c00
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_treeherder.py
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+from taskgraph.util.treeherder import split_symbol, join_symbol
+
+
+class TestSymbols(unittest.TestCase):
+
+    def test_split_no_group(self):
+        self.assertEqual(split_symbol('xy'), ('?', 'xy'))
+
+    def test_split_with_group(self):
+        self.assertEqual(split_symbol('ab(xy)'), ('ab', 'xy'))
+
+    def test_join_no_group(self):
+        self.assertEqual(join_symbol('?', 'xy'), 'xy')
+
+    def test_join_with_group(self):
+        self.assertEqual(join_symbol('ab', 'xy'), 'ab(xy)')
diff --git a/taskcluster/taskgraph/test/test_util_yaml.py b/taskcluster/taskgraph/test/test_util_yaml.py
new file mode 100644
index 000000000..d4ff410db
--- /dev/null
+++ b/taskcluster/taskgraph/test/test_util_yaml.py
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import unittest
+
+from ..util import yaml
+from mozunit import MockedOpen
+
+FOO_YML = """\
+prop:
+    - val1
+"""
+
+
+class TestYaml(unittest.TestCase):
+
+    def test_load(self):
+        with MockedOpen({'/dir1/dir2/foo.yml': FOO_YML}):
+            self.assertEqual(yaml.load_yaml("/dir1/dir2", "foo.yml"),
+                             {'prop': ['val1']})
diff --git a/taskcluster/taskgraph/test/util.py b/taskcluster/taskgraph/test/util.py
new file mode 100644
index 000000000..cf9a49ad3
--- /dev/null
+++ b/taskcluster/taskgraph/test/util.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from ..task import base
+
+
+class TestTask(base.Task):
+
+    def __init__(self, kind=None, label=None, attributes=None, task=None):
+        super(TestTask, self).__init__(
+                kind or 'test',
+                label or 'test-label',
+                attributes or {},
+                task or {})
+
+    @classmethod
+    def load_tasks(cls, kind, path, config, parameters):
+        return []
+
+    def get_dependencies(self, taskgraph):
+        return []
diff --git a/taskcluster/taskgraph/transforms/__init__.py b/taskcluster/taskgraph/transforms/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/__init__.py
diff --git a/taskcluster/taskgraph/transforms/android_stuff.py b/taskcluster/taskgraph/transforms/android_stuff.py
new file mode 100644
index 000000000..cb1e0fa5b
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/android_stuff.py
@@ -0,0 +1,46 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Set dynamic task description properties of the android stuff.  Temporary!
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def setup_task(config, tasks):
+    for task in tasks:
+        task['label'] = task['name']
+        env = task['worker'].setdefault('env', {})
+        env.update({
+            'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+            'GECKO_HEAD_REF': config.params['head_rev'],
+            'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+            'GECKO_HEAD_REV': config.params['head_rev'],
+            'MOZ_BUILD_DATE': config.params['moz_build_date'],
+            'MOZ_SCM_LEVEL': config.params['level'],
+            'MH_BRANCH': config.params['project'],
+        })
+
+        task['worker'].setdefault('caches', []).append({
+            'type': 'persistent',
+            'name': 'level-{}-{}-tc-vcs'.format(
+                config.params['level'], config.params['project']),
+            'mount-point': "/home/worker/.tc-vcs",
+        })
+
+        if int(config.params['level']) > 1:
+            task['worker'].setdefault('caches', []).append({
+                'type': 'persistent',
+                'name': 'level-{}-{}-build-{}-workspace'.format(
+                    config.params['level'], config.params['project'], task['name']),
+                'mount-point': "/home/worker/workspace",
+            })
+
+        del task['name']
+        yield task
diff --git a/taskcluster/taskgraph/transforms/base.py b/taskcluster/taskgraph/transforms/base.py
new file mode 100644
index 000000000..aab139252
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/base.py
@@ -0,0 +1,126 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import re
+import pprint
+import voluptuous
+
+
+class TransformConfig(object):
+    """A container for configuration affecting transforms.  The `config`
+    argument to transforms is an instance of this class, possibly with
+    additional kind-specific attributes beyond those set here."""
+    def __init__(self, kind, path, config, params):
+        # the name of the current kind
+        self.kind = kind
+
+        # the path to the kind configuration directory
+        self.path = path
+
+        # the parsed contents of kind.yml
+        self.config = config
+
+        # the parameters for this task-graph generation run
+        self.params = params
+
+
+class TransformSequence(object):
+    """
+    Container for a sequence of transforms.  Each transform is represented as a
+    callable taking (config, items) and returning a generator which will yield
+    transformed items.  The resulting sequence has the same interface.
+
+    This is convenient to use in a file full of transforms, as it provides a
+    decorator, @transforms.add, that will add the decorated function to the
+    sequence.
+    """
+
+    def __init__(self, transforms=None):
+        self.transforms = transforms or []
+
+    def __call__(self, config, items):
+        for xform in self.transforms:
+            items = xform(config, items)
+            if items is None:
+                raise Exception("Transform {} is not a generator".format(xform))
+        return items
+
+    def __repr__(self):
+        return '\n'.join(
+            ['TransformSequence(['] +
+            [repr(x) for x in self.transforms] +
+            ['])'])
+
+    def add(self, func):
+        self.transforms.append(func)
+        return func
+
+
+def validate_schema(schema, obj, msg_prefix):
+    """
+    Validate that object satisfies schema.  If not, generate a useful exception
+    beginning with msg_prefix.
+    """
+    try:
+        return schema(obj)
+    except voluptuous.MultipleInvalid as exc:
+        msg = [msg_prefix]
+        for error in exc.errors:
+            msg.append(str(error))
+        raise Exception('\n'.join(msg) + '\n' + pprint.pformat(obj))
+
+
+def get_keyed_by(item, field, item_name, subfield=None):
+    """
+    For values which can either accept a literal value, or be keyed by some
+    other attribute of the item, perform that lookup.  For example, this supports
+
+        chunks:
+            by-test-platform:
+                macosx-10.11/debug: 13
+                win.*: 6
+                default: 12
+
+    The `item_name` parameter is used to generate useful error messages.
+    The `subfield` parameter, if specified, allows access to a second level
+    of the item dictionary: item[field][subfield]. For example, this supports
+
+        mozharness:
+            config:
+                by-test-platform:
+                    default: ...
+    """
+    value = item[field]
+    if not isinstance(value, dict):
+        return value
+    if subfield:
+        value = item[field][subfield]
+        if not isinstance(value, dict):
+            return value
+
+    assert len(value) == 1, "Invalid attribute {} in {}".format(field, item_name)
+    keyed_by = value.keys()[0]
+    values = value[keyed_by]
+    if keyed_by.startswith('by-'):
+        keyed_by = keyed_by[3:]  # extract just the keyed-by field name
+        if item[keyed_by] in values:
+            return values[item[keyed_by]]
+        for k in values.keys():
+            if re.match(k, item[keyed_by]):
+                return values[k]
+        if 'default' in values:
+            return values['default']
+        for k in item[keyed_by], 'default':
+            if k in values:
+                return values[k]
+        else:
+            raise Exception(
+                "Neither {} {} nor 'default' found while determining item {} in {}".format(
+                    keyed_by, item[keyed_by], field, item_name))
+    else:
+        raise Exception(
+            "Invalid attribute {} keyed-by value {} in {}".format(
+                field, keyed_by, item_name))
diff --git a/taskcluster/taskgraph/transforms/build.py b/taskcluster/taskgraph/transforms/build.py
new file mode 100644
index 000000000..3875cbbb1
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build.py
@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Apply some defaults and minor modifications to the jobs defined in the build
+kind.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, jobs):
+    """Set defaults, including those that differ per worker implementation"""
+    for job in jobs:
+        job['treeherder'].setdefault('kind', 'build')
+        job['treeherder'].setdefault('tier', 1)
+        if job['worker']['implementation'] in ('docker-worker', 'docker-engine'):
+            job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'})
+            job['worker']['chain-of-trust'] = True
+            job.setdefault('extra', {})
+            job['extra'].setdefault('chainOfTrust', {})
+            job['extra']['chainOfTrust'].setdefault('inputs', {})
+            job['extra']['chainOfTrust']['inputs']['docker-image'] = {
+                "task-reference": "<docker-image>"
+            }
+        yield job
diff --git a/taskcluster/taskgraph/transforms/build_attrs.py b/taskcluster/taskgraph/transforms/build_attrs.py
new file mode 100644
index 000000000..56c007614
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/build_attrs.py
@@ -0,0 +1,33 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_build_attributes(config, jobs):
+    """
+    Set the build_platform and build_type attributes based on the job name.
+    Although not all jobs using this transform are actual "builds", the try
+    option syntax treats them as such, and this arranges the attributes
+    appropriately for that purpose.
+    """
+    for job in jobs:
+        build_platform, build_type = job['name'].split('/')
+
+        # pgo builds are represented as a different platform, type opt
+        if build_type == 'pgo':
+            build_platform = build_platform + '-pgo'
+            build_type = 'opt'
+
+        attributes = job.setdefault('attributes', {})
+        attributes.update({
+            'build_platform': build_platform,
+            'build_type': build_type,
+        })
+
+        yield job
diff --git a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
new file mode 100644
index 000000000..3817faa50
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -0,0 +1,77 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This file contains a whitelist of gecko.v2 index route job names.  The intent
+of this whitelist is to raise an alarm when new jobs are added.  If those jobs
+already run in Buildbot, then it's important that the generated index routes
+match (and that only one of Buildbot and TaskCluster be tier-1 at any time).
+If the jobs are new and never ran in Buildbot, then their job name can be added
+here without any further fuss.
+
+Once all jobs have been ported from Buildbot, this file can be removed.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+# please keep me in lexical order
+JOB_NAME_WHITELIST = set([
+    'android-api-15-debug',
+    'android-api-15-gradle-dependencies-opt',
+    'android-api-15-gradle-opt',
+    'android-api-15-opt',
+    'android-api-15-nightly-opt',
+    'android-api-15-partner-sample1-opt',
+    'android-l10n-opt',
+    'android-x86-opt',
+    'aries-debug',
+    'aries-eng-opt',
+    'browser-haz-debug',
+    'linux32-l10n-opt',
+    'linux64-artifact-opt',
+    'linux64-asan-debug',
+    'linux64-asan-opt',
+    'linux64-ccov-opt',
+    'linux64-debug',
+    'linux64-jsdcov-opt',
+    'linux64-l10n-opt',
+    'linux64-opt',
+    'linux64-pgo',
+    'linux64-st-an-opt',
+    'linux64-valgrind-opt',
+    'linux-debug',
+    'linux-opt',
+    'linux-pgo',
+    'macosx64-debug',
+    'macosx64-opt',
+    'macosx64-st-an-opt',
+    'nexus-5-l-eng-debug',
+    'nexus-5-l-eng-opt',
+    'osx-10-10',
+    'shell-haz-debug',
+    'sm-arm64-sim-debug',
+    'sm-arm-sim-debug',
+    'sm-asan-opt',
+    'sm-compacting-debug',
+    'sm-mozjs-sys-debug',
+    'sm-msan-opt',
+    'sm-nonunified-debug',
+    'sm-package-opt',
+    'sm-plaindebug-debug',
+    'sm-plain-opt',
+    'sm-rootanalysis-debug',
+    'sm-tsan-opt',
+    'win32-debug',
+    'win32-opt',
+    'win32-pgo',
+    'win64-debug',
+    'win64-opt',
+    'win64-pgo',
+])
+
+JOB_NAME_WHITELIST_ERROR = """\
+The gecko-v2 job name {} is not in the whitelist in __file__.
+If this job runs on Buildbot, please ensure that the job names match between
+Buildbot and TaskCluster, then add the job name to the whitelist.  If this is a
+new job, there is nothing to check -- just add the job to the whitelist.
+"""
diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py
new file mode 100644
index 000000000..a0860c032
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/__init__.py
@@ -0,0 +1,164 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Convert a job description into a task description.
+
+Jobs descriptions are similar to task descriptions, but they specify how to run
+the job at a higher level, using a "run" field that can be interpreted by
+run-using handlers in `taskcluster/taskgraph/transforms/job`.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+import logging
+import os
+
+from taskgraph.transforms.base import validate_schema, TransformSequence
+from taskgraph.transforms.task import task_description_schema
+from voluptuous import (
+    Optional,
+    Required,
+    Schema,
+    Extra,
+)
+
+logger = logging.getLogger(__name__)
+
+# Voluptuous uses marker objects as dictionary *keys*, but they are not
+# comparable, so we cast all of the keys back to regular strings
+task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
+
+# Schema for a build description
+job_description_schema = Schema({
+    # The name of the job and the job's label.  At least one must be specified,
+    # and the label will be generated from the name if necessary, by prepending
+    # the kind.
+    Optional('name'): basestring,
+    Optional('label'): basestring,
+
+    # the following fields are passed directly through to the task description,
+    # possibly modified by the run implementation.  See
+    # taskcluster/taskgraph/transforms/task.py for the schema details.
+    Required('description'): task_description_schema['description'],
+    Optional('attributes'): task_description_schema['attributes'],
+    Optional('dependencies'): task_description_schema['dependencies'],
+    Optional('expires-after'): task_description_schema['expires-after'],
+    Optional('routes'): task_description_schema['routes'],
+    Optional('scopes'): task_description_schema['scopes'],
+    Optional('extra'): task_description_schema['extra'],
+    Optional('treeherder'): task_description_schema['treeherder'],
+    Optional('index'): task_description_schema['index'],
+    Optional('run-on-projects'): task_description_schema['run-on-projects'],
+    Optional('coalesce-name'): task_description_schema['coalesce-name'],
+    Optional('worker-type'): task_description_schema['worker-type'],
+    Required('worker'): task_description_schema['worker'],
+    Optional('when'): task_description_schema['when'],
+
+    # A description of how to run this job.
+    'run': {
+        # The key to a job implementation in a peer module to this one
+        'using': basestring,
+
+        # Any remaining content is verified against that job implementation's
+        # own schema.
+        Extra: object,
+    },
+})
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def validate(config, jobs):
+    for job in jobs:
+        yield validate_schema(job_description_schema, job,
+                              "In job {!r}:".format(job['name']))
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    """Given a build description, create a task description"""
+    # import plugin modules first, before iterating over jobs
+    import_all()
+    for job in jobs:
+        if 'label' not in job:
+            if 'name' not in job:
+                raise Exception("job has neither a name nor a label")
+            job['label'] = '{}-{}'.format(config.kind, job['name'])
+        if job['name']:
+            del job['name']
+
+        taskdesc = copy.deepcopy(job)
+
+        # fill in some empty defaults to make run implementations easier
+        taskdesc.setdefault('attributes', {})
+        taskdesc.setdefault('dependencies', {})
+        taskdesc.setdefault('routes', [])
+        taskdesc.setdefault('scopes', [])
+        taskdesc.setdefault('extra', {})
+
+        # give the function for job.run.using on this worker implementation a
+        # chance to set up the task description.
+        configure_taskdesc_for_run(config, job, taskdesc)
+        del taskdesc['run']
+
+        # yield only the task description, discarding the job description
+        yield taskdesc
+
+# A registry of all functions decorated with run_job_using
+registry = {}
+
+
+def run_job_using(worker_implementation, run_using, schema=None):
+    """Register the decorated function as able to set up a task description for
+    jobs with the given worker implementation and `run.using` property.  If
+    `schema` is given, the job's run field will be verified to match it.
+
+    The decorated function should have the signature `using_foo(config, job,
+    taskdesc) and should modify the task description in-place.  The skeleton of
+    the task description is already set up, but without a payload."""
+    def wrap(func):
+        for_run_using = registry.setdefault(run_using, {})
+        if worker_implementation in for_run_using:
+            raise Exception("run_job_using({!r}, {!r}) already exists: {!r}".format(
+                run_using, worker_implementation, for_run_using[run_using]))
+        for_run_using[worker_implementation] = (func, schema)
+        return func
+    return wrap
+
+
+def configure_taskdesc_for_run(config, job, taskdesc):
+    """
+    Run the appropriate function for this job against the given task
+    description.
+
+    This will raise an appropriate error if no function exists, or if the job's
+    run is not valid according to the schema.
+    """
+    run_using = job['run']['using']
+    if run_using not in registry:
+        raise Exception("no functions for run.using {!r}".format(run_using))
+
+    worker_implementation = job['worker']['implementation']
+    if worker_implementation not in registry[run_using]:
+        raise Exception("no functions for run.using {!r} on {!r}".format(
+            run_using, worker_implementation))
+
+    func, schema = registry[run_using][worker_implementation]
+    if schema:
+        job['run'] = validate_schema(
+                schema, job['run'],
+                "In job.run using {!r} for job {!r}:".format(
+                    job['run']['using'], job['label']))
+
+    func(config, job, taskdesc)
+
+
+def import_all():
+    """Import all modules that are siblings of this one, triggering the decorator
+    above in the process."""
+    for f in os.listdir(os.path.dirname(__file__)):
+        if f.endswith('.py') and f not in ('commmon.py', '__init__.py'):
+            __import__('taskgraph.transforms.job.' + f[:-3])
diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py
new file mode 100644
index 000000000..59a51d75a
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/common.py
@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Common support for various job types.  These functions are all named after the
+worker implementation they operate on, and take the same three parameters, for
+consistency.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}'
+
+
+def docker_worker_add_workspace_cache(config, job, taskdesc):
+    """Add the workspace cache based on the build platform/type and level,
+    except on try where workspace caches are not used."""
+    if config.params['project'] == 'try':
+        return
+
+    taskdesc['worker'].setdefault('caches', []).append({
+        'type': 'persistent',
+        'name': 'level-{}-{}-build-{}-{}-workspace'.format(
+            config.params['level'], config.params['project'],
+            taskdesc['attributes']['build_platform'],
+            taskdesc['attributes']['build_type'],
+        ),
+        'mount-point': "/home/worker/workspace",
+    })
+
+
+def docker_worker_add_tc_vcs_cache(config, job, taskdesc):
+    taskdesc['worker'].setdefault('caches', []).append({
+        'type': 'persistent',
+        'name': 'level-{}-{}-tc-vcs'.format(
+            config.params['level'], config.params['project']),
+        'mount-point': "/home/worker/.tc-vcs",
+    })
+
+
+def docker_worker_add_public_artifacts(config, job, taskdesc):
+    taskdesc['worker'].setdefault('artifacts', []).append({
+        'name': 'public/build',
+        'path': '/home/worker/artifacts/',
+        'type': 'directory',
+    })
+
+
+def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc):
+    """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker."""
+    env = taskdesc['worker'].setdefault('env', {})
+    env.update({
+        'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+        'GECKO_HEAD_REF': config.params['head_rev'],
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+    })
+
+
+def docker_worker_support_vcs_checkout(config, job, taskdesc):
+    """Update a job/task with parameters to enable a VCS checkout.
+
+    The configuration is intended for tasks using "run-task" and its
+    VCS checkout behavior.
+    """
+    level = config.params['level']
+
+    taskdesc['worker'].setdefault('caches', []).append({
+        'type': 'persistent',
+        # History of versions:
+        #
+        # ``level-%s-checkouts`` was initially used and contained a number
+        # of backwards incompatible changes, such as moving HG_STORE_PATH
+        # from a separate cache to this cache.
+        #
+        # ``v1`` was introduced to provide a clean break from the unversioned
+        # cache.
+        'name': 'level-%s-checkouts-v1' % level,
+        'mount-point': '/home/worker/checkouts',
+    })
+
+    taskdesc['worker'].setdefault('env', {}).update({
+        'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+        'HG_STORE_PATH': '/home/worker/checkouts/hg-store',
+    })
+
+    # Give task access to hgfingerprint secret so it can pin the certificate
+    # for hg.mozilla.org.
+    taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint')
+    taskdesc['worker']['taskcluster-proxy'] = True
+
+
+def docker_worker_setup_secrets(config, job, taskdesc):
+    """Set up access to secrets via taskcluster-proxy.  The value of
+    run['secrets'] should be a boolean or a list of secret names that
+    can be accessed."""
+    if not job['run'].get('secrets'):
+        return
+
+    taskdesc['worker']['taskcluster-proxy'] = True
+    secrets = job['run']['secrets']
+    if secrets is True:
+        secrets = ['*']
+    for sec in secrets:
+        taskdesc['scopes'].append(SECRET_SCOPE.format(
+            job['treeherder']['kind'], config.params['level'], sec))
diff --git a/taskcluster/taskgraph/transforms/job/hazard.py b/taskcluster/taskgraph/transforms/job/hazard.py
new file mode 100644
index 000000000..c5b500843
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/hazard.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running hazard jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_workspace_cache,
+    docker_worker_setup_secrets,
+    docker_worker_add_public_artifacts,
+    docker_worker_support_vcs_checkout,
+)
+
+haz_run_schema = Schema({
+    Required('using'): 'hazard',
+
+    # The command to run within the task image (passed through to the worker)
+    Required('command'): basestring,
+
+    # The tooltool manifest to use; default in the script is used if omitted
+    Optional('tooltool-manifest'): basestring,
+
+    # The mozconfig to use; default in the script is used if omitted
+    Optional('mozconfig'): basestring,
+
+    # The set of secret names to which the task has access; these are prefixed
+    # with `project/releng/gecko/{treeherder.kind}/level-{level}/`.   Setting
+    # this will enable any worker features required and set the task's scopes
+    # appropriately.  `true` here means ['*'], all secrets.  Not supported on
+    # Windows
+    Required('secrets', default=False): Any(bool, [basestring]),
+})
+
+
+@run_job_using("docker-worker", "hazard", schema=haz_run_schema)
+def docker_worker_hazard(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_setup_secrets(config, job, taskdesc)
+    docker_worker_support_vcs_checkout(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    # script parameters
+    if run.get('tooltool-manifest'):
+        env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+    if run.get('mozconfig'):
+        env['MOZCONFIG'] = run['mozconfig']
+
+    # tooltool downloads
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    worker['relengapi-proxy'] = True
+    taskdesc['scopes'].extend([
+        'docker-worker:relengapi-proxy:tooltool.download.public',
+    ])
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    # build-haz-linux.sh needs this otherwise it assumes the checkout is in
+    # the workspace.
+    env['GECKO_DIR'] = '/home/worker/checkouts/gecko'
+
+    worker['command'] = [
+        '/home/worker/bin/run-task',
+        '--chown-recursive', '/home/worker/tooltool-cache',
+        '--chown-recursive', '/home/worker/workspace',
+        '--vcs-checkout', '/home/worker/checkouts/gecko',
+        '--',
+        '/bin/bash', '-c', run['command']
+    ]
diff --git a/taskcluster/taskgraph/transforms/job/mach.py b/taskcluster/taskgraph/transforms/job/mach.py
new file mode 100644
index 000000000..8df202dbc
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mach.py
@@ -0,0 +1,30 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running mach tasks (via run-task)
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.run_task import docker_worker_run_task
+from voluptuous import Schema, Required
+
+mach_schema = Schema({
+    Required('using'): 'mach',
+
+    # The mach command (omitting `./mach`) to run
+    Required('mach'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mach", schema=mach_schema)
+def docker_worker_mach(config, job, taskdesc):
+    run = job['run']
+
+    # defer to the run_task implementation
+    run['command'] = 'cd /home/worker/checkouts/gecko && ./mach ' + run['mach']
+    run['checkout'] = True
+    del run['mach']
+    docker_worker_run_task(config, job, taskdesc)
diff --git a/taskcluster/taskgraph/transforms/job/mozharness.py b/taskcluster/taskgraph/transforms/job/mozharness.py
new file mode 100644
index 000000000..fb3cd00dd
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+
+Support for running jobs via mozharness.  Ideally, most stuff gets run this
+way, and certainly anything using mozharness should use this approach.
+
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_workspace_cache,
+    docker_worker_add_gecko_vcs_env_vars,
+    docker_worker_setup_secrets,
+    docker_worker_add_public_artifacts,
+    docker_worker_support_vcs_checkout,
+)
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+mozharness_run_schema = Schema({
+    Required('using'): 'mozharness',
+
+    # the mozharness script used to run this task, relative to the testing/
+    # directory and using forward slashes even on Windows
+    Required('script'): basestring,
+
+    # the config files required for the task, relative to
+    # testing/mozharness/configs and using forward slashes even on Windows
+    Required('config'): [basestring],
+
+    # any additional actions to pass to the mozharness command; not supported
+    # on Windows
+    Optional('actions'): [basestring],
+
+    # any additional options (without leading --) to be passed to mozharness;
+    # not supported on Windows
+    Optional('options'): [basestring],
+
+    # --custom-build-variant-cfg value (not supported on Windows)
+    Optional('custom-build-variant-cfg'): basestring,
+
+    # If not false, tooltool downloads will be enabled via relengAPIProxy
+    # for either just public files, or all files.  Not supported on Windows
+    Required('tooltool-downloads', default=False): Any(
+        False,
+        'public',
+        'internal',
+    ),
+
+    # The set of secret names to which the task has access; these are prefixed
+    # with `project/releng/gecko/{treeherder.kind}/level-{level}/`.  Setting
+    # this will enable any worker features required and set the task's scopes
+    # appropriately.  `true` here means ['*'], all secrets.  Not supported on
+    # Windows
+    Required('secrets', default=False): Any(bool, [basestring]),
+
+    # If true, taskcluster proxy will be enabled; note that it may also be enabled
+    # automatically e.g., for secrets support.  Not supported on Windows.
+    Required('taskcluster-proxy', default=False): bool,
+
+    # If true, the build scripts will start Xvfb.  Not supported on Windows.
+    Required('need-xvfb', default=False): bool,
+
+    # If false, indicate that builds should skip producing artifacts.  Not
+    # supported on Windows.
+    Required('keep-artifacts', default=True): bool,
+
+    # If specified, use the in-tree job script specified.
+    Optional('job-script'): basestring,
+})
+
+
+@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_docker_worker_setup(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['implementation'] = job['worker']['implementation']
+
+    # running via mozharness assumes desktop-build (which contains build.sh)
+    taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"}
+
+    worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
+    worker['taskcluster-proxy'] = run.get('taskcluster-proxy')
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+    docker_worker_add_workspace_cache(config, job, taskdesc)
+    docker_worker_support_vcs_checkout(config, job, taskdesc)
+
+    env = worker.setdefault('env', {})
+    env.update({
+        'MOZHARNESS_CONFIG': ' '.join(run['config']),
+        'MOZHARNESS_SCRIPT': run['script'],
+        'MH_BRANCH': config.params['project'],
+        'MH_BUILD_POOL': 'taskcluster',
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    if 'actions' in run:
+        env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions'])
+
+    if 'options' in run:
+        env['MOZHARNESS_OPTIONS'] = ' '.join(run['options'])
+
+    if 'custom-build-variant-cfg' in run:
+        env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg']
+
+    if 'job-script' in run:
+        env['JOB_SCRIPT'] = run['job-script']
+
+    # if we're not keeping artifacts, set some env variables to empty values
+    # that will cause the build process to skip copying the results to the
+    # artifacts directory.  This will have no effect for operations that are
+    # not builds.
+    if not run['keep-artifacts']:
+        env['DIST_TARGET_UPLOADS'] = ''
+        env['DIST_UPLOADS'] = ''
+
+    # Xvfb
+    if run['need-xvfb']:
+        env['NEED_XVFB'] = 'true'
+
+    # tooltool downloads
+    if run['tooltool-downloads']:
+        worker['relengapi-proxy'] = True
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'tooltool-cache',
+            'mount-point': '/home/worker/tooltool-cache',
+        })
+        taskdesc['scopes'].extend([
+            'docker-worker:relengapi-proxy:tooltool.download.public',
+        ])
+        if run['tooltool-downloads'] == 'internal':
+            taskdesc['scopes'].append(
+                'docker-worker:relengapi-proxy:tooltool.download.internal')
+        env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+        env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+        env['TOOLTOOL_REV'] = 'master'
+
+    # Retry if mozharness returns TBPL_RETRY
+    worker['retry-exit-status'] = 4
+
+    docker_worker_setup_secrets(config, job, taskdesc)
+
+    command = [
+        '/home/worker/bin/run-task',
+        # Various caches/volumes are default owned by root:root.
+        '--chown-recursive', '/home/worker/workspace',
+        '--chown-recursive', '/home/worker/tooltool-cache',
+        '--vcs-checkout', '/home/worker/workspace/build/src',
+        '--tools-checkout', '/home/worker/workspace/build/tools',
+        '--',
+    ]
+    command.append("/home/worker/workspace/build/src/{}".format(
+        run.get('job-script',
+                "taskcluster/scripts/builder/build-linux.sh"
+                )))
+
+    worker['command'] = command
+
+
+# We use the generic worker to run tasks on Windows
+@run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
+def mozharness_on_windows(config, job, taskdesc):
+    run = job['run']
+
+    # fail if invalid run options are included
+    invalid = []
+    for prop in ['actions', 'custom-build-variant-cfg',
+                 'tooltool-downloads', 'secrets', 'taskcluster-proxy',
+                 'need-xvfb']:
+        if prop in run and run[prop]:
+            invalid.append(prop)
+    if not run.get('keep-artifacts', True):
+        invalid.append('keep-artifacts')
+    if invalid:
+        raise Exception("Jobs run using mozharness on Windows do not support properties " +
+                        ', '.join(invalid))
+
+    worker = taskdesc['worker']
+
+    worker['artifacts'] = [{
+        'path': r'public\build',
+        'type': 'directory',
+    }]
+
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool',
+        'TOOLTOOL_REV': 'master',
+    })
+
+    mh_command = [r'c:\mozilla-build\python\python.exe']
+    mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
+    for cfg in run['config']:
+        mh_command.append('--config ' + cfg.replace('/', '\\'))
+    mh_command.append('--branch ' + config.params['project'])
+    mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
+    for option in run.get('options', []):
+        mh_command.append('--' + option)
+
+    hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
+    hg_command.append('robustcheckout')
+    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
+    hg_command.append('--purge')
+    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
+    hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
+    hg_command.append(env['GECKO_HEAD_REPOSITORY'])
+    hg_command.append('.\\build\\src')
+
+    worker['command'] = [
+        ' '.join(hg_command),
+        ' '.join(mh_command)
+    ]
diff --git a/taskcluster/taskgraph/transforms/job/run_task.py b/taskcluster/taskgraph/transforms/job/run_task.py
new file mode 100644
index 000000000..296fe43ee
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/run_task.py
@@ -0,0 +1,59 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running jobs that are invoked via the `run-task` script.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import copy
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_support_vcs_checkout,
+)
+from voluptuous import Schema, Required, Any
+
+run_task_schema = Schema({
+    Required('using'): 'run-task',
+
+    # if true, add a cache at ~worker/.cache, which is where things like pip
+    # tend to hide their caches.  This cache is never added for level-1 jobs.
+    Required('cache-dotcache', default=False): bool,
+
+    # if true (the default), perform a checkout in /home/worker/checkouts/gecko
+    Required('checkout', default=True): bool,
+
+    # The command arguments to pass to the `run-task` script, after the
+    # checkout arguments.  If a list, it will be passed directly; otherwise
+    # it will be included in a single argument to `bash -cx`.
+    Required('command'): Any([basestring], basestring),
+})
+
+
+@run_job_using("docker-worker", "run-task", schema=run_task_schema)
+def docker_worker_run_task(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker'] = copy.deepcopy(job['worker'])
+
+    if run['checkout']:
+        docker_worker_support_vcs_checkout(config, job, taskdesc)
+
+    if run.get('cache-dotcache') and int(config.params['level']) > 1:
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'level-{level}-{project}-dotcache'.format(**config.params),
+            'mount-point': '/home/worker/.cache',
+        })
+
+    run_command = run['command']
+    if isinstance(run_command, basestring):
+        run_command = ['bash', '-cx', run_command]
+    command = ['/home/worker/bin/run-task']
+    if run['checkout']:
+        command.append('--vcs-checkout=/home/worker/checkouts/gecko')
+    command.append('--')
+    command.extend(run_command)
+    worker['command'] = command
diff --git a/taskcluster/taskgraph/transforms/job/spidermonkey.py b/taskcluster/taskgraph/transforms/job/spidermonkey.py
new file mode 100644
index 000000000..d78b78504
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py
@@ -0,0 +1,86 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running spidermonkey jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from voluptuous import Schema, Required, Optional, Any
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_public_artifacts,
+    docker_worker_support_vcs_checkout,
+)
+
+sm_run_schema = Schema({
+    Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate'),
+
+    # The SPIDERMONKEY_VARIANT
+    Required('spidermonkey-variant'): basestring,
+
+    # The tooltool manifest to use; default from sm-tooltool-config.sh  is used
+    # if omitted
+    Optional('tooltool-manifest'): basestring,
+})
+
+
+@run_job_using("docker-worker", "spidermonkey")
+@run_job_using("docker-worker", "spidermonkey-package")
+@run_job_using("docker-worker", "spidermonkey-mozjs-crate")
+def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    if int(config.params['level']) > 1:
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'level-{}-{}-build-spidermonkey-workspace'.format(
+                config.params['level'], config.params['project']),
+            'mount-point': "/home/worker/workspace",
+        })
+
+    docker_worker_add_public_artifacts(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZHARNESS_DISABLE': 'true',
+        'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'],
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+    })
+
+    # tooltool downloads; note that this script downloads using the API
+    # endpoiint directly, rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    if run.get('tooltool-manifest'):
+        env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest']
+
+    docker_worker_support_vcs_checkout(config, job, taskdesc)
+
+    script = "build-sm.sh"
+    if run['using'] == 'spidermonkey-package':
+        script = "build-sm-package.sh"
+    elif run['using'] == 'spidermonkey-mozjs-crate':
+        script = "build-sm-mozjs-crate.sh"
+
+    worker['command'] = [
+        '/home/worker/bin/run-task',
+        '--chown-recursive', '/home/worker/workspace',
+        '--chown-recursive', '/home/worker/tooltool-cache',
+        '--vcs-checkout', '/home/worker/workspace/build/src',
+        '--',
+        '/bin/bash',
+        '-c',
+        'cd /home/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script
+    ]
diff --git a/taskcluster/taskgraph/transforms/job/toolchain.py b/taskcluster/taskgraph/transforms/job/toolchain.py
new file mode 100644
index 000000000..d814f7824
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/job/toolchain.py
@@ -0,0 +1,115 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Support for running toolchain-building jobs via dedicated scripts
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from voluptuous import Schema, Required
+
+from taskgraph.transforms.job import run_job_using
+from taskgraph.transforms.job.common import (
+    docker_worker_add_tc_vcs_cache,
+    docker_worker_add_gecko_vcs_env_vars
+)
+
+toolchain_run_schema = Schema({
+    Required('using'): 'toolchain-script',
+
+    # the script (in taskcluster/scripts/misc) to run
+    Required('script'): basestring,
+})
+
+
+@run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema)
+def docker_worker_toolchain(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+    worker['artifacts'] = []
+    worker['caches'] = []
+
+    worker['artifacts'].append({
+        'name': 'public',
+        'path': '/home/worker/workspace/artifacts/',
+        'type': 'directory',
+    })
+
+    docker_worker_add_tc_vcs_cache(config, job, taskdesc)
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLS_DISABLE': 'true',
+    })
+
+    # tooltool downloads; note that this downloads using the API endpoint directly,
+    # rather than via relengapi-proxy
+    worker['caches'].append({
+        'type': 'persistent',
+        'name': 'tooltool-cache',
+        'mount-point': '/home/worker/tooltool-cache',
+    })
+    env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache'
+    env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool'
+    env['TOOLTOOL_REV'] = 'master'
+
+    command = ' && '.join([
+        "cd /home/worker/",
+        "./bin/checkout-sources.sh",
+        "./workspace/build/src/taskcluster/scripts/misc/" + run['script'],
+    ])
+    worker['command'] = ["/bin/bash", "-c", command]
+
+
+@run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema)
+def windows_toolchain(config, job, taskdesc):
+    run = job['run']
+
+    worker = taskdesc['worker']
+
+    worker['artifacts'] = [{
+        'path': r'public\build',
+        'type': 'directory',
+    }]
+
+    docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc)
+
+    # We fetch LLVM SVN into this.
+    svn_cache = 'level-{}-toolchain-clang-cl-build-svn'.format(config.params['level'])
+    worker['mounts'] = [{
+        'cache-name': svn_cache,
+        'path': r'llvm-sources',
+    }]
+    taskdesc['scopes'].extend([
+        'generic-worker:cache:' + svn_cache,
+    ])
+
+    env = worker['env']
+    env.update({
+        'MOZ_BUILD_DATE': config.params['moz_build_date'],
+        'MOZ_SCM_LEVEL': config.params['level'],
+        'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool',
+        'TOOLTOOL_REV': 'master',
+    })
+
+    hg = r'c:\Program Files\Mercurial\hg.exe'
+    hg_command = ['"{}"'.format(hg)]
+    hg_command.append('robustcheckout')
+    hg_command.extend(['--sharebase', 'y:\\hg-shared'])
+    hg_command.append('--purge')
+    hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
+    hg_command.extend(['--revision', '%GECKO_HEAD_REV%'])
+    hg_command.append('%GECKO_HEAD_REPOSITORY%')
+    hg_command.append('.\\build\\src')
+
+    bash = r'c:\mozilla-build\msys\bin\bash'
+    worker['command'] = [
+        ' '.join(hg_command),
+        # do something intelligent.
+        r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script'])
+    ]
diff --git a/taskcluster/taskgraph/transforms/l10n.py b/taskcluster/taskgraph/transforms/l10n.py
new file mode 100644
index 000000000..42137b558
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/l10n.py
@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Do transforms specific to l10n kind
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def mh_config_replace_project(config, jobs):
+    """ Replaces {project} in mh config entries with the current project """
+    # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
+    for job in jobs:
+        if not job['run'].get('using') == 'mozharness':
+            # Nothing to do, not mozharness
+            yield job
+            continue
+        job['run']['config'] = map(
+            lambda x: x.format(project=config.params['project']),
+            job['run']['config']
+            )
+        yield job
+
+
+@transforms.add
+def mh_options_replace_project(config, jobs):
+    """ Replaces {project} in mh option entries with the current project """
+    # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot
+    for job in jobs:
+        if not job['run'].get('using') == 'mozharness':
+            # Nothing to do, not mozharness
+            yield job
+            continue
+        job['run']['options'] = map(
+            lambda x: x.format(project=config.params['project']),
+            job['run']['options']
+            )
+        yield job
diff --git a/taskcluster/taskgraph/transforms/marionette_harness.py b/taskcluster/taskgraph/transforms/marionette_harness.py
new file mode 100644
index 000000000..a24db470c
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/marionette_harness.py
@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Set dynamic task description properties of the marionette-harness task.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def setup_task(config, tasks):
+    for task in tasks:
+        del task['name']
+        task['label'] = 'marionette-harness'
+        env = task['worker'].setdefault('env', {})
+        env.update({
+            'GECKO_BASE_REPOSITORY': config.params['base_repository'],
+            'GECKO_HEAD_REF': config.params['head_rev'],
+            'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+            'GECKO_HEAD_REV': config.params['head_rev'],
+            'MOZ_BUILD_DATE': config.params['moz_build_date'],
+            'MOZ_SCM_LEVEL': config.params['level'],
+        })
+
+        task['worker']['caches'] = [{
+            'type': 'persistent',
+            'name': 'level-{}-{}-tc-vcs'.format(
+                config.params['level'], config.params['project']),
+            'mount-point': "/home/worker/.tc-vcs",
+        }]
+
+        yield task
diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py
new file mode 100644
index 000000000..6e371e4ba
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -0,0 +1,648 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+These transformations take a task description and turn it into a TaskCluster
+task definition (along with attributes, label, etc.).  The input to these
+transformations is generic to any kind of task, but abstracts away some of the
+complexities of worker implementations, scopes, and treeherder annotations.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import time
+
+from taskgraph.util.treeherder import split_symbol
+from taskgraph.transforms.base import (
+    validate_schema,
+    TransformSequence
+)
+from voluptuous import Schema, Any, Required, Optional, Extra
+
+from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR
+
+# shortcut for a string where task references are allowed
+taskref_or_string = Any(
+    basestring,
+    {Required('task-reference'): basestring})
+
+# A task description is a general description of a TaskCluster task
+task_description_schema = Schema({
+    # the label for this task
+    Required('label'): basestring,
+
+    # description of the task (for metadata)
+    Required('description'): basestring,
+
+    # attributes for this task
+    Optional('attributes'): {basestring: object},
+
+    # dependencies of this task, keyed by name; these are passed through
+    # verbatim and subject to the interpretation of the Task's get_dependencies
+    # method.
+    Optional('dependencies'): {basestring: object},
+
+    # expiration and deadline times, relative to task creation, with units
+    # (e.g., "14 days").  Defaults are set based on the project.
+    Optional('expires-after'): basestring,
+    Optional('deadline-after'): basestring,
+
+    # custom routes for this task; the default treeherder routes will be added
+    # automatically
+    Optional('routes'): [basestring],
+
+    # custom scopes for this task; any scopes required for the worker will be
+    # added automatically
+    Optional('scopes'): [basestring],
+
+    # custom "task.extra" content
+    Optional('extra'): {basestring: object},
+
+    # treeherder-related information; see
+    # https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json
+    # If not specified, no treeherder extra information or routes will be
+    # added to the task
+    Optional('treeherder'): {
+        # either a bare symbol, or "grp(sym)".
+        'symbol': basestring,
+
+        # the job kind
+        'kind': Any('build', 'test', 'other'),
+
+        # tier for this task
+        'tier': int,
+
+        # task platform, in the form platform/collection, used to set
+        # treeherder.machine.platform and treeherder.collection or
+        # treeherder.labels
+        'platform': basestring,
+
+        # treeherder environments (defaults to both staging and production)
+        Required('environments', default=['production', 'staging']): ['production', 'staging'],
+    },
+
+    # information for indexing this build so its artifacts can be discovered;
+    # if omitted, the build will not be indexed.
+    Optional('index'): {
+        # the name of the product this build produces
+        'product': Any('firefox', 'mobile'),
+
+        # the names to use for this job in the TaskCluster index
+        'job-name': Any(
+            # Assuming the job is named "normally", this is the v2 job name,
+            # and the v1 and buildbot routes will be determined appropriately.
+            basestring,
+
+            # otherwise, give separate names for each of the legacy index
+            # routes; if a name is omitted, no corresponding route will be
+            # created.
+            {
+                # the name as it appears in buildbot routes
+                Optional('buildbot'): basestring,
+                Required('gecko-v2'): basestring,
+            }
+        ),
+
+        # The rank that the task will receive in the TaskCluster
+        # index.  A newly completed task supercedes the currently
+        # indexed task iff it has a higher rank.  If unspecified,
+        # 'by-tier' behavior will be used.
+        'rank': Any(
+            # Rank is equal the timestamp of the build_date for tier-1
+            # tasks, and zero for non-tier-1.  This sorts tier-{2,3}
+            # builds below tier-1 in the index.
+            'by-tier',
+
+            # Rank is given as an integer constant (e.g. zero to make
+            # sure a task is last in the index).
+            int,
+
+            # Rank is equal to the timestamp of the build_date.  This
+            # option can be used to override the 'by-tier' behavior
+            # for non-tier-1 tasks.
+            'build_date',
+        ),
+    },
+
+    # The `run_on_projects` attribute, defaulting to "all".  This dictates the
+    # projects on which this task should be included in the target task set.
+    # See the attributes documentation for details.
+    Optional('run-on-projects'): [basestring],
+
+    # If the task can be coalesced, this is the name used in the coalesce key
+    # the project, etc. will be added automatically.  Note that try (level 1)
+    # tasks are never coalesced
+    Optional('coalesce-name'): basestring,
+
+    # the provisioner-id/worker-type for the task.  The following parameters will
+    # be substituted in this string:
+    #  {level} -- the scm level of this push
+    'worker-type': basestring,
+
+    # information specific to the worker implementation that will run this task
+    'worker': Any({
+        Required('implementation'): Any('docker-worker', 'docker-engine'),
+
+        # For tasks that will run in docker-worker or docker-engine, this is the
+        # name of the docker image or in-tree docker image to run the task in.  If
+        # in-tree, then a dependency will be created automatically.  This is
+        # generally `desktop-test`, or an image that acts an awful lot like it.
+        Required('docker-image'): Any(
+            # a raw Docker image path (repo/image:tag)
+            basestring,
+            # an in-tree generated docker image (from `testing/docker/<name>`)
+            {'in-tree': basestring}
+        ),
+
+        # worker features that should be enabled
+        Required('relengapi-proxy', default=False): bool,
+        Required('chain-of-trust', default=False): bool,
+        Required('taskcluster-proxy', default=False): bool,
+        Required('allow-ptrace', default=False): bool,
+        Required('loopback-video', default=False): bool,
+        Required('loopback-audio', default=False): bool,
+
+        # caches to set up for the task
+        Optional('caches'): [{
+            # only one type is supported by any of the workers right now
+            'type': 'persistent',
+
+            # name of the cache, allowing re-use by subsequent tasks naming the
+            # same cache
+            'name': basestring,
+
+            # location in the task image where the cache will be mounted
+            'mount-point': basestring,
+        }],
+
+        # artifacts to extract from the task image after completion
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            'type': Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            'path': basestring,
+
+            # name of the produced artifact (root of the names for
+            # type=directory)
+            'name': basestring,
+        }],
+
+        # environment variables
+        Required('env', default={}): {basestring: taskref_or_string},
+
+        # the command to run
+        'command': [taskref_or_string],
+
+        # the maximum time to run, in seconds
+        'max-run-time': int,
+
+        # the exit status code that indicates the task should be retried
+        Optional('retry-exit-status'): int,
+
+    }, {
+        Required('implementation'): 'generic-worker',
+
+        # command is a list of commands to run, sequentially
+        'command': [taskref_or_string],
+
+        # artifacts to extract from the task image after completion; note that artifacts
+        # for the generic worker cannot have names
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            'type': Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            'path': basestring,
+        }],
+
+        # directories and/or files to be mounted
+        Optional('mounts'): [{
+            # a unique name for the cache volume
+            'cache-name': basestring,
+
+            # task image path for the cache
+            'path': basestring,
+        }],
+
+        # environment variables
+        Required('env', default={}): {basestring: taskref_or_string},
+
+        # the maximum time to run, in seconds
+        'max-run-time': int,
+
+        # os user groups for test task workers
+        Optional('os-groups', default=[]): [basestring],
+    }, {
+        Required('implementation'): 'buildbot-bridge',
+
+        # see
+        # https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml
+        'buildername': basestring,
+        'sourcestamp': {
+            'branch': basestring,
+            Optional('revision'): basestring,
+            Optional('repository'): basestring,
+            Optional('project'): basestring,
+        },
+        'properties': {
+            'product': basestring,
+            Extra: basestring,  # additional properties are allowed
+        },
+    }, {
+        'implementation': 'macosx-engine',
+
+        # A link for an executable to download
+        Optional('link'): basestring,
+
+        # the command to run
+        Required('command'): [taskref_or_string],
+
+        # environment variables
+        Optional('env'): {basestring: taskref_or_string},
+
+        # artifacts to extract from the task image after completion
+        Optional('artifacts'): [{
+            # type of artifact -- simple file, or recursive directory
+            Required('type'): Any('file', 'directory'),
+
+            # task image path from which to read artifact
+            Required('path'): basestring,
+
+            # name of the produced artifact (root of the names for
+            # type=directory)
+            Required('name'): basestring,
+        }],
+    }),
+
+    # The "when" section contains descriptions of the circumstances
+    # under which this task can be "optimized", that is, left out of the
+    # task graph because it is unnecessary.
+    Optional('when'): Any({
+        # This task only needs to be run if a file matching one of the given
+        # patterns has changed in the push.  The patterns use the mozpack
+        # match function (python/mozbuild/mozpack/path.py).
+        Optional('files-changed'): [basestring],
+    }),
+})
+
+GROUP_NAMES = {
+    'tc': 'Executed by TaskCluster',
+    'tc-e10s': 'Executed by TaskCluster with e10s',
+    'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster',
+    'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s',
+    'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster',
+    'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s',
+    'tc-M': 'Mochitests executed by TaskCluster',
+    'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s',
+    'tc-R': 'Reftests executed by TaskCluster',
+    'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
+    'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
+    'tc-W': 'Web platform tests executed by TaskCluster',
+    'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
+    'tc-X': 'Xpcshell tests executed by TaskCluster',
+    'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
+    'Aries': 'Aries Device Image',
+    'Nexus 5-L': 'Nexus 5-L Device Image',
+    'Cc': 'Toolchain builds',
+    'SM-tc': 'Spidermonkey builds',
+}
+UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
+
+BUILDBOT_ROUTE_TEMPLATES = [
+    "index.buildbot.branches.{project}.{job-name-buildbot}",
+    "index.buildbot.revisions.{head_rev}.{project}.{job-name-buildbot}",
+]
+
+V2_ROUTE_TEMPLATES = [
+    "index.gecko.v2.{project}.latest.{product}.{job-name-gecko-v2}",
+    "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name-gecko-v2}",
+    "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name-gecko-v2}",
+]
+
+# the roots of the treeherder routes, keyed by treeherder environment
+TREEHERDER_ROUTE_ROOTS = {
+    'production': 'tc-treeherder',
+    'staging': 'tc-treeherder-stage',
+}
+
+COALESCE_KEY = 'builds.{project}.{name}'
+
+# define a collection of payload builders, depending on the worker implementation
+payload_builders = {}
+
+
+def payload_builder(name):
+    def wrap(func):
+        payload_builders[name] = func
+        return func
+    return wrap
+
+
+@payload_builder('docker-worker')
+def build_docker_worker_payload(config, task, task_def):
+    worker = task['worker']
+
+    image = worker['docker-image']
+    if isinstance(image, dict):
+        docker_image_task = 'build-docker-image-' + image['in-tree']
+        task.setdefault('dependencies', {})['docker-image'] = docker_image_task
+        image = {
+            "path": "public/image.tar.zst",
+            "taskId": {"task-reference": "<docker-image>"},
+            "type": "task-image",
+        }
+
+    features = {}
+
+    if worker.get('relengapi-proxy'):
+        features['relengAPIProxy'] = True
+
+    if worker.get('taskcluster-proxy'):
+        features['taskclusterProxy'] = True
+
+    if worker.get('allow-ptrace'):
+        features['allowPtrace'] = True
+        task_def['scopes'].append('docker-worker:feature:allowPtrace')
+
+    if worker.get('chain-of-trust'):
+        features['chainOfTrust'] = True
+
+    capabilities = {}
+
+    for lo in 'audio', 'video':
+        if worker.get('loopback-' + lo):
+            capitalized = 'loopback' + lo.capitalize()
+            devices = capabilities.setdefault('devices', {})
+            devices[capitalized] = True
+            task_def['scopes'].append('docker-worker:capability:device:' + capitalized)
+
+    task_def['payload'] = payload = {
+        'command': worker['command'],
+        'image': image,
+        'env': worker['env'],
+    }
+
+    if 'max-run-time' in worker:
+        payload['maxRunTime'] = worker['max-run-time']
+
+    if 'retry-exit-status' in worker:
+        payload['onExitStatus'] = {'retry': [worker['retry-exit-status']]}
+
+    if 'artifacts' in worker:
+        artifacts = {}
+        for artifact in worker['artifacts']:
+            artifacts[artifact['name']] = {
+                'path': artifact['path'],
+                'type': artifact['type'],
+                'expires': task_def['expires'],  # always expire with the task
+            }
+        payload['artifacts'] = artifacts
+
+    if 'caches' in worker:
+        caches = {}
+        for cache in worker['caches']:
+            caches[cache['name']] = cache['mount-point']
+            task_def['scopes'].append('docker-worker:cache:' + cache['name'])
+        payload['cache'] = caches
+
+    if features:
+        payload['features'] = features
+    if capabilities:
+        payload['capabilities'] = capabilities
+
+    # coalesce / superseding
+    if 'coalesce-name' in task and int(config.params['level']) > 1:
+        key = COALESCE_KEY.format(
+            project=config.params['project'],
+            name=task['coalesce-name'])
+        payload['supersederUrl'] = "https://coalesce.mozilla-releng.net/v1/list/" + key
+
+
+@payload_builder('generic-worker')
+def build_generic_worker_payload(config, task, task_def):
+    worker = task['worker']
+
+    artifacts = []
+
+    for artifact in worker['artifacts']:
+        artifacts.append({
+            'path': artifact['path'],
+            'type': artifact['type'],
+            'expires': task_def['expires'],  # always expire with the task
+        })
+
+    mounts = []
+
+    for mount in worker.get('mounts', []):
+        mounts.append({
+            'cacheName': mount['cache-name'],
+            'directory': mount['path']
+        })
+
+    task_def['payload'] = {
+        'command': worker['command'],
+        'artifacts': artifacts,
+        'env': worker.get('env', {}),
+        'mounts': mounts,
+        'maxRunTime': worker['max-run-time'],
+        'osGroups': worker.get('os-groups', []),
+    }
+
+    if 'retry-exit-status' in worker:
+        raise Exception("retry-exit-status not supported in generic-worker")
+
+
+@payload_builder('macosx-engine')
+def build_macosx_engine_payload(config, task, task_def):
+    worker = task['worker']
+    artifacts = map(lambda artifact: {
+        'name': artifact['name'],
+        'path': artifact['path'],
+        'type': artifact['type'],
+        'expires': task_def['expires'],
+    }, worker['artifacts'])
+
+    task_def['payload'] = {
+        'link': worker['link'],
+        'command': worker['command'],
+        'env': worker['env'],
+        'artifacts': artifacts,
+    }
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def validate(config, tasks):
+    for task in tasks:
+        yield validate_schema(
+            task_description_schema, task,
+            "In task {!r}:".format(task.get('label', '?no-label?')))
+
+
+@transforms.add
+def add_index_routes(config, tasks):
+    for task in tasks:
+        index = task.get('index')
+        routes = task.setdefault('routes', [])
+
+        if not index:
+            yield task
+            continue
+
+        job_name = index['job-name']
+        # unpack the v2 name to v1 and buildbot names
+        if isinstance(job_name, basestring):
+            base_name, type_name = job_name.rsplit('-', 1)
+            job_name = {
+                'buildbot': base_name,
+                'gecko-v2': '{}-{}'.format(base_name, type_name),
+            }
+
+        if job_name['gecko-v2'] not in JOB_NAME_WHITELIST:
+            raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name['gecko-v2']))
+
+        subs = config.params.copy()
+        for n in job_name:
+            subs['job-name-' + n] = job_name[n]
+        subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S",
+                                                time.gmtime(config.params['build_date']))
+        subs['product'] = index['product']
+
+        if 'buildbot' in job_name:
+            for tpl in BUILDBOT_ROUTE_TEMPLATES:
+                routes.append(tpl.format(**subs))
+        if 'gecko-v2' in job_name:
+            for tpl in V2_ROUTE_TEMPLATES:
+                routes.append(tpl.format(**subs))
+
+        # The default behavior is to rank tasks according to their tier
+        extra_index = task.setdefault('extra', {}).setdefault('index', {})
+        rank = index.get('rank', 'by-tier')
+
+        if rank == 'by-tier':
+            # rank is zero for non-tier-1 tasks and based on pushid for others;
+            # this sorts tier-{2,3} builds below tier-1 in the index
+            tier = task.get('treeherder', {}).get('tier', 3)
+            extra_index['rank'] = 0 if tier > 1 else int(config.params['build_date'])
+        elif rank == 'build_date':
+            extra_index['rank'] = int(config.params['build_date'])
+        else:
+            extra_index['rank'] = rank
+
+        del task['index']
+        yield task
+
+
+@transforms.add
+def build_task(config, tasks):
+    for task in tasks:
+        worker_type = task['worker-type'].format(level=str(config.params['level']))
+        provisioner_id, worker_type = worker_type.split('/', 1)
+
+        routes = task.get('routes', [])
+        scopes = task.get('scopes', [])
+
+        # set up extra
+        extra = task.get('extra', {})
+        task_th = task.get('treeherder')
+        if task_th:
+            extra['treeherderEnv'] = task_th['environments']
+
+            treeherder = extra.setdefault('treeherder', {})
+
+            machine_platform, collection = task_th['platform'].split('/', 1)
+            treeherder['machine'] = {'platform': machine_platform}
+            treeherder['collection'] = {collection: True}
+
+            groupSymbol, symbol = split_symbol(task_th['symbol'])
+            if groupSymbol != '?':
+                treeherder['groupSymbol'] = groupSymbol
+                if groupSymbol not in GROUP_NAMES:
+                    raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol))
+                treeherder['groupName'] = GROUP_NAMES[groupSymbol]
+            treeherder['symbol'] = symbol
+            treeherder['jobKind'] = task_th['kind']
+            treeherder['tier'] = task_th['tier']
+
+            routes.extend([
+                '{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env],
+                                        config.params['project'],
+                                        config.params['head_rev'],
+                                        config.params['pushlog_id'])
+                for env in task_th['environments']
+            ])
+
+        if 'expires-after' not in task:
+            task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year'
+
+        if 'deadline-after' not in task:
+            task['deadline-after'] = '1 day'
+
+        if 'coalesce-name' in task and int(config.params['level']) > 1:
+            key = COALESCE_KEY.format(
+                project=config.params['project'],
+                name=task['coalesce-name'])
+            routes.append('coalesce.v1.' + key)
+
+        task_def = {
+            'provisionerId': provisioner_id,
+            'workerType': worker_type,
+            'routes': routes,
+            'created': {'relative-datestamp': '0 seconds'},
+            'deadline': {'relative-datestamp': task['deadline-after']},
+            'expires': {'relative-datestamp': task['expires-after']},
+            'scopes': scopes,
+            'metadata': {
+                'description': task['description'],
+                'name': task['label'],
+                'owner': config.params['owner'],
+                'source': '{}/file/{}/{}'.format(
+                    config.params['head_repository'],
+                    config.params['head_rev'],
+                    config.path),
+            },
+            'extra': extra,
+            'tags': {'createdForUser': config.params['owner']},
+        }
+
+        # add the payload and adjust anything else as required (e.g., scopes)
+        payload_builders[task['worker']['implementation']](config, task, task_def)
+
+        attributes = task.get('attributes', {})
+        attributes['run_on_projects'] = task.get('run-on-projects', ['all'])
+
+        yield {
+            'label': task['label'],
+            'task': task_def,
+            'dependencies': task.get('dependencies', {}),
+            'attributes': attributes,
+            'when': task.get('when', {}),
+        }
+
+
+# Check that the v2 route templates match those used by Mozharness.  This can
+# go away once Mozharness builds are no longer performed in Buildbot, and the
+# Mozharness code referencing routes.json is deleted.
+def check_v2_routes():
+    with open("testing/mozharness/configs/routes.json", "rb") as f:
+        routes_json = json.load(f)
+
+    # we only deal with the 'routes' key here
+    routes = routes_json['routes']
+
+    # we use different variables than mozharness
+    for mh, tg in [
+            ('{index}', 'index'),
+            ('{build_product}', '{product}'),
+            ('{build_name}-{build_type}', '{job-name-gecko-v2}'),
+            ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]:
+        routes = [r.replace(mh, tg) for r in routes]
+
+    if sorted(routes) != sorted(V2_ROUTE_TEMPLATES):
+        raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: "
+                        "%s vs %s" % (V2_ROUTE_TEMPLATES, routes))
+
+check_v2_routes()
diff --git a/taskcluster/taskgraph/transforms/tests/__init__.py b/taskcluster/taskgraph/transforms/tests/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/__init__.py
diff --git a/taskcluster/taskgraph/transforms/tests/all_kinds.py b/taskcluster/taskgraph/transforms/tests/all_kinds.py
new file mode 100644
index 000000000..f2aa1f841
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/all_kinds.py
@@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Changes here apply to all tests, regardless of kind.
+
+This is a great place for:
+
+ * Applying rules based on platform, project, etc. that should span kinds
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.util.treeherder import split_symbol, join_symbol
+from taskgraph.transforms.base import TransformSequence, get_keyed_by
+
+import copy
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_worker_implementation(config, tests):
+    """Set the worker implementation based on the test platform."""
+    for test in tests:
+        if test['test-platform'].startswith('win'):
+            test['worker-implementation'] = 'generic-worker'
+        elif test['test-platform'].startswith('macosx'):
+            test['worker-implementation'] = 'macosx-engine'
+        else:
+            test['worker-implementation'] = 'docker-worker'
+        yield test
+
+
+@transforms.add
+def set_tier(config, tests):
+    """Set the tier based on policy for all test descriptions that do not
+    specify a tier otherwise."""
+    for test in tests:
+        # only override if not set for the test
+        if 'tier' not in test:
+            if test['test-platform'] in ['linux64/debug',
+                                         'linux64-asan/opt',
+                                         'android-4.3-arm7-api-15/debug',
+                                         'android-x86/opt']:
+                test['tier'] = 1
+            else:
+                test['tier'] = 2
+        yield test
+
+
+@transforms.add
+def set_expires_after(config, tests):
+    """Try jobs expire after 2 weeks; everything else lasts 1 year.  This helps
+    keep storage costs low."""
+    for test in tests:
+        if 'expires-after' not in test:
+            if config.params['project'] == 'try':
+                test['expires-after'] = "14 days"
+            else:
+                test['expires-after'] = "1 year"
+        yield test
+
+
+@transforms.add
+def set_download_symbols(config, tests):
+    """In general, we download symbols immediately for debug builds, but only
+    on demand for everything else. ASAN builds shouldn't download
+    symbols since they don't product symbol zips see bug 1283879"""
+    for test in tests:
+        if test['test-platform'].split('/')[-1] == 'debug':
+            test['mozharness']['download-symbols'] = True
+        elif test['build-platform'] == 'linux64-asan/opt':
+            if 'download-symbols' in test['mozharness']:
+                del test['mozharness']['download-symbols']
+        else:
+            test['mozharness']['download-symbols'] = 'ondemand'
+        yield test
+
+
+@transforms.add
+def resolve_keyed_by(config, tests):
+    """Resolve fields that can be keyed by platform, etc."""
+    fields = [
+        'instance-size',
+        'max-run-time',
+        'chunks',
+        'e10s',
+        'suite',
+        'run-on-projects',
+    ]
+    for test in tests:
+        for field in fields:
+            test[field] = get_keyed_by(item=test, field=field, item_name=test['test-name'])
+        test['mozharness']['config'] = get_keyed_by(item=test,
+                                                    field='mozharness',
+                                                    subfield='config',
+                                                    item_name=test['test-name'])
+        test['mozharness']['extra-options'] = get_keyed_by(item=test,
+                                                           field='mozharness',
+                                                           subfield='extra-options',
+                                                           item_name=test['test-name'])
+        yield test
+
+
+@transforms.add
+def split_chunks(config, tests):
+    """Based on the 'chunks' key, split tests up into chunks by duplicating
+    them and assigning 'this-chunk' appropriately and updating the treeherder
+    symbol."""
+    for test in tests:
+        if test['chunks'] == 1:
+            test['this-chunk'] = 1
+            yield test
+            continue
+
+        for this_chunk in range(1, test['chunks'] + 1):
+            # copy the test and update with the chunk number
+            chunked = copy.deepcopy(test)
+            chunked['this-chunk'] = this_chunk
+
+            # add the chunk number to the TH symbol
+            group, symbol = split_symbol(chunked['treeherder-symbol'])
+            symbol += str(this_chunk)
+            chunked['treeherder-symbol'] = join_symbol(group, symbol)
+
+            yield chunked
+
+
+@transforms.add
+def set_retry_exit_status(config, tests):
+    """Set the retry exit status to TBPL_RETRY, the value returned by mozharness
+       scripts to indicate a transient failure that should be retried."""
+    for test in tests:
+        test['retry-exit-status'] = 4
+        yield test
diff --git a/taskcluster/taskgraph/transforms/tests/android_test.py b/taskcluster/taskgraph/transforms/tests/android_test.py
new file mode 100644
index 000000000..7c13b16f5
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/android_test.py
@@ -0,0 +1,42 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+These transforms are specific to the android-test kind, and apply defaults to
+the test descriptions appropriate to that kind.
+
+Both the input to and output from these transforms must conform to
+`taskgraph.transforms.tests.test:test_schema`.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.transforms.base import TransformSequence
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, tests):
+    for test in tests:
+        # all Android test tasks download internal objects from tooltool
+        test['mozharness']['tooltool-downloads'] = True
+        test['mozharness']['build-artifact-name'] = 'public/build/target.apk'
+        test['mozharness']['actions'] = ['get-secrets']
+        yield test
+
+
+@transforms.add
+def set_treeherder_machine_platform(config, tests):
+    """Set the appropriate task.extra.treeherder.machine.platform"""
+    # The build names for these build platforms have partially evolved over the
+    # years..  This is temporary until we can clean up the handling of
+    # platforms
+    translation = {
+        'android-api-15/debug': 'android-4-3-armv7-api15/debug',
+        'android-api-15/opt': 'android-4-3-armv7-api15/opt',
+        'android-x86/opt': 'android-4-2-x86/opt',
+    }
+    for test in tests:
+        build_platform = test['build-platform']
+        test['treeherder-machine-platform'] = translation.get(build_platform, build_platform)
+        yield test
diff --git a/taskcluster/taskgraph/transforms/tests/desktop_test.py b/taskcluster/taskgraph/transforms/tests/desktop_test.py
new file mode 100644
index 000000000..44a907903
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/desktop_test.py
@@ -0,0 +1,118 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+These transforms are specific to the desktop-test kind, and apply defaults to
+the test descriptions appropriate to that kind.
+
+Both the input to and output from these transforms must conform to
+`taskgraph.transforms.tests.test:test_schema`.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+from taskgraph.transforms.base import TransformSequence, get_keyed_by
+from taskgraph.util.treeherder import split_symbol, join_symbol
+
+import copy
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def set_defaults(config, tests):
+    for test in tests:
+        build_platform = test['build-platform']
+        if build_platform.startswith('macosx'):
+            target = 'target.dmg'
+        else:
+            target = 'target.tar.bz2'
+        test['mozharness']['build-artifact-name'] = 'public/build/' + target
+        # all desktop tests want to run the bits that require node
+        test['mozharness']['set-moz-node-path'] = True
+        yield test
+
+
+@transforms.add
+def set_treeherder_machine_platform(config, tests):
+    """Set the appropriate task.extra.treeherder.machine.platform"""
+    # Linux64 build platforms for asan and pgo are specified differently to
+    # treeherder.  This is temporary until we can clean up the handling of
+    # platforms
+    translation = {
+        'linux64-asan/opt': 'linux64/asan',
+        'linux64-pgo/opt': 'linux64/pgo',
+        'macosx64/debug': 'osx-10-10/debug',
+        'macosx64/opt': 'osx-10-10/opt',
+    }
+    for test in tests:
+        build_platform = test['build-platform']
+        test_platform = test['test-platform']
+        test['treeherder-machine-platform'] = translation.get(build_platform, test_platform)
+        yield test
+
+
+@transforms.add
+def set_asan_docker_image(config, tests):
+    """Set the appropriate task.extra.treeherder.docker-image"""
+    # Linux64-asan has many leaks with running mochitest-media jobs
+    # on Ubuntu 16.04, please remove this when bug 1289209 is resolved
+    for test in tests:
+        if test['suite'] == 'mochitest/mochitest-media' and \
+           test['build-platform'] == 'linux64-asan/opt':
+            test['docker-image'] = {"in-tree": "desktop-test"}
+        yield test
+
+
+@transforms.add
+def split_e10s(config, tests):
+    for test in tests:
+        e10s = get_keyed_by(item=test, field='e10s',
+                            item_name=test['test-name'])
+        test.setdefault('attributes', {})
+        test['e10s'] = False
+        test['attributes']['e10s'] = False
+
+        if e10s == 'both':
+            yield test
+            test = copy.deepcopy(test)
+            e10s = True
+        if e10s:
+            test['test-name'] += '-e10s'
+            test['e10s'] = True
+            test['attributes']['e10s'] = True
+            group, symbol = split_symbol(test['treeherder-symbol'])
+            if group != '?':
+                group += '-e10s'
+            test['treeherder-symbol'] = join_symbol(group, symbol)
+            test['mozharness'].setdefault('extra-options', []).append('--e10s')
+        yield test
+
+
+@transforms.add
+def allow_software_gl_layers(config, tests):
+    for test in tests:
+
+        # since this value defaults to true, but is not applicable on windows,
+        # it's overriden for that platform here.
+        allow = not test['test-platform'].startswith('win') \
+            and get_keyed_by(item=test, field='allow-software-gl-layers',
+                             item_name=test['test-name'])
+        if allow:
+            assert test['instance-size'] != 'legacy',\
+                   'Software GL layers on a legacy instance is disallowed (bug 1296086).'
+
+            # This should be set always once bug 1296086 is resolved.
+            test['mozharness'].setdefault('extra-options', [])\
+                              .append("--allow-software-gl-layers")
+
+        yield test
+
+
+@transforms.add
+def add_os_groups(config, tests):
+    for test in tests:
+        if test['test-platform'].startswith('win'):
+            groups = get_keyed_by(item=test, field='os-groups', item_name=test['test-name'])
+            if groups:
+                test['os-groups'] = groups
+        yield test
diff --git a/taskcluster/taskgraph/transforms/tests/make_task_description.py b/taskcluster/taskgraph/transforms/tests/make_task_description.py
new file mode 100644
index 000000000..fc3f94893
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py
@@ -0,0 +1,445 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+These transforms construct a task description to run the given test, based on a
+test description.  The implementation here is shared among all test kinds, but
+contains specific support for how we run tests in Gecko (via mozharness,
+invoked in particular ways).
+
+This is a good place to translate a test-description option such as
+`single-core: true` to the implementation of that option in a task description
+(worker options, mozharness commandline, environment variables, etc.)
+
+The test description should be fully formed by the time it reaches these
+transforms, and these transforms should not embody any specific knowledge about
+what should run where. this is the wrong place for special-casing platforms,
+for example - use `all_tests.py` instead.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.transforms.job.common import (
+    docker_worker_support_vcs_checkout,
+)
+
+import logging
+import os.path
+
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+WORKER_TYPE = {
+    # default worker types keyed by instance-size
+    'large': 'aws-provisioner-v1/gecko-t-linux-large',
+    'xlarge': 'aws-provisioner-v1/gecko-t-linux-xlarge',
+    'legacy': 'aws-provisioner-v1/gecko-t-linux-medium',
+    'default': 'aws-provisioner-v1/gecko-t-linux-large',
+    # windows worker types keyed by test-platform
+    'windows7-32-vm': 'aws-provisioner-v1/gecko-t-win7-32',
+    'windows7-32': 'aws-provisioner-v1/gecko-t-win7-32-gpu',
+    'windows10-64-vm': 'aws-provisioner-v1/gecko-t-win10-64',
+    'windows10-64': 'aws-provisioner-v1/gecko-t-win10-64-gpu'
+}
+
+ARTIFACTS = [
+    # (artifact name prefix, in-image path)
+    ("public/logs/", "build/upload/logs/"),
+    ("public/test", "artifacts/"),
+    ("public/test_info/", "build/blobber_upload_dir/"),
+]
+
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def make_task_description(config, tests):
+    """Convert *test* descriptions to *task* descriptions (input to
+    taskgraph.transforms.task)"""
+
+    for test in tests:
+        label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name'])
+        if test['chunks'] > 1:
+            label += '-{}'.format(test['this-chunk'])
+
+        build_label = test['build-label']
+
+        unittest_try_name = test.get('unittest-try-name', test['test-name'])
+
+        attr_build_platform, attr_build_type = test['build-platform'].split('/', 1)
+
+        suite = test['suite']
+        if '/' in suite:
+            suite, flavor = suite.split('/', 1)
+        else:
+            flavor = suite
+
+        attributes = test.get('attributes', {})
+        attributes.update({
+            'build_platform': attr_build_platform,
+            'build_type': attr_build_type,
+            # only keep the first portion of the test platform
+            'test_platform': test['test-platform'].split('/')[0],
+            'test_chunk': str(test['this-chunk']),
+            'unittest_suite': suite,
+            'unittest_flavor': flavor,
+            'unittest_try_name': unittest_try_name,
+        })
+
+        taskdesc = {}
+        taskdesc['label'] = label
+        taskdesc['description'] = test['description']
+        taskdesc['attributes'] = attributes
+        taskdesc['dependencies'] = {'build': build_label}
+        taskdesc['deadline-after'] = '1 day'
+        taskdesc['expires-after'] = test['expires-after']
+        taskdesc['routes'] = []
+        taskdesc['run-on-projects'] = test.get('run-on-projects', ['all'])
+        taskdesc['scopes'] = []
+        taskdesc['extra'] = {
+            'chunks': {
+                'current': test['this-chunk'],
+                'total': test['chunks'],
+            },
+            'suite': {
+                'name': suite,
+                'flavor': flavor,
+            },
+        }
+        taskdesc['treeherder'] = {
+            'symbol': test['treeherder-symbol'],
+            'kind': 'test',
+            'tier': test['tier'],
+            'platform': test.get('treeherder-machine-platform', test['build-platform']),
+        }
+
+        # the remainder (the worker-type and worker) differs depending on the
+        # worker implementation
+        worker_setup_functions[test['worker-implementation']](config, test, taskdesc)
+
+        # yield only the task description, discarding the test description
+        yield taskdesc
+
+
+worker_setup_functions = {}
+
+
+def worker_setup_function(name):
+    def wrap(func):
+        worker_setup_functions[name] = func
+        return func
+    return wrap
+
+
+@worker_setup_function("docker-engine")
+@worker_setup_function("docker-worker")
+def docker_worker_setup(config, test, taskdesc):
+
+    artifacts = [
+        # (artifact name prefix, in-image path)
+        ("public/logs/", "/home/worker/workspace/build/upload/logs/"),
+        ("public/test", "/home/worker/artifacts/"),
+        ("public/test_info/", "/home/worker/workspace/build/blobber_upload_dir/"),
+    ]
+    mozharness = test['mozharness']
+
+    installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name'])
+    test_packages_url = ARTIFACT_URL.format('<build>',
+                                            'public/build/target.test_packages.json')
+    mozharness_url = ARTIFACT_URL.format('<build>',
+                                         'public/build/mozharness.zip')
+
+    taskdesc['worker-type'] = WORKER_TYPE[test['instance-size']]
+
+    worker = taskdesc['worker'] = {}
+    worker['implementation'] = test['worker-implementation']
+    worker['docker-image'] = test['docker-image']
+
+    worker['allow-ptrace'] = True  # required for all tests, for crashreporter
+    worker['relengapi-proxy'] = False  # but maybe enabled for tooltool below
+    worker['loopback-video'] = test['loopback-video']
+    worker['loopback-audio'] = test['loopback-audio']
+    worker['max-run-time'] = test['max-run-time']
+    worker['retry-exit-status'] = test['retry-exit-status']
+
+    worker['artifacts'] = [{
+        'name': prefix,
+        'path': os.path.join('/home/worker/workspace', path),
+        'type': 'directory',
+    } for (prefix, path) in artifacts]
+
+    worker['caches'] = [{
+        'type': 'persistent',
+        'name': 'level-{}-{}-test-workspace'.format(
+            config.params['level'], config.params['project']),
+        'mount-point': "/home/worker/workspace",
+    }]
+
+    env = worker['env'] = {
+        'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
+        'MOZHARNESS_SCRIPT': mozharness['script'],
+        'MOZILLA_BUILD_URL': {'task-reference': installer_url},
+        'NEED_PULSEAUDIO': 'true',
+        'NEED_WINDOW_MANAGER': 'true',
+    }
+
+    if mozharness['set-moz-node-path']:
+        env['MOZ_NODE_PATH'] = '/usr/local/bin/node'
+
+    if 'actions' in mozharness:
+        env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions'])
+
+    if config.params['project'] == 'try':
+        env['TRY_COMMIT_MSG'] = config.params['message']
+
+    # handle some of the mozharness-specific options
+
+    if mozharness['tooltool-downloads']:
+        worker['relengapi-proxy'] = True
+        worker['caches'].append({
+            'type': 'persistent',
+            'name': 'tooltool-cache',
+            'mount-point': '/home/worker/tooltool-cache',
+        })
+        taskdesc['scopes'].extend([
+            'docker-worker:relengapi-proxy:tooltool.download.internal',
+            'docker-worker:relengapi-proxy:tooltool.download.public',
+        ])
+
+    # assemble the command line
+    command = [
+        '/home/worker/bin/run-task',
+        # The workspace cache/volume is default owned by root:root.
+        '--chown', '/home/worker/workspace',
+    ]
+
+    # Support vcs checkouts regardless of whether the task runs from
+    # source or not in case it is needed on an interactive loaner.
+    docker_worker_support_vcs_checkout(config, test, taskdesc)
+
+    # If we have a source checkout, run mozharness from it instead of
+    # downloading a zip file with the same content.
+    if test['checkout']:
+        command.extend(['--vcs-checkout', '/home/worker/checkouts/gecko'])
+        env['MOZHARNESS_PATH'] = '/home/worker/checkouts/gecko/testing/mozharness'
+    else:
+        env['MOZHARNESS_URL'] = {'task-reference': mozharness_url}
+
+    command.extend([
+        '--',
+        '/home/worker/bin/test-linux.sh',
+    ])
+
+    if mozharness.get('no-read-buildbot-config'):
+        command.append("--no-read-buildbot-config")
+    command.extend([
+        {"task-reference": "--installer-url=" + installer_url},
+        {"task-reference": "--test-packages-url=" + test_packages_url},
+    ])
+    command.extend(mozharness.get('extra-options', []))
+
+    # TODO: remove the need for run['chunked']
+    if mozharness.get('chunked') or test['chunks'] > 1:
+        # Implement mozharness['chunking-args'], modifying command in place
+        if mozharness['chunking-args'] == 'this-chunk':
+            command.append('--total-chunk={}'.format(test['chunks']))
+            command.append('--this-chunk={}'.format(test['this-chunk']))
+        elif mozharness['chunking-args'] == 'test-suite-suffix':
+            suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
+            for i, c in enumerate(command):
+                if isinstance(c, basestring) and c.startswith('--test-suite'):
+                    command[i] += suffix
+
+    if 'download-symbols' in mozharness:
+        download_symbols = mozharness['download-symbols']
+        download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
+        command.append('--download-symbols=' + download_symbols)
+
+    worker['command'] = command
+
+
+def normpath(path):
+    return path.replace('/', '\\')
+
+
+def get_firefox_version():
+    with open('browser/config/version.txt', 'r') as f:
+        return f.readline().strip()
+
+
+@worker_setup_function('generic-worker')
+def generic_worker_setup(config, test, taskdesc):
+    artifacts = [
+        {
+            'path': 'public\\logs\\localconfig.json',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_critical.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_error.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_fatal.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_info.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_raw.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\logs\\log_warning.log',
+            'type': 'file'
+        },
+        {
+            'path': 'public\\test_info',
+            'type': 'directory'
+        }
+    ]
+    mozharness = test['mozharness']
+
+    build_platform = taskdesc['attributes']['build_platform']
+    test_platform = test['test-platform'].split('/')[0]
+
+    target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform)
+
+    installer_url = ARTIFACT_URL.format(
+        '<build>', 'public/build/{}.zip'.format(target))
+    test_packages_url = ARTIFACT_URL.format(
+        '<build>', 'public/build/{}.test_packages.json'.format(target))
+    mozharness_url = ARTIFACT_URL.format(
+        '<build>', 'public/build/mozharness.zip')
+
+    taskdesc['worker-type'] = WORKER_TYPE[test_platform]
+
+    taskdesc['scopes'].extend(
+        ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']])
+
+    worker = taskdesc['worker'] = {}
+    worker['os-groups'] = test['os-groups']
+    worker['implementation'] = test['worker-implementation']
+    worker['max-run-time'] = test['max-run-time']
+    worker['artifacts'] = artifacts
+
+    env = worker['env'] = {
+        # Bug 1306989
+        'APPDATA': '%cd%\\AppData\\Roaming',
+        'LOCALAPPDATA': '%cd%\\AppData\\Local',
+        'TEMP': '%cd%\\AppData\\Local\\Temp',
+        'TMP': '%cd%\\AppData\\Local\\Temp',
+        'USERPROFILE': '%cd%',
+    }
+
+    # assemble the command line
+    mh_command = [
+        'c:\\mozilla-build\\python\\python.exe',
+        '-u',
+        'mozharness\\scripts\\' + normpath(mozharness['script'])
+    ]
+    for mh_config in mozharness['config']:
+        mh_command.extend(['--cfg', 'mozharness\\configs\\' + normpath(mh_config)])
+    mh_command.extend(mozharness.get('extra-options', []))
+    if mozharness.get('no-read-buildbot-config'):
+        mh_command.append('--no-read-buildbot-config')
+    mh_command.extend(['--installer-url', installer_url])
+    mh_command.extend(['--test-packages-url', test_packages_url])
+    if mozharness.get('download-symbols'):
+        if isinstance(mozharness['download-symbols'], basestring):
+            mh_command.extend(['--download-symbols', mozharness['download-symbols']])
+        else:
+            mh_command.extend(['--download-symbols', 'true'])
+
+    # TODO: remove the need for run['chunked']
+    if mozharness.get('chunked') or test['chunks'] > 1:
+        # Implement mozharness['chunking-args'], modifying command in place
+        if mozharness['chunking-args'] == 'this-chunk':
+            mh_command.append('--total-chunk={}'.format(test['chunks']))
+            mh_command.append('--this-chunk={}'.format(test['this-chunk']))
+        elif mozharness['chunking-args'] == 'test-suite-suffix':
+            suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
+            for i, c in enumerate(mh_command):
+                if isinstance(c, basestring) and c.startswith('--test-suite'):
+                    mh_command[i] += suffix
+
+    worker['command'] = [
+        'mkdir {} {}'.format(env['APPDATA'], env['TMP']),
+        {'task-reference': 'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)},
+        'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip',
+        {'task-reference': ' '.join(mh_command)},
+        'xcopy build\\blobber_upload_dir public\\test_info /e /i',
+        'copy /y logs\\*.* public\\logs\\'
+    ]
+
+
+@worker_setup_function("macosx-engine")
+def macosx_engine_setup(config, test, taskdesc):
+    mozharness = test['mozharness']
+
+    installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name'])
+    test_packages_url = ARTIFACT_URL.format('<build>',
+                                            'public/build/target.test_packages.json')
+    mozharness_url = ARTIFACT_URL.format('<build>',
+                                         'public/build/mozharness.zip')
+
+    # for now we have only 10.10 machines
+    taskdesc['worker-type'] = 'tc-worker-provisioner/gecko-t-osx-10-10'
+
+    worker = taskdesc['worker'] = {}
+    worker['implementation'] = test['worker-implementation']
+
+    worker['artifacts'] = [{
+        'name': prefix.rstrip('/'),
+        'path': path.rstrip('/'),
+        'type': 'directory',
+    } for (prefix, path) in ARTIFACTS]
+
+    worker['env'] = {
+        'GECKO_HEAD_REPOSITORY': config.params['head_repository'],
+        'GECKO_HEAD_REV': config.params['head_rev'],
+        'MOZHARNESS_CONFIG': ' '.join(mozharness['config']),
+        'MOZHARNESS_SCRIPT': mozharness['script'],
+        'MOZHARNESS_URL': {'task-reference': mozharness_url},
+        'MOZILLA_BUILD_URL': {'task-reference': installer_url},
+    }
+
+    # assemble the command line
+
+    worker['link'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format(
+        config.params['head_repository'], config.params['head_rev']
+    )
+
+    command = worker['command'] = ["./test-macosx.sh"]
+    if mozharness.get('no-read-buildbot-config'):
+        command.append("--no-read-buildbot-config")
+    command.extend([
+        {"task-reference": "--installer-url=" + installer_url},
+        {"task-reference": "--test-packages-url=" + test_packages_url},
+    ])
+    if mozharness.get('include-blob-upload-branch'):
+        command.append('--blob-upload-branch=' + config.params['project'])
+    command.extend(mozharness.get('extra-options', []))
+
+    # TODO: remove the need for run['chunked']
+    if mozharness.get('chunked') or test['chunks'] > 1:
+        # Implement mozharness['chunking-args'], modifying command in place
+        if mozharness['chunking-args'] == 'this-chunk':
+            command.append('--total-chunk={}'.format(test['chunks']))
+            command.append('--this-chunk={}'.format(test['this-chunk']))
+        elif mozharness['chunking-args'] == 'test-suite-suffix':
+            suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk']))
+            for i, c in enumerate(command):
+                if isinstance(c, basestring) and c.startswith('--test-suite'):
+                    command[i] += suffix
+
+    if 'download-symbols' in mozharness:
+        download_symbols = mozharness['download-symbols']
+        download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols)
+        command.append('--download-symbols=' + download_symbols)
diff --git a/taskcluster/taskgraph/transforms/tests/test_description.py b/taskcluster/taskgraph/transforms/tests/test_description.py
new file mode 100644
index 000000000..1365919fe
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/tests/test_description.py
@@ -0,0 +1,235 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This file defines the schema for tests -- the things in `tests.yml`.  It should
+be run both before and after the kind-specific transforms, to ensure that the
+transforms do not generate invalid tests.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import validate_schema
+from voluptuous import (
+    Any,
+    Optional,
+    Required,
+    Schema,
+)
+
+
+# Schema for a test description
+#
+# *****WARNING*****
+#
+# This is a great place for baffling cruft to accumulate, and that makes
+# everyone move more slowly.  Be considerate of your fellow hackers!
+# See the warnings in taskcluster/docs/how-tos.rst
+#
+# *****WARNING*****
+test_description_schema = Schema({
+    # description of the suite, for the task metadata
+    'description': basestring,
+
+    # test suite name, or <suite>/<flavor>
+    Required('suite'): Any(
+        basestring,
+        {'by-test-platform': {basestring: basestring}},
+    ),
+
+    # the name by which this test suite is addressed in try syntax; defaults to
+    # the test-name
+    Optional('unittest-try-name'): basestring,
+
+    # the symbol, or group(symbol), under which this task should appear in
+    # treeherder.
+    'treeherder-symbol': basestring,
+
+    # the value to place in task.extra.treeherder.machine.platform; ideally
+    # this is the same as build-platform, and that is the default, but in
+    # practice it's not always a match.
+    Optional('treeherder-machine-platform'): basestring,
+
+    # attributes to appear in the resulting task (later transforms will add the
+    # common attributes)
+    Optional('attributes'): {basestring: object},
+
+    # The `run_on_projects` attribute, defaulting to "all".  This dictates the
+    # projects on which this task should be included in the target task set.
+    # See the attributes documentation for details.
+    Optional('run-on-projects', default=['all']): Any(
+        [basestring],
+        {'by-test-platform': {basestring: [basestring]}},
+    ),
+
+    # the sheriffing tier for this task (default: set based on test platform)
+    Optional('tier'): int,
+
+    # number of chunks to create for this task.  This can be keyed by test
+    # platform by passing a dictionary in the `by-test-platform` key.  If the
+    # test platform is not found, the key 'default' will be tried.
+    Required('chunks', default=1): Any(
+        int,
+        {'by-test-platform': {basestring: int}},
+    ),
+
+    # the time (with unit) after which this task is deleted; default depends on
+    # the branch (see below)
+    Optional('expires-after'): basestring,
+
+    # Whether to run this task with e10s (desktop-test only).  If false, run
+    # without e10s; if true, run with e10s; if 'both', run one task with and
+    # one task without e10s.  E10s tasks have "-e10s" appended to the test name
+    # and treeherder group.
+    Required('e10s', default='both'): Any(
+        bool, 'both',
+        {'by-test-platform': {basestring: Any(bool, 'both')}},
+    ),
+
+    # The EC2 instance size to run these tests on.
+    Required('instance-size', default='default'): Any(
+        Any('default', 'large', 'xlarge', 'legacy'),
+        {'by-test-platform': {basestring: Any('default', 'large', 'xlarge', 'legacy')}},
+    ),
+
+    # Whether the task requires loopback audio or video (whatever that may mean
+    # on the platform)
+    Required('loopback-audio', default=False): bool,
+    Required('loopback-video', default=False): bool,
+
+    # Whether the test can run using a software GL implementation on Linux
+    # using the GL compositor. May not be used with "legacy" sized instances
+    # due to poor LLVMPipe performance (bug 1296086).
+    Optional('allow-software-gl-layers', default=True): bool,
+
+    # The worker implementation for this test, as dictated by policy and by the
+    # test platform.
+    Optional('worker-implementation'): Any(
+        'docker-worker',
+        'macosx-engine',
+        'generic-worker',
+        # coming soon:
+        'docker-engine',
+        'buildbot-bridge',
+    ),
+
+    # For tasks that will run in docker-worker or docker-engine, this is the
+    # name of the docker image or in-tree docker image to run the task in.  If
+    # in-tree, then a dependency will be created automatically.  This is
+    # generally `desktop-test`, or an image that acts an awful lot like it.
+    Required('docker-image', default={'in-tree': 'desktop-test'}): Any(
+        # a raw Docker image path (repo/image:tag)
+        basestring,
+        # an in-tree generated docker image (from `testing/docker/<name>`)
+        {'in-tree': basestring}
+    ),
+
+    # seconds of runtime after which the task will be killed.  Like 'chunks',
+    # this can be keyed by test pltaform.
+    Required('max-run-time', default=3600): Any(
+        int,
+        {'by-test-platform': {basestring: int}},
+    ),
+
+    # the exit status code that indicates the task should be retried
+    Optional('retry-exit-status'): int,
+
+    # Whether to perform a gecko checkout.
+    Required('checkout', default=False): bool,
+
+    # What to run
+    Required('mozharness'): Any({
+        # the mozharness script used to run this task
+        Required('script'): basestring,
+
+        # the config files required for the task
+        Required('config'): Any(
+            [basestring],
+            {'by-test-platform': {basestring: [basestring]}},
+        ),
+
+        # any additional actions to pass to the mozharness command
+        Optional('actions'): [basestring],
+
+        # additional command-line options for mozharness, beyond those
+        # automatically added
+        Required('extra-options', default=[]): Any(
+            [basestring],
+            {'by-test-platform': {basestring: [basestring]}},
+        ),
+
+        # the artifact name (including path) to test on the build task; this is
+        # generally set in a per-kind transformation
+        Optional('build-artifact-name'): basestring,
+
+        # If true, tooltool downloads will be enabled via relengAPIProxy.
+        Required('tooltool-downloads', default=False): bool,
+
+        # This mozharness script also runs in Buildbot and tries to read a
+        # buildbot config file, so tell it not to do so in TaskCluster
+        Required('no-read-buildbot-config', default=False): bool,
+
+        # Add --blob-upload-branch=<project> mozharness parameter
+        Optional('include-blob-upload-branch'): bool,
+
+        # The setting for --download-symbols (if omitted, the option will not
+        # be passed to mozharness)
+        Optional('download-symbols'): Any(True, 'ondemand'),
+
+        # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the
+        # environment.  This is more than just a helpful path setting -- it
+        # causes xpcshell tests to start additional servers, and runs
+        # additional tests.
+        Required('set-moz-node-path', default=False): bool,
+
+        # If true, include chunking information in the command even if the number
+        # of chunks is 1
+        Required('chunked', default=False): bool,
+
+        # The chunking argument format to use
+        Required('chunking-args', default='this-chunk'): Any(
+            # Use the usual --this-chunk/--total-chunk arguments
+            'this-chunk',
+            # Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below
+            'test-suite-suffix',
+        ),
+
+        # the string to append to the `--test-suite` arugment when
+        # chunking-args = test-suite-suffix; "<CHUNK>" in this string will
+        # be replaced with the chunk number.
+        Optional('chunk-suffix'): basestring,
+    }),
+
+    # The current chunk; this is filled in by `all_kinds.py`
+    Optional('this-chunk'): int,
+
+    # os user groups for test task workers; required scopes, will be
+    # added automatically
+    Optional('os-groups', default=[]): Any(
+        [basestring],
+        # todo: create a dedicated elevated worker group and name here
+        {'by-test-platform': {basestring: [basestring]}},
+    ),
+
+    # -- values supplied by the task-generation infrastructure
+
+    # the platform of the build this task is testing
+    'build-platform': basestring,
+
+    # the label of the build task generating the materials to test
+    'build-label': basestring,
+
+    # the platform on which the tests will run
+    'test-platform': basestring,
+
+    # the name of the test (the key in tests.yml)
+    'test-name': basestring,
+
+}, required=True)
+
+
+# TODO: can we have validate and validate_full for before and after?
+def validate(config, tests):
+    for test in tests:
+        yield validate_schema(test_description_schema, test,
+                              "In test {!r}:".format(test['test-name']))
diff --git a/taskcluster/taskgraph/transforms/upload_symbols.py b/taskcluster/taskgraph/transforms/upload_symbols.py
new file mode 100644
index 000000000..9b4884a97
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/upload_symbols.py
@@ -0,0 +1,36 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the upload-symbols task description template,
+  taskcluster/ci/upload-symbols/job-template.yml
+into an actual task description.
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+
+
+transforms = TransformSequence()
+
+
+@transforms.add
+def fill_template(config, tasks):
+    for task in tasks:
+        # Fill out the dynamic fields in the task description
+        task['label'] = task['build-label'] + '-upload-symbols'
+        task['dependencies'] = {'build': task['build-label']}
+        task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params['head_repository']
+        task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev']
+
+        build_platform, build_type = task['build-platform'].split('/')
+        attributes = task.setdefault('attributes', {})
+        attributes['build_platform'] = build_platform
+        attributes['build_type'] = build_type
+
+        # clear out the stuff that's not part of a task description
+        del task['build-label']
+        del task['build-platform']
+
+        yield task
diff --git a/taskcluster/taskgraph/try_option_syntax.py b/taskcluster/taskgraph/try_option_syntax.py
new file mode 100644
index 000000000..b5988db98
--- /dev/null
+++ b/taskcluster/taskgraph/try_option_syntax.py
@@ -0,0 +1,559 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import copy
+import logging
+import re
+import shlex
+
+logger = logging.getLogger(__name__)
+
+TRY_DELIMITER = 'try:'
+
+# The build type aliases are very cryptic and only used in try flags these are
+# mappings from the single char alias to a longer more recognizable form.
+BUILD_TYPE_ALIASES = {
+    'o': 'opt',
+    'd': 'debug'
+}
+
+# consider anything in this whitelist of kinds to be governed by -b/-p
+BUILD_KINDS = set([
+    'build',
+    'artifact-build',
+    'hazard',
+    'l10n',
+    'upload-symbols',
+    'valgrind',
+    'static-analysis',
+    'spidermonkey',
+])
+
+# anything in this list is governed by -j
+JOB_KINDS = set([
+    'source-check',
+    'toolchain',
+    'marionette-harness',
+    'android-stuff',
+])
+
+
+# mapping from shortcut name (usable with -u) to a boolean function identifying
+# matching test names
+def alias_prefix(prefix):
+    return lambda name: name.startswith(prefix)
+
+
+def alias_contains(infix):
+    return lambda name: infix in name
+
+
+def alias_matches(pattern):
+    pattern = re.compile(pattern)
+    return lambda name: pattern.match(name)
+
+UNITTEST_ALIASES = {
+    # Aliases specify shorthands that can be used in try syntax.  The shorthand
+    # is the dictionary key, with the value representing a pattern for matching
+    # unittest_try_names.
+    #
+    # Note that alias expansion is performed in the absence of any chunk
+    # prefixes.  For example, the first example above would replace "foo-7"
+    # with "foobar-7".  Note that a few aliases allowed chunks to be specified
+    # without a leading `-`, for example 'mochitest-dt1'. That's no longer
+    # supported.
+    'cppunit': alias_prefix('cppunit'),
+    'crashtest': alias_prefix('crashtest'),
+    'crashtest-e10s': alias_prefix('crashtest-e10s'),
+    'e10s': alias_contains('e10s'),
+    'external-media-tests': alias_prefix('external-media-tests'),
+    'firefox-ui-functional': alias_prefix('firefox-ui-functional'),
+    'firefox-ui-functional-e10s': alias_prefix('firefox-ui-functional-e10s'),
+    'gaia-js-integration': alias_contains('gaia-js-integration'),
+    'gtest': alias_prefix('gtest'),
+    'jittest': alias_prefix('jittest'),
+    'jittests': alias_prefix('jittest'),
+    'jsreftest': alias_prefix('jsreftest'),
+    'jsreftest-e10s': alias_prefix('jsreftest-e10s'),
+    'marionette': alias_prefix('marionette'),
+    'marionette-e10s': alias_prefix('marionette-e10s'),
+    'mochitest': alias_prefix('mochitest'),
+    'mochitests': alias_prefix('mochitest'),
+    'mochitest-e10s': alias_prefix('mochitest-e10s'),
+    'mochitests-e10s': alias_prefix('mochitest-e10s'),
+    'mochitest-debug': alias_prefix('mochitest-debug-'),
+    'mochitest-a11y': alias_contains('mochitest-a11y'),
+    'mochitest-bc': alias_prefix('mochitest-browser-chrome'),
+    'mochitest-e10s-bc': alias_prefix('mochitest-e10s-browser-chrome'),
+    'mochitest-browser-chrome': alias_prefix('mochitest-browser-chrome'),
+    'mochitest-e10s-browser-chrome': alias_prefix('mochitest-e10s-browser-chrome'),
+    'mochitest-chrome': alias_contains('mochitest-chrome'),
+    'mochitest-dt': alias_prefix('mochitest-devtools-chrome'),
+    'mochitest-e10s-dt': alias_prefix('mochitest-e10s-devtools-chrome'),
+    'mochitest-gl': alias_prefix('mochitest-webgl'),
+    'mochitest-gl-e10s': alias_prefix('mochitest-webgl-e10s'),
+    'mochitest-gpu': alias_prefix('mochitest-gpu'),
+    'mochitest-gpu-e10s': alias_prefix('mochitest-gpu-e10s'),
+    'mochitest-clipboard': alias_prefix('mochitest-clipboard'),
+    'mochitest-clipboard-e10s': alias_prefix('mochitest-clipboard-e10s'),
+    'mochitest-jetpack': alias_prefix('mochitest-jetpack'),
+    'mochitest-media': alias_prefix('mochitest-media'),
+    'mochitest-media-e10s': alias_prefix('mochitest-media-e10s'),
+    'mochitest-vg': alias_prefix('mochitest-valgrind'),
+    'reftest': alias_matches(r'^(plain-)?reftest.*$'),
+    'reftest-no-accel': alias_matches(r'^(plain-)?reftest-no-accel.*$'),
+    'reftests': alias_matches(r'^(plain-)?reftest.*$'),
+    'reftests-e10s': alias_matches(r'^(plain-)?reftest-e10s.*$'),
+    'robocop': alias_prefix('robocop'),
+    'web-platform-test': alias_prefix('web-platform-tests'),
+    'web-platform-tests': alias_prefix('web-platform-tests'),
+    'web-platform-tests-e10s': alias_prefix('web-platform-tests-e10s'),
+    'web-platform-tests-reftests': alias_prefix('web-platform-tests-reftests'),
+    'web-platform-tests-reftests-e10s': alias_prefix('web-platform-tests-reftests-e10s'),
+    'xpcshell': alias_prefix('xpcshell'),
+}
+
+# unittest platforms can be specified by substring of the "pretty name", which
+# is basically the old Buildbot builder name.  This dict has {pretty name,
+# [test_platforms]} translations, This includes only the most commonly-used
+# substrings.  This is intended only for backward-compatibility.  New test
+# platforms should have their `test_platform` spelled out fully in try syntax.
+UNITTEST_PLATFORM_PRETTY_NAMES = {
+    'Ubuntu': ['linux', 'linux64', 'linux64-asan'],
+    'x64': ['linux64', 'linux64-asan'],
+    'Android 4.3': ['android-4.3-arm7-api-15'],
+    # other commonly-used substrings for platforms not yet supported with
+    # in-tree taskgraphs:
+    # '10.10': [..TODO..],
+    # '10.10.5': [..TODO..],
+    # '10.6': [..TODO..],
+    # '10.8': [..TODO..],
+    # 'Android 2.3 API9': [..TODO..],
+    # 'Windows 7':  [..TODO..],
+    # 'Windows 7 VM': [..TODO..],
+    # 'Windows 8':  [..TODO..],
+    # 'Windows XP': [..TODO..],
+    # 'win32': [..TODO..],
+    # 'win64': [..TODO..],
+}
+
+# We have a few platforms for which we want to do some "extra" builds, or at
+# least build-ish things.  Sort of.  Anyway, these other things are implemented
+# as different "platforms".  These do *not* automatically ride along with "-p
+# all"
+RIDEALONG_BUILDS = {
+    'android-api-15': [
+        'android-api-15-l10n',
+    ],
+    'linux': [
+        'linux-l10n',
+    ],
+    'linux64': [
+        'linux64-l10n',
+        'sm-plain',
+        'sm-nonunified',
+        'sm-arm-sim',
+        'sm-arm64-sim',
+        'sm-compacting',
+        'sm-rootanalysis',
+        'sm-package',
+        'sm-tsan',
+        'sm-asan',
+        'sm-mozjs-sys',
+        'sm-msan',
+    ],
+}
+
+TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$')
+
+
+class TryOptionSyntax(object):
+
+    def __init__(self, message, full_task_graph):
+        """
+        Parse a "try syntax" formatted commit message.  This is the old "-b do -p
+        win32 -u all" format.  Aliases are applied to map short names to full
+        names.
+
+        The resulting object has attributes:
+
+        - build_types: a list containing zero or more of 'opt' and 'debug'
+        - platforms: a list of selected platform names, or None for all
+        - unittests: a list of tests, of the form given below, or None for all
+        - jobs: a list of requested job names, or None for all
+        - trigger_tests: the number of times tests should be triggered (--rebuild)
+        - interactive: true if --interactive
+        - notifications: either None if no notifications or one of 'all' or 'failure'
+
+        Note that -t is currently completely ignored.
+
+        The unittests and talos lists contain dictionaries of the form:
+
+        {
+            'test': '<suite name>',
+            'platforms': [..platform names..], # to limit to only certain platforms
+            'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks
+        }
+        """
+        self.jobs = []
+        self.build_types = []
+        self.platforms = []
+        self.unittests = []
+        self.talos = []
+        self.trigger_tests = 0
+        self.interactive = False
+        self.notifications = None
+
+        # shlex used to ensure we split correctly when giving values to argparse.
+        parts = shlex.split(self.escape_whitespace_in_brackets(message))
+        try_idx = None
+        for idx, part in enumerate(parts):
+            if part == TRY_DELIMITER:
+                try_idx = idx
+                break
+
+        if try_idx is None:
+            return
+
+        # Argument parser based on try flag flags
+        parser = argparse.ArgumentParser()
+        parser.add_argument('-b', '--build', dest='build_types')
+        parser.add_argument('-p', '--platform', nargs='?',
+                            dest='platforms', const='all', default='all')
+        parser.add_argument('-u', '--unittests', nargs='?',
+                            dest='unittests', const='all', default='all')
+        parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='all')
+        parser.add_argument('-i', '--interactive',
+                            dest='interactive', action='store_true', default=False)
+        parser.add_argument('-e', '--all-emails',
+                            dest='notifications', action='store_const', const='all')
+        parser.add_argument('-f', '--failure-emails',
+                            dest='notifications', action='store_const', const='failure')
+        parser.add_argument('-j', '--job', dest='jobs', action='append')
+        # In order to run test jobs multiple times
+        parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1)
+        args, _ = parser.parse_known_args(parts[try_idx:])
+
+        self.jobs = self.parse_jobs(args.jobs)
+        self.build_types = self.parse_build_types(args.build_types)
+        self.platforms = self.parse_platforms(args.platforms)
+        self.unittests = self.parse_test_option(
+            "unittest_try_name", args.unittests, full_task_graph)
+        self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph)
+        self.trigger_tests = args.trigger_tests
+        self.interactive = args.interactive
+        self.notifications = args.notifications
+
+    def parse_jobs(self, jobs_arg):
+        if not jobs_arg or jobs_arg == ['all']:
+            return None
+        expanded = []
+        for job in jobs_arg:
+            expanded.extend(j.strip() for j in job.split(','))
+        return expanded
+
+    def parse_build_types(self, build_types_arg):
+        if build_types_arg is None:
+            build_types_arg = []
+        build_types = filter(None, [BUILD_TYPE_ALIASES.get(build_type) for
+                             build_type in build_types_arg])
+        return build_types
+
+    def parse_platforms(self, platform_arg):
+        if platform_arg == 'all':
+            return None
+
+        results = []
+        for build in platform_arg.split(','):
+            results.append(build)
+            if build in RIDEALONG_BUILDS:
+                results.extend(RIDEALONG_BUILDS[build])
+                logger.info("platform %s triggers ridealong builds %s" %
+                            (build, ', '.join(RIDEALONG_BUILDS[build])))
+
+        return results
+
+    def parse_test_option(self, attr_name, test_arg, full_task_graph):
+        '''
+
+        Parse a unittest (-u) or talos (-t) option, in the context of a full
+        task graph containing available `unittest_try_name` or `talos_try_name`
+        attributes.  There are three cases:
+
+            - test_arg is == 'none' (meaning an empty list)
+            - test_arg is == 'all' (meaning use the list of jobs for that job type)
+            - test_arg is comma string which needs to be parsed
+        '''
+
+        # Empty job list case...
+        if test_arg is None or test_arg == 'none':
+            return []
+
+        all_platforms = set(t.attributes['test_platform']
+                            for t in full_task_graph.tasks.itervalues()
+                            if 'test_platform' in t.attributes)
+
+        tests = self.parse_test_opts(test_arg, all_platforms)
+
+        if not tests:
+            return []
+
+        all_tests = set(t.attributes[attr_name]
+                        for t in full_task_graph.tasks.itervalues()
+                        if attr_name in t.attributes)
+
+        # Special case where tests is 'all' and must be expanded
+        if tests[0]['test'] == 'all':
+            results = []
+            all_entry = tests[0]
+            for test in all_tests:
+                entry = {'test': test}
+                # If there are platform restrictions copy them across the list.
+                if 'platforms' in all_entry:
+                    entry['platforms'] = list(all_entry['platforms'])
+                results.append(entry)
+            return self.parse_test_chunks(all_tests, results)
+        else:
+            return self.parse_test_chunks(all_tests, tests)
+
+    def parse_test_opts(self, input_str, all_platforms):
+        '''
+        Parse `testspec,testspec,..`, where each testspec is a test name
+        optionally followed by a list of test platforms or negated platforms in
+        `[]`.
+
+        No brackets indicates that tests should run on all platforms for which
+        builds are available.  If testspecs are provided, then each is treated,
+        from left to right, as an instruction to include or (if negated)
+        exclude a set of test platforms.  A single spec may expand to multiple
+        test platforms via UNITTEST_PLATFORM_PRETTY_NAMES.  If the first test
+        spec is negated, processing begins with the full set of available test
+        platforms; otherwise, processing begins with an empty set of test
+        platforms.
+        '''
+
+        # Final results which we will return.
+        tests = []
+
+        cur_test = {}
+        token = ''
+        in_platforms = False
+
+        def normalize_platforms():
+            if 'platforms' not in cur_test:
+                return
+            # if the first spec is a negation, start with all platforms
+            if cur_test['platforms'][0][0] == '-':
+                platforms = all_platforms.copy()
+            else:
+                platforms = []
+            for platform in cur_test['platforms']:
+                if platform[0] == '-':
+                    platforms = [p for p in platforms if p != platform[1:]]
+                else:
+                    platforms.append(platform)
+            cur_test['platforms'] = platforms
+
+        def add_test(value):
+            normalize_platforms()
+            cur_test['test'] = value.strip()
+            tests.insert(0, cur_test)
+
+        def add_platform(value):
+            platform = value.strip()
+            if platform[0] == '-':
+                negated = True
+                platform = platform[1:]
+            else:
+                negated = False
+            platforms = UNITTEST_PLATFORM_PRETTY_NAMES.get(platform, [platform])
+            if negated:
+                platforms = ["-" + p for p in platforms]
+            cur_test['platforms'] = platforms + cur_test.get('platforms', [])
+
+        # This might be somewhat confusing but we parse the string _backwards_ so
+        # there is no ambiguity over what state we are in.
+        for char in reversed(input_str):
+
+            # , indicates exiting a state
+            if char == ',':
+
+                # Exit a particular platform.
+                if in_platforms:
+                    add_platform(token)
+
+                # Exit a particular test.
+                else:
+                    add_test(token)
+                    cur_test = {}
+
+                # Token must always be reset after we exit a state
+                token = ''
+            elif char == '[':
+                # Exiting platform state entering test state.
+                add_platform(token)
+                token = ''
+                in_platforms = False
+            elif char == ']':
+                # Entering platform state.
+                in_platforms = True
+            else:
+                # Accumulator.
+                token = char + token
+
+        # Handle any left over tokens.
+        if token:
+            add_test(token)
+
+        return tests
+
+    def handle_alias(self, test, all_tests):
+        '''
+        Expand a test if its name refers to an alias, returning a list of test
+        dictionaries cloned from the first (to maintain any metadata).
+        '''
+        if test['test'] not in UNITTEST_ALIASES:
+            return [test]
+
+        alias = UNITTEST_ALIASES[test['test']]
+
+        def mktest(name):
+            newtest = copy.deepcopy(test)
+            newtest['test'] = name
+            return newtest
+
+        def exprmatch(alias):
+            return [t for t in all_tests if alias(t)]
+
+        return [mktest(t) for t in exprmatch(alias)]
+
+    def parse_test_chunks(self, all_tests, tests):
+        '''
+        Test flags may include parameters to narrow down the number of chunks in a
+        given push. We don't model 1 chunk = 1 job in taskcluster so we must check
+        each test flag to see if it is actually specifying a chunk.
+        '''
+        results = []
+        seen_chunks = {}
+        for test in tests:
+            matches = TEST_CHUNK_SUFFIX.match(test['test'])
+            if matches:
+                name = matches.group(1)
+                chunk = matches.group(2)
+                if name in seen_chunks:
+                    seen_chunks[name].add(chunk)
+                else:
+                    seen_chunks[name] = {chunk}
+                    test['test'] = name
+                    test['only_chunks'] = seen_chunks[name]
+                    results.append(test)
+            else:
+                results.extend(self.handle_alias(test, all_tests))
+
+        # uniquify the results over the test names
+        results = {test['test']: test for test in results}.values()
+        return results
+
+    def find_all_attribute_suffixes(self, graph, prefix):
+        rv = set()
+        for t in graph.tasks.itervalues():
+            for a in t.attributes:
+                if a.startswith(prefix):
+                    rv.add(a[len(prefix):])
+        return sorted(rv)
+
+    def escape_whitespace_in_brackets(self, input_str):
+        '''
+        In tests you may restrict them by platform [] inside of the brackets
+        whitespace may occur this is typically invalid shell syntax so we escape it
+        with backslash sequences    .
+        '''
+        result = ""
+        in_brackets = False
+        for char in input_str:
+            if char == '[':
+                in_brackets = True
+                result += char
+                continue
+
+            if char == ']':
+                in_brackets = False
+                result += char
+                continue
+
+            if char == ' ' and in_brackets:
+                result += '\ '
+                continue
+
+            result += char
+
+        return result
+
+    def task_matches(self, attributes):
+        attr = attributes.get
+
+        def check_run_on_projects():
+            return set(['try', 'all']) & set(attr('run_on_projects', []))
+
+        def match_test(try_spec, attr_name):
+            if attr('build_type') not in self.build_types:
+                return False
+            if self.platforms is not None:
+                if attr('build_platform') not in self.platforms:
+                    return False
+            else:
+                if not check_run_on_projects():
+                    return False
+            if try_spec is None:
+                return True
+            # TODO: optimize this search a bit
+            for test in try_spec:
+                if attr(attr_name) == test['test']:
+                    break
+            else:
+                return False
+            if 'platforms' in test and attr('test_platform') not in test['platforms']:
+                return False
+            if 'only_chunks' in test and attr('test_chunk') not in test['only_chunks']:
+                return False
+            return True
+
+        if attr('kind') in ('desktop-test', 'android-test'):
+            return match_test(self.unittests, 'unittest_try_name')
+        elif attr('kind') in JOB_KINDS:
+            if self.jobs is None:
+                return True
+            if attr('build_platform') in self.jobs:
+                return True
+        elif attr('kind') in BUILD_KINDS:
+            if attr('build_type') not in self.build_types:
+                return False
+            elif self.platforms is None:
+                # for "-p all", look for try in the 'run_on_projects' attribute
+                return check_run_on_projects()
+            else:
+                if attr('build_platform') not in self.platforms:
+                    return False
+            return True
+        else:
+            return False
+
+    def __str__(self):
+        def none_for_all(list):
+            if list is None:
+                return '<all>'
+            return ', '.join(str(e) for e in list)
+
+        return "\n".join([
+            "build_types: " + ", ".join(self.build_types),
+            "platforms: " + none_for_all(self.platforms),
+            "unittests: " + none_for_all(self.unittests),
+            "jobs: " + none_for_all(self.jobs),
+            "trigger_tests: " + str(self.trigger_tests),
+            "interactive: " + str(self.interactive),
+            "notifications: " + self.notifications,
+        ])
diff --git a/taskcluster/taskgraph/util/__init__.py b/taskcluster/taskgraph/util/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/taskcluster/taskgraph/util/__init__.py
diff --git a/taskcluster/taskgraph/util/attributes.py b/taskcluster/taskgraph/util/attributes.py
new file mode 100644
index 000000000..b44a3364f
--- /dev/null
+++ b/taskcluster/taskgraph/util/attributes.py
@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def attrmatch(attributes, **kwargs):
+    """Determine whether the given set of task attributes matches.  The
+    conditions are given as keyword arguments, where each keyword names an
+    attribute.  The keyword value can be a literal, a set, or a callable.  A
+    literal must match the attribute exactly.  Given a set, the attribute value
+    must be in the set.  A callable is called with the attribute value.  If an
+    attribute is specified as a keyword argument but not present in the
+    attributes, the result is False."""
+    for kwkey, kwval in kwargs.iteritems():
+        if kwkey not in attributes:
+            return False
+        attval = attributes[kwkey]
+        if isinstance(kwval, set):
+            if attval not in kwval:
+                return False
+        elif callable(kwval):
+            if not kwval(attval):
+                return False
+        elif kwval != attributes[kwkey]:
+            return False
+    return True
diff --git a/taskcluster/taskgraph/util/docker.py b/taskcluster/taskgraph/util/docker.py
new file mode 100644
index 000000000..df97e57bc
--- /dev/null
+++ b/taskcluster/taskgraph/util/docker.py
@@ -0,0 +1,160 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import hashlib
+import os
+import shutil
+import subprocess
+import tarfile
+import tempfile
+
+from mozpack.archive import (
+    create_tar_gz_from_files,
+)
+
+
+GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
+DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker')
+INDEX_PREFIX = 'docker.images.v2'
+ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
+
+
+def docker_image(name, default_version=None):
+    '''Determine the docker image name, including repository and tag, from an
+    in-tree docker file.'''
+    try:
+        with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f:
+            registry = f.read().strip()
+    except IOError:
+        with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f:
+            registry = f.read().strip()
+
+    try:
+        with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f:
+            version = f.read().strip()
+    except IOError:
+        if not default_version:
+            raise
+
+        version = default_version
+
+    return '{}/{}:{}'.format(registry, name, version)
+
+
+def generate_context_hash(topsrcdir, image_path, image_name):
+    """Generates a sha256 hash for context directory used to build an image."""
+
+    # It is a bit unfortunate we have to create a temp file here - it would
+    # be nicer to use an in-memory buffer.
+    fd, p = tempfile.mkstemp()
+    os.close(fd)
+    try:
+        return create_context_tar(topsrcdir, image_path, p, image_name)
+    finally:
+        os.unlink(p)
+
+
+def create_context_tar(topsrcdir, context_dir, out_path, prefix):
+    """Create a context tarball.
+
+    A directory ``context_dir`` containing a Dockerfile will be assembled into
+    a gzipped tar file at ``out_path``. Files inside the archive will be
+    prefixed by directory ``prefix``.
+
+    We also scan the source Dockerfile for special syntax that influences
+    context generation.
+
+    If a line in the Dockerfile has the form ``# %include <path>``,
+    the relative path specified on that line will be matched against
+    files in the source repository and added to the context under the
+    path ``topsrcdir/``. If an entry is a directory, we add all files
+    under that directory.
+
+    Returns the SHA-256 hex digest of the created archive.
+    """
+    archive_files = {}
+
+    for root, dirs, files in os.walk(context_dir):
+        for f in files:
+            source_path = os.path.join(root, f)
+            rel = source_path[len(context_dir) + 1:]
+            archive_path = os.path.join(prefix, rel)
+            archive_files[archive_path] = source_path
+
+    # Parse Dockerfile for special syntax of extra files to include.
+    with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh:
+        for line in fh:
+            line = line.rstrip()
+            if not line.startswith('# %include'):
+                continue
+
+            p = line[len('# %include '):].strip()
+            if os.path.isabs(p):
+                raise Exception('extra include path cannot be absolute: %s' % p)
+
+            fs_path = os.path.normpath(os.path.join(topsrcdir, p))
+            # Check for filesystem traversal exploits.
+            if not fs_path.startswith(topsrcdir):
+                raise Exception('extra include path outside topsrcdir: %s' % p)
+
+            if not os.path.exists(fs_path):
+                raise Exception('extra include path does not exist: %s' % p)
+
+            if os.path.isdir(fs_path):
+                for root, dirs, files in os.walk(fs_path):
+                    for f in files:
+                        source_path = os.path.join(root, f)
+                        archive_path = os.path.join(prefix, 'topsrcdir', p, f)
+                        archive_files[archive_path] = source_path
+            else:
+                archive_path = os.path.join(prefix, 'topsrcdir', p)
+                archive_files[archive_path] = fs_path
+
+    with open(out_path, 'wb') as fh:
+        create_tar_gz_from_files(fh, archive_files, '%s.tar.gz' % prefix)
+
+    h = hashlib.sha256()
+    with open(out_path, 'rb') as fh:
+        while True:
+            data = fh.read(32768)
+            if not data:
+                break
+            h.update(data)
+    return h.hexdigest()
+
+
+def build_from_context(docker_bin, context_path, prefix, tag=None):
+    """Build a Docker image from a context archive.
+
+    Given the path to a `docker` binary, a image build tar.gz (produced with
+    ``create_context_tar()``, a prefix in that context containing files, and
+    an optional ``tag`` for the produced image, build that Docker image.
+    """
+    d = tempfile.mkdtemp()
+    try:
+        with tarfile.open(context_path, 'r:gz') as tf:
+            tf.extractall(d)
+
+        # If we wanted to do post-processing of the Dockerfile, this is
+        # where we'd do it.
+
+        args = [
+            docker_bin,
+            'build',
+            # Use --no-cache so we always get the latest package updates.
+            '--no-cache',
+        ]
+
+        if tag:
+            args.extend(['-t', tag])
+
+        args.append('.')
+
+        res = subprocess.call(args, cwd=os.path.join(d, prefix))
+        if res:
+            raise Exception('error building image')
+    finally:
+        shutil.rmtree(d)
diff --git a/taskcluster/taskgraph/util/python_path.py b/taskcluster/taskgraph/util/python_path.py
new file mode 100644
index 000000000..b14223ca6
--- /dev/null
+++ b/taskcluster/taskgraph/util/python_path.py
@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+
+def find_object(path):
+    """
+    Find a Python object given a path of the form <modulepath>:<objectpath>.
+    Conceptually equivalent to
+
+        def find_object(modulepath, objectpath):
+            import <modulepath> as mod
+            return mod.<objectpath>
+    """
+    if path.count(':') != 1:
+        raise ValueError(
+            'python path {!r} does not have the form "module:object"'.format(path))
+
+    modulepath, objectpath = path.split(':')
+    obj = __import__(modulepath)
+    for a in modulepath.split('.')[1:]:
+        obj = getattr(obj, a)
+    for a in objectpath.split('.'):
+        obj = getattr(obj, a)
+    return obj
diff --git a/taskcluster/taskgraph/util/seta.py b/taskcluster/taskgraph/util/seta.py
new file mode 100644
index 000000000..a0cd30675
--- /dev/null
+++ b/taskcluster/taskgraph/util/seta.py
@@ -0,0 +1,85 @@
+import json
+import logging
+import requests
+from redo import retry
+from requests import exceptions
+
+logger = logging.getLogger(__name__)
+headers = {
+    'User-Agent': 'TaskCluster'
+}
+
+# It's a list of project name which SETA is useful on
+SETA_PROJECTS = ['mozilla-inbound', 'autoland']
+SETA_ENDPOINT = "https://seta.herokuapp.com/data/setadetails/?branch=%s"
+
+
+class SETA(object):
+    """
+    Interface to the SETA service, which defines low-value tasks that can be optimized out
+    of the taskgraph.
+    """
+    def __init__(self):
+        # cached low value tasks, by project
+        self.low_value_tasks = {}
+
+    def query_low_value_tasks(self, project):
+        # Request the set of low value tasks from the SETA service.  Low value tasks will be
+        # optimized out of the task graph.
+        if project not in SETA_PROJECTS:
+            logger.debug("SETA is not enabled for project `{}`".format(project))
+            return []
+
+        logger.debug("Querying SETA service for low-value tasks on {}".format(project))
+        low_value_tasks = []
+
+        url = SETA_ENDPOINT % project
+        # Try to fetch the SETA data twice, falling back to an empty list of low value tasks.
+        # There are 10 seconds between each try.
+        try:
+            logger.debug("Retrieving low-value jobs list from SETA")
+            response = retry(requests.get, attempts=2, sleeptime=10,
+                             args=(url, ),
+                             kwargs={'timeout': 5, 'headers': headers})
+            task_list = json.loads(response.content).get('jobtypes', '')
+            if len(task_list) > 0:
+                low_value_tasks = task_list.values()[0]
+
+            # Bug 1315145, disable SETA for tier-1 platforms until backfill is implemented.
+            low_value_tasks = [x for x in low_value_tasks if x.find('debug') == -1]
+            low_value_tasks = [x for x in low_value_tasks if x.find('asan') == -1]
+
+        # In the event of request times out, requests will raise a TimeoutError.
+        except exceptions.Timeout:
+            logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
+
+        # In the event of a network problem (e.g. DNS failure, refused connection, etc),
+        # requests will raise a ConnectionError.
+        except exceptions.ConnectionError:
+            logger.warning("SETA server is timeout, we will treat all test tasks as high value.")
+
+        # In the event of the rare invalid HTTP response(e.g 404, 401),
+        # requests will raise an HTTPError exception
+        except exceptions.HTTPError:
+            logger.warning("We got bad Http response from ouija,"
+                           " we will treat all test tasks as high value.")
+
+        # We just print the error out as a debug message if we failed to catch the exception above
+        except exceptions.RequestException as error:
+            logger.warning(error)
+
+        # When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426)
+        except ValueError as error:
+            logger.warning("Invalid JSON, possible server error: {}".format(error))
+
+        return low_value_tasks
+
+    def is_low_value_task(self, label, project):
+        # cache the low value tasks per project to avoid repeated SETA server queries
+        if project not in self.low_value_tasks:
+            self.low_value_tasks[project] = self.query_low_value_tasks(project)
+        return label in self.low_value_tasks[project]
+
+# create a single instance of this class, and expose its `is_low_value_task`
+# bound method as a module-level function
+is_low_value_task = SETA().is_low_value_task
diff --git a/taskcluster/taskgraph/util/templates.py b/taskcluster/taskgraph/util/templates.py
new file mode 100644
index 000000000..97620fa75
--- /dev/null
+++ b/taskcluster/taskgraph/util/templates.py
@@ -0,0 +1,155 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+
+import pystache
+import yaml
+import copy
+
+# Key used in template inheritance...
+INHERITS_KEY = '$inherits'
+
+
+def merge_to(source, dest):
+    '''
+    Merge dict and arrays (override scalar values)
+
+    Keys from source override keys from dest, and elements from lists in source
+    are appended to lists in dest.
+
+    :param dict source: to copy from
+    :param dict dest: to copy to (modified in place)
+    '''
+
+    for key, value in source.items():
+        # Override mismatching or empty types
+        if type(value) != type(dest.get(key)):  # noqa
+            dest[key] = source[key]
+            continue
+
+        # Merge dict
+        if isinstance(value, dict):
+            merge_to(value, dest[key])
+            continue
+
+        if isinstance(value, list):
+            dest[key] = dest[key] + source[key]
+            continue
+
+        dest[key] = source[key]
+
+    return dest
+
+
+def merge(*objects):
+    '''
+    Merge the given objects, using the semantics described for merge_to, with
+    objects later in the list taking precedence.  From an inheritance
+    perspective, "parents" should be listed before "children".
+
+    Returns the result without modifying any arguments.
+    '''
+    if len(objects) == 1:
+        return copy.deepcopy(objects[0])
+    return merge_to(objects[-1], merge(*objects[:-1]))
+
+
+class TemplatesException(Exception):
+    pass
+
+
+class Templates():
+    '''
+    The taskcluster integration makes heavy use of yaml to describe tasks this
+    class handles the loading/rendering.
+    '''
+
+    def __init__(self, root):
+        '''
+        Initialize the template render.
+
+        :param str root: Root path where to load yaml files.
+        '''
+        if not root:
+            raise TemplatesException('Root is required')
+
+        if not os.path.isdir(root):
+            raise TemplatesException('Root must be a directory')
+
+        self.root = root
+
+    def _inherits(self, path, obj, properties, seen):
+        blueprint = obj.pop(INHERITS_KEY)
+        seen.add(path)
+
+        # Resolve the path here so we can detect circular references.
+        template = self.resolve_path(blueprint.get('from'))
+        variables = blueprint.get('variables', {})
+
+        # Passed parameters override anything in the task itself.
+        for key in properties:
+            variables[key] = properties[key]
+
+        if not template:
+            msg = '"{}" inheritance template missing'.format(path)
+            raise TemplatesException(msg)
+
+        if template in seen:
+            msg = 'Error while handling "{}" in "{}" circular template' + \
+                  'inheritance seen \n  {}'
+            raise TemplatesException(msg.format(path, template, seen))
+
+        try:
+            out = self.load(template, variables, seen)
+        except TemplatesException as e:
+            msg = 'Error expanding parent ("{}") of "{}" original error {}'
+            raise TemplatesException(msg.format(template, path, str(e)))
+
+        # Anything left in obj is merged into final results (and overrides)
+        return merge_to(obj, out)
+
+    def render(self, path, content, parameters, seen):
+        '''
+        Renders a given yaml string.
+
+        :param str path:  used to prevent infinite recursion in inheritance.
+        :param str content: Of yaml file.
+        :param dict parameters: For mustache templates.
+        :param set seen: Seen files (used for inheritance)
+        '''
+        content = pystache.render(content, parameters)
+        result = yaml.load(content)
+
+        # In addition to the usual template logic done by mustache we also
+        # handle special '$inherit' dict keys.
+        if isinstance(result, dict) and INHERITS_KEY in result:
+            return self._inherits(path, result, parameters, seen)
+
+        return result
+
+    def resolve_path(self, path):
+        return os.path.join(self.root, path)
+
+    def load(self, path, parameters=None, seen=None):
+        '''
+        Load an render the given yaml path.
+
+        :param str path: Location of yaml file to load (relative to root).
+        :param dict parameters: To template yaml file with.
+        '''
+        seen = seen or set()
+
+        if not path:
+            raise TemplatesException('path is required')
+
+        path = self.resolve_path(path)
+
+        if not os.path.isfile(path):
+            raise TemplatesException('"{}" is not a file'.format(path))
+
+        content = open(path).read()
+        return self.render(path, content, parameters, seen)
diff --git a/taskcluster/taskgraph/util/time.py b/taskcluster/taskgraph/util/time.py
new file mode 100644
index 000000000..160aaa70c
--- /dev/null
+++ b/taskcluster/taskgraph/util/time.py
@@ -0,0 +1,114 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Python port of the ms.js node module this is not a direct port some things are
+# more complicated or less precise and we lean on time delta here.
+
+import re
+import datetime
+
+PATTERN = re.compile(
+    '((?:\d+)?\.?\d+) *([a-z]+)'
+)
+
+
+def seconds(value):
+    return datetime.timedelta(seconds=int(value))
+
+
+def minutes(value):
+    return datetime.timedelta(minutes=int(value))
+
+
+def hours(value):
+    return datetime.timedelta(hours=int(value))
+
+
+def days(value):
+    return datetime.timedelta(days=int(value))
+
+
+def months(value):
+    # See warning in years(), below
+    return datetime.timedelta(days=int(value) * 30)
+
+
+def years(value):
+    # Warning here "years" are vague don't use this for really sensitive date
+    # computation the idea is to give you a absolute amount of time in the
+    # future which is not the same thing as "precisely on this date next year"
+    return datetime.timedelta(days=int(value) * 365)
+
+ALIASES = {}
+ALIASES['seconds'] = ALIASES['second'] = ALIASES['s'] = seconds
+ALIASES['minutes'] = ALIASES['minute'] = ALIASES['min'] = minutes
+ALIASES['hours'] = ALIASES['hour'] = ALIASES['h'] = hours
+ALIASES['days'] = ALIASES['day'] = ALIASES['d'] = days
+ALIASES['months'] = ALIASES['month'] = ALIASES['mo'] = months
+ALIASES['years'] = ALIASES['year'] = ALIASES['y'] = years
+
+
+class InvalidString(Exception):
+    pass
+
+
+class UnknownTimeMeasurement(Exception):
+    pass
+
+
+def value_of(input_str):
+    '''
+    Convert a string to a json date in the future
+    :param str input_str: (ex: 1d, 2d, 6years, 2 seconds)
+    :returns: Unit given in seconds
+    '''
+
+    matches = PATTERN.search(input_str)
+
+    if matches is None or len(matches.groups()) < 2:
+        raise InvalidString("'{}' is invalid string".format(input_str))
+
+    value, unit = matches.groups()
+
+    if unit not in ALIASES:
+        raise UnknownTimeMeasurement(
+            '{} is not a valid time measure use one of {}'.format(
+                unit,
+                sorted(ALIASES.keys())
+            )
+        )
+
+    return ALIASES[unit](value)
+
+
+def json_time_from_now(input_str, now=None, datetime_format=False):
+    '''
+    :param str input_str: Input string (see value of)
+    :param datetime now: Optionally set the definition of `now`
+    :param boolean datetime_format: Set `True` to get a `datetime` output
+    :returns: JSON string representation of time in future.
+    '''
+
+    if now is None:
+        now = datetime.datetime.utcnow()
+
+    time = now + value_of(input_str)
+
+    if datetime_format is True:
+        return time
+    else:
+        # Sorta a big hack but the json schema validator for date does not like the
+        # ISO dates until 'Z' (for timezone) is added...
+        return time.isoformat() + 'Z'
+
+
+def current_json_time(datetime_format=False):
+    '''
+    :param boolean datetime_format: Set `True` to get a `datetime` output
+    :returns: JSON string representation of the current time.
+    '''
+    if datetime_format is True:
+        return datetime.datetime.utcnow()
+    else:
+        return datetime.datetime.utcnow().isoformat() + 'Z'
diff --git a/taskcluster/taskgraph/util/treeherder.py b/taskcluster/taskgraph/util/treeherder.py
new file mode 100644
index 000000000..e66db582f
--- /dev/null
+++ b/taskcluster/taskgraph/util/treeherder.py
@@ -0,0 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+import re
+
+
+def split_symbol(treeherder_symbol):
+    """Split a symbol expressed as grp(sym) into its two parts.  If no group is
+    given, the returned group is '?'"""
+    groupSymbol = '?'
+    symbol = treeherder_symbol
+    if '(' in symbol:
+        groupSymbol, symbol = re.match(r'([^(]*)\(([^)]*)\)', symbol).groups()
+    return groupSymbol, symbol
+
+
+def join_symbol(group, symbol):
+    """Perform the reverse of split_symbol, combining the given group and
+    symbol.  If the group is '?', then it is omitted."""
+    if group == '?':
+        return symbol
+    return '{}({})'.format(group, symbol)
diff --git a/taskcluster/taskgraph/util/yaml.py b/taskcluster/taskgraph/util/yaml.py
new file mode 100644
index 000000000..4e541b775
--- /dev/null
+++ b/taskcluster/taskgraph/util/yaml.py
@@ -0,0 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import yaml
+
+
+def load_yaml(path, name):
+    """Convenience function to load a YAML file in the given path.  This is
+    useful for loading kind configuration files from the kind path."""
+    filename = os.path.join(path, name)
+    with open(filename, "rb") as f:
+        return yaml.load(f)