diff options
Diffstat (limited to 'taskcluster/taskgraph/test')
22 files changed, 2472 insertions, 0 deletions
diff --git a/taskcluster/taskgraph/test/__init__.py b/taskcluster/taskgraph/test/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/test/__init__.py diff --git a/taskcluster/taskgraph/test/automationrelevance.json b/taskcluster/taskgraph/test/automationrelevance.json new file mode 100644 index 000000000..8adfc446d --- /dev/null +++ b/taskcluster/taskgraph/test/automationrelevance.json @@ -0,0 +1,425 @@ +{ + "changesets": [ + { + "author": "James Long <longster@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300866", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866" + } + ], + "date": [ + 1473196655.0, + 14400 + ], + "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/debugger/new/index.html" + ], + "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "parents": [ + "37c9349b4e8167a61b08b7e119c21ea177b98942" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312890, + "reviewers": [ + { + "name": "jlast", + "revset": "reviewer(jlast)" + }, + { + "name": "bgrins", + "revset": "reviewer(bgrins)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Wes Kocher <wkocher@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473208638.0, + 25200 + ], + "desc": "Merge m-c to fx-team, a=merge", + "extra": { + "branch": "default" + }, + "files": [ + "taskcluster/scripts/builder/build-l10n.sh" + ], + "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "parents": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312891, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Towkir Ahmed <towkir17@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1296648", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648" + } + ], + "date": [ + 1472957580.0, + 14400 + ], + "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/themes/rules.css" + ], + "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "parents": [ + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312892, + "reviewers": [ + { + "name": "ntim", + "revset": "reviewer(ntim)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Oriol <oriol-bugzilla@hotmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300336", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336" + } + ], + "date": [ + 1472921160.0, + 14400 + ], + "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/webconsole/test/browser_webconsole_output_06.js", + "devtools/server/actors/object.js" + ], + "node": "99c542fa43a72ee863c813b5624048d1b443549b", + "parents": [ + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312893, + "reviewers": [ + { + "name": "fitzgen", + "revset": "reviewer(fitzgen)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Ruturaj Vartak <ruturaj@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1472854020.0, + -7200 + ], + "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK", + "extra": { + "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd", + "branch": "default", + "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba", + "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js", + "devtools/server/actors/highlighters/eye-dropper.js" + ], + "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "parents": [ + "99c542fa43a72ee863c813b5624048d1b443549b" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312894, + "reviewers": [ + { + "name": "pbro", + "revset": "reviewer(pbro)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Matteo Ferretti <mferretti@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1299154", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154" + } + ], + "date": [ + 1472629906.0, + -7200 + ], + "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n", + "extra": { + "branch": "default", + "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200" + }, + "files": [ + "docshell/base/nsDocShell.cpp", + "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html" + ], + "node": "541c9086c0f27fba60beecc9bc94543103895c86", + "parents": [ + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312895, + "reviewers": [ + { + "name": "mstange", + "revset": "reviewer(mstange)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Patrick Brosset <pbrosset@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1473239449.0, + -7200 + ], + "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js" + ], + "node": "041a925171e431bf51fb50193ab19d156088c89a", + "parents": [ + "541c9086c0f27fba60beecc9bc94543103895c86" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312896, + "reviewers": [ + { + "name": "me", + "revset": "reviewer(me)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473261233.0, + -7200 + ], + "desc": "merge fx-team to mozilla-central a=merge", + "extra": { + "branch": "default" + }, + "files": [], + "node": "a14f88a9af7a59e677478694bafd9375ac53683e", + "parents": [ + "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9", + "041a925171e431bf51fb50193ab19d156088c89a" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312897, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + } + ], + "visible": true +} + diff --git a/taskcluster/taskgraph/test/test_create.py b/taskcluster/taskgraph/test/test_create.py new file mode 100644 index 000000000..b8da3aec0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_create.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import os + +from .. import create +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask + +from mozunit import main + + +class TestCreate(unittest.TestCase): + + def setUp(self): + self.old_task_id = os.environ.get('TASK_ID') + if 'TASK_ID' in os.environ: + del os.environ['TASK_ID'] + self.created_tasks = {} + self.old_create_task = create._create_task + create._create_task = self.fake_create_task + + def tearDown(self): + create._create_task = self.old_create_task + if self.old_task_id: + os.environ['TASK_ID'] = self.old_task_id + elif 'TASK_ID' in os.environ: + del os.environ['TASK_ID'] + + def fake_create_task(self, session, task_id, label, task_def): + self.created_tasks[task_id] = task_def + + def test_create_tasks(self): + tasks = { + 'tid-a': TestTask(label='a', task={'payload': 'hello world'}), + 'tid-b': TestTask(label='b', task={'payload': 'hello world'}), + } + label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} + graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) + taskgraph = TaskGraph(tasks, graph) + + create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) + + for tid, task in self.created_tasks.iteritems(): + self.assertEqual(task['payload'], 'hello world') + self.assertEqual(task['schedulerId'], 'gecko-level-4') + # make sure the dependencies exist, at least + for depid in task.get('dependencies', []): + if depid is 'decisiontask': + # Don't look for decisiontask here + continue + self.assertIn(depid, self.created_tasks) + + def test_create_task_without_dependencies(self): + "a task with no dependencies depends on the decision task" + os.environ['TASK_ID'] = 'decisiontask' + tasks = { + 'tid-a': TestTask(label='a', task={'payload': 'hello world'}), + } + label_to_taskid = {'a': 'tid-a'} + graph = Graph(nodes={'tid-a'}, edges=set()) + taskgraph = TaskGraph(tasks, graph) + + create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) + + for tid, task in self.created_tasks.iteritems(): + self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']]) + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_decision.py b/taskcluster/taskgraph/test/test_decision.py new file mode 100644 index 000000000..364965206 --- /dev/null +++ b/taskcluster/taskgraph/test/test_decision.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import json +import yaml +import shutil +import unittest +import tempfile + +from .. import decision +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + + +class TestDecision(unittest.TestCase): + + def test_taskgraph_to_json(self): + tasks = { + 'a': TestTask(label='a', attributes={'attr': 'a-task'}), + 'b': TestTask(label='b', task={'task': 'def'}), + } + graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) + taskgraph = TaskGraph(tasks, graph) + + res = taskgraph.to_json() + + self.assertEqual(res, { + 'a': { + 'label': 'a', + 'attributes': {'attr': 'a-task', 'kind': 'test'}, + 'task': {}, + 'dependencies': {'edgelabel': 'b'}, + 'kind_implementation': 'taskgraph.test.util:TestTask', + }, + 'b': { + 'label': 'b', + 'attributes': {'kind': 'test'}, + 'task': {'task': 'def'}, + 'dependencies': {}, + 'kind_implementation': 'taskgraph.test.util:TestTask', + } + }) + + def test_write_artifact_json(self): + data = [{'some': 'data'}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.json", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f: + self.assertEqual(json.load(f), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = 'artifacts' + + def test_write_artifact_yml(self): + data = [{'some': 'data'}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.yml", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f: + self.assertEqual(yaml.safe_load(f), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = 'artifacts' + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_files_changed.py b/taskcluster/taskgraph/test/test_files_changed.py new file mode 100644 index 000000000..0e3366b3c --- /dev/null +++ b/taskcluster/taskgraph/test/test_files_changed.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import os +import unittest + +from .. import files_changed + +PARAMS = { + 'head_repository': 'https://hg.mozilla.org/mozilla-central', + 'head_rev': 'a14f88a9af7a', +} + +FILES_CHANGED = [ + 'devtools/client/debugger/new/index.html', + 'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js', + 'devtools/client/inspector/test/head.js', + 'devtools/client/themes/rules.css', + 'devtools/client/webconsole/test/browser_webconsole_output_06.js', + 'devtools/server/actors/highlighters/eye-dropper.js', + 'devtools/server/actors/object.js', + 'docshell/base/nsDocShell.cpp', + 'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html', + 'taskcluster/scripts/builder/build-l10n.sh', +] + + +class FakeResponse: + + def json(self): + with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + + def setUp(self): + files_changed._cache.clear() + self.old_get = files_changed.requests.get + + def fake_get(url, **kwargs): + return FakeResponse() + files_changed.requests.get = fake_get + + def tearDown(self): + files_changed.requests.get = self.old_get + + def test_get_changed_files(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])), + FILES_CHANGED) + + +class TestCheck(unittest.TestCase): + + def setUp(self): + files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + + def test_check_match(self): + self.assertTrue(files_changed.check(PARAMS, ["devtools/**"])) diff --git a/taskcluster/taskgraph/test/test_generator.py b/taskcluster/taskgraph/test/test_generator.py new file mode 100644 index 000000000..f1b466e4d --- /dev/null +++ b/taskcluster/taskgraph/test/test_generator.py @@ -0,0 +1,129 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..generator import TaskGraphGenerator, Kind +from .. import graph +from ..task import base +from mozunit import main + + +class FakeTask(base.Task): + + def __init__(self, **kwargs): + self.i = kwargs.pop('i') + super(FakeTask, self).__init__(**kwargs) + + @classmethod + def load_tasks(cls, kind, path, config, parameters, loaded_tasks): + return [cls(kind=kind, + label='{}-t-{}'.format(kind, i), + attributes={'tasknum': str(i)}, + task={}, + i=i) + for i in range(3)] + + def get_dependencies(self, full_task_set): + i = self.i + if i > 0: + return [('{}-t-{}'.format(self.kind, i - 1), 'prev')] + else: + return [] + + def optimize(self, params): + return False, None + + +class FakeKind(Kind): + + def _get_impl_class(self): + return FakeTask + + def load_tasks(self, parameters, loaded_tasks): + FakeKind.loaded_kinds.append(self.name) + return super(FakeKind, self).load_tasks(parameters, loaded_tasks) + + +class WithFakeKind(TaskGraphGenerator): + + def _load_kinds(self): + for kind_name, deps in self.parameters['kinds']: + yield FakeKind( + kind_name, '/fake', + {'kind-dependencies': deps} if deps else {}) + + +class TestGenerator(unittest.TestCase): + + def maketgg(self, target_tasks=None, kinds=[('fake', [])]): + FakeKind.loaded_kinds = [] + self.target_tasks = target_tasks or [] + + def target_tasks_method(full_task_graph, parameters): + return self.target_tasks + return WithFakeKind('/root', {'kinds': kinds}, target_tasks_method) + + def test_kind_ordering(self): + "When task kinds depend on each other, they are loaded in postorder" + self.tgg = self.maketgg(kinds=[ + ('fake3', ['fake2', 'fake1']), + ('fake2', ['fake1']), + ('fake1', []), + ]) + self.tgg._run_until('full_task_set') + self.assertEqual(FakeKind.loaded_kinds, ['fake1', 'fake2', 'fake3']) + + def test_full_task_set(self): + "The full_task_set property has all tasks" + self.tgg = self.maketgg() + self.assertEqual(self.tgg.full_task_set.graph, + graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'}, set())) + self.assertEqual(sorted(self.tgg.full_task_set.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1', 'fake-t-2'])) + + def test_full_task_graph(self): + "The full_task_graph property has all tasks, and links" + self.tgg = self.maketgg() + self.assertEqual(self.tgg.full_task_graph.graph, + graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'}, + { + ('fake-t-1', 'fake-t-0', 'prev'), + ('fake-t-2', 'fake-t-1', 'prev'), + })) + self.assertEqual(sorted(self.tgg.full_task_graph.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1', 'fake-t-2'])) + + def test_target_task_set(self): + "The target_task_set property has the targeted tasks" + self.tgg = self.maketgg(['fake-t-1']) + self.assertEqual(self.tgg.target_task_set.graph, + graph.Graph({'fake-t-1'}, set())) + self.assertEqual(self.tgg.target_task_set.tasks.keys(), + ['fake-t-1']) + + def test_target_task_graph(self): + "The target_task_graph property has the targeted tasks and deps" + self.tgg = self.maketgg(['fake-t-1']) + self.assertEqual(self.tgg.target_task_graph.graph, + graph.Graph({'fake-t-0', 'fake-t-1'}, + {('fake-t-1', 'fake-t-0', 'prev')})) + self.assertEqual(sorted(self.tgg.target_task_graph.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1'])) + + def test_optimized_task_graph(self): + "The optimized task graph contains task ids" + self.tgg = self.maketgg(['fake-t-2']) + tid = self.tgg.label_to_taskid + self.assertEqual( + self.tgg.optimized_task_graph.graph, + graph.Graph({tid['fake-t-0'], tid['fake-t-1'], tid['fake-t-2']}, { + (tid['fake-t-1'], tid['fake-t-0'], 'prev'), + (tid['fake-t-2'], tid['fake-t-1'], 'prev'), + })) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_graph.py b/taskcluster/taskgraph/test/test_graph.py new file mode 100644 index 000000000..5c4c950a7 --- /dev/null +++ b/taskcluster/taskgraph/test/test_graph.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..graph import Graph +from mozunit import main + + +class TestGraph(unittest.TestCase): + + tree = Graph(set(['a', 'b', 'c', 'd', 'e', 'f', 'g']), { + ('a', 'b', 'L'), + ('a', 'c', 'L'), + ('b', 'd', 'K'), + ('b', 'e', 'K'), + ('c', 'f', 'N'), + ('c', 'g', 'N'), + }) + + linear = Graph(set(['1', '2', '3', '4']), { + ('1', '2', 'L'), + ('2', '3', 'L'), + ('3', '4', 'L'), + }) + + diamonds = Graph(set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']), + set(tuple(x) for x in + 'AFL ADL BDL BEL CEL CHL DFL DGL EGL EHL FIL GIL GJL HJL'.split() + )) + + multi_edges = Graph(set(['1', '2', '3', '4']), { + ('2', '1', 'red'), + ('2', '1', 'blue'), + ('3', '1', 'red'), + ('3', '2', 'blue'), + ('3', '2', 'green'), + ('4', '3', 'green'), + }) + + disjoint = Graph(set(['1', '2', '3', '4', 'α', 'β', 'γ']), { + ('2', '1', 'red'), + ('3', '1', 'red'), + ('3', '2', 'green'), + ('4', '3', 'green'), + ('α', 'β', 'πράσινο'), + ('β', 'γ', 'κόκκινο'), + ('α', 'γ', 'μπλε'), + }) + + def test_transitive_closure_empty(self): + "transitive closure of an empty set is an empty graph" + g = Graph(set(['a', 'b', 'c']), {('a', 'b', 'L'), ('a', 'c', 'L')}) + self.assertEqual(g.transitive_closure(set()), + Graph(set(), set())) + + def test_transitive_closure_disjoint(self): + "transitive closure of a disjoint set is a subset" + g = Graph(set(['a', 'b', 'c']), set()) + self.assertEqual(g.transitive_closure(set(['a', 'c'])), + Graph(set(['a', 'c']), set())) + + def test_transitive_closure_trees(self): + "transitive closure of a tree, at two non-root nodes, is the two subtrees" + self.assertEqual(self.tree.transitive_closure(set(['b', 'c'])), + Graph(set(['b', 'c', 'd', 'e', 'f', 'g']), { + ('b', 'd', 'K'), + ('b', 'e', 'K'), + ('c', 'f', 'N'), + ('c', 'g', 'N'), + })) + + def test_transitive_closure_multi_edges(self): + "transitive closure of a tree with multiple edges between nodes keeps those edges" + self.assertEqual(self.multi_edges.transitive_closure(set(['3'])), + Graph(set(['1', '2', '3']), { + ('2', '1', 'red'), + ('2', '1', 'blue'), + ('3', '1', 'red'), + ('3', '2', 'blue'), + ('3', '2', 'green'), + })) + + def test_transitive_closure_disjoint_edges(self): + "transitive closure of a disjoint graph keeps those edges" + self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])), + Graph(set(['1', '2', '3', 'β', 'γ']), { + ('2', '1', 'red'), + ('3', '1', 'red'), + ('3', '2', 'green'), + ('β', 'γ', 'κόκκινο'), + })) + + def test_transitive_closure_linear(self): + "transitive closure of a linear graph includes all nodes in the line" + self.assertEqual(self.linear.transitive_closure(set(['1'])), self.linear) + + def test_visit_postorder_empty(self): + "postorder visit of an empty graph is empty" + self.assertEqual(list(Graph(set(), set()).visit_postorder()), []) + + def assert_postorder(self, seq, all_nodes): + seen = set() + for e in seq: + for l, r, n in self.tree.edges: + if l == e: + self.failUnless(r in seen) + seen.add(e) + self.assertEqual(seen, all_nodes) + + def test_visit_postorder_tree(self): + "postorder visit of a tree satisfies invariant" + self.assert_postorder(self.tree.visit_postorder(), self.tree.nodes) + + def test_visit_postorder_diamonds(self): + "postorder visit of a graph full of diamonds satisfies invariant" + self.assert_postorder(self.diamonds.visit_postorder(), self.diamonds.nodes) + + def test_visit_postorder_multi_edges(self): + "postorder visit of a graph with duplicate edges satisfies invariant" + self.assert_postorder(self.multi_edges.visit_postorder(), self.multi_edges.nodes) + + def test_visit_postorder_disjoint(self): + "postorder visit of a disjoint graph satisfies invariant" + self.assert_postorder(self.disjoint.visit_postorder(), self.disjoint.nodes) + + def test_links_dict(self): + "link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.links_dict(), { + '2': set(['1']), + '3': set(['1', '2']), + '4': set(['3']), + }) + + def test_named_links_dict(self): + "named link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.named_links_dict(), { + '2': dict(red='1', blue='1'), + '3': dict(red='1', blue='2', green='2'), + '4': dict(green='3'), + }) + + def test_reverse_links_dict(self): + "reverse link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.reverse_links_dict(), { + '1': set(['2', '3']), + '2': set(['3']), + '3': set(['4']), + }) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_optimize.py b/taskcluster/taskgraph/test/test_optimize.py new file mode 100644 index 000000000..8d2ddf247 --- /dev/null +++ b/taskcluster/taskgraph/test/test_optimize.py @@ -0,0 +1,256 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..optimize import optimize_task_graph, resolve_task_references +from ..optimize import annotate_task_graph, get_subgraph +from ..taskgraph import TaskGraph +from .. import graph +from .util import TestTask + + +class TestResolveTaskReferences(unittest.TestCase): + + def do(self, input, output): + taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)} + self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), output) + + def test_in_list(self): + "resolve_task_references resolves task references in a list" + self.do({'in-a-list': ['stuff', {'task-reference': '<edge1>'}]}, + {'in-a-list': ['stuff', 'tid1']}) + + def test_in_dict(self): + "resolve_task_references resolves task references in a dict" + self.do({'in-a-dict': {'stuff': {'task-reference': '<edge2>'}}}, + {'in-a-dict': {'stuff': 'tid2'}}) + + def test_multiple(self): + "resolve_task_references resolves multiple references in the same string" + self.do({'multiple': {'task-reference': 'stuff <edge1> stuff <edge2> after'}}, + {'multiple': 'stuff tid1 stuff tid2 after'}) + + def test_embedded(self): + "resolve_task_references resolves ebmedded references" + self.do({'embedded': {'task-reference': 'stuff before <edge3> stuff after'}}, + {'embedded': 'stuff before tid3 stuff after'}) + + def test_escaping(self): + "resolve_task_references resolves escapes in task references" + self.do({'escape': {'task-reference': '<<><edge3>>'}}, + {'escape': '<tid3>'}) + + def test_invalid(self): + "resolve_task_references raises a KeyError on reference to an invalid task" + self.assertRaisesRegexp( + KeyError, + "task 'subject' has no dependency named 'no-such'", + lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {}) + ) + + +class OptimizingTask(TestTask): + # the `optimize` method on this class is overridden direclty in the tests + # below. + pass + + +class TestOptimize(unittest.TestCase): + + kind = None + + def make_task(self, label, task_def=None, optimized=None, task_id=None): + task_def = task_def or {'sample': 'task-def'} + task = OptimizingTask(label=label, task=task_def) + task.optimized = optimized + task.task_id = task_id + return task + + def make_graph(self, *tasks_and_edges): + tasks = {t.label: t for t in tasks_and_edges if isinstance(t, OptimizingTask)} + edges = {e for e in tasks_and_edges if not isinstance(e, OptimizingTask)} + return TaskGraph(tasks, graph.Graph(set(tasks), edges)) + + def assert_annotations(self, graph, **annotations): + def repl(task_id): + return 'SLUGID' if task_id and len(task_id) == 22 else task_id + got_annotations = { + t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues() + } + self.assertEqual(got_annotations, annotations) + + def test_annotate_task_graph_no_optimize(self): + "annotating marks everything as un-optimized if the kind returns that" + OptimizingTask.optimize = lambda self, params: (False, None) + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(False, None), + task3=(False, None) + ) + + def test_annotate_task_graph_taskid_without_optimize(self): + "raises exception if kind returns a taskid without optimizing" + OptimizingTask.optimize = lambda self, params: (False, 'some-taskid') + graph = self.make_graph(self.make_task('task1')) + self.assertRaises( + Exception, + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + ) + + def test_annotate_task_graph_optimize_away_dependency(self): + "raises exception if kind optimizes away a task on which another depends" + OptimizingTask.optimize = \ + lambda self, params: (True, None) if self.label == 'task1' else (False, None) + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + ('task2', 'task1', 'build'), + ) + self.assertRaises( + Exception, + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + ) + + def test_annotate_task_graph_do_not_optimize(self): + "annotating marks everything as un-optimized if in do_not_optimize" + OptimizingTask.optimize = lambda self, params: (True, 'taskid') + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + ('task2', 'task1', 'build'), + ) + label_to_taskid = {} + annotate_task_graph(graph, {}, {'task1', 'task2'}, + graph.graph.named_links_dict(), label_to_taskid, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(False, None) + ) + self.assertEqual + + def test_annotate_task_graph_nos_do_not_propagate(self): + "a task with a non-optimized dependency can be optimized" + OptimizingTask.optimize = \ + lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid') + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + annotate_task_graph(graph, {}, set(), + graph.graph.named_links_dict(), {}, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(True, 'taskid'), + task3=(True, 'taskid') + ) + + def test_get_subgraph_single_dep(self): + "when a single dependency is optimized, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task('task2', optimized=False), + self.make_task('task3', optimized=False), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + label_to_taskid = {'task1': 'dep1'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task2 = label_to_taskid['task2'] + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task2, task3}) + self.assertEqual(sub.graph.edges, {(task2, task3, 'image')}) + self.assertEqual(sub.tasks[task2].task_id, task2) + self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), + sorted([task3, 'dep1'])) + self.assertEqual(sub.tasks[task3].task_id, task3) + self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), []) + + def test_get_subgraph_dep_chain(self): + "when a dependency chain is optimized, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task('task2', optimized=True, task_id='dep2'), + self.make_task('task3', optimized=False), + ('task2', 'task1', 'build'), + ('task3', 'task2', 'image'), + ) + label_to_taskid = {'task1': 'dep1', 'task2': 'dep2'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task3}) + self.assertEqual(sub.graph.edges, set()) + self.assertEqual(sub.tasks[task3].task_id, task3) + self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), ['dep2']) + + def test_get_subgraph_opt_away(self): + "when a leaf task is optimized away, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=False), + self.make_task('task2', optimized=True), + ('task2', 'task1', 'build'), + ) + label_to_taskid = {'task2': 'dep2'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task1 = label_to_taskid['task1'] + self.assertEqual(sub.graph.nodes, {task1}) + self.assertEqual(sub.graph.edges, set()) + self.assertEqual(sub.tasks[task1].task_id, task1) + self.assertEqual(sorted(sub.tasks[task1].task['dependencies']), []) + + def test_get_subgraph_refs_resolved(self): + "get_subgraph resolves task references" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task( + 'task2', + optimized=False, + task_def={'payload': {'task-reference': 'http://<build>/<test>'}} + ), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'test'), + self.make_task('task3', optimized=False), + ) + label_to_taskid = {'task1': 'dep1'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task2 = label_to_taskid['task2'] + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task2, task3}) + self.assertEqual(sub.graph.edges, {(task2, task3, 'test')}) + self.assertEqual(sub.tasks[task2].task_id, task2) + self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1'])) + self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3) + self.assertEqual(sub.tasks[task3].task_id, task3) + + def test_optimize(self): + "optimize_task_graph annotates and extracts the subgraph from a simple graph" + OptimizingTask.optimize = \ + lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None) + input = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + opt, label_to_taskid = optimize_task_graph(input, {}, set()) + self.assertEqual(opt.graph, graph.Graph( + {label_to_taskid['task2'], label_to_taskid['task3']}, + {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')})) diff --git a/taskcluster/taskgraph/test/test_parameters.py b/taskcluster/taskgraph/test/test_parameters.py new file mode 100644 index 000000000..43d853d7b --- /dev/null +++ b/taskcluster/taskgraph/test/test_parameters.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..parameters import Parameters, load_parameters_file, PARAMETER_NAMES +from mozunit import main, MockedOpen + + +class TestParameters(unittest.TestCase): + + vals = {n: n for n in PARAMETER_NAMES} + + def test_Parameters_immutable(self): + p = Parameters(**self.vals) + + def assign(): + p['head_ref'] = 20 + self.assertRaises(Exception, assign) + + def test_Parameters_missing_KeyError(self): + p = Parameters(**self.vals) + self.assertRaises(KeyError, lambda: p['z']) + + def test_Parameters_invalid_KeyError(self): + """even if the value is present, if it's not a valid property, raise KeyError""" + p = Parameters(xyz=10, **self.vals) + self.assertRaises(KeyError, lambda: p['xyz']) + + def test_Parameters_get(self): + p = Parameters(head_ref=10, level=20) + self.assertEqual(p['head_ref'], 10) + + def test_Parameters_check(self): + p = Parameters(**self.vals) + p.check() # should not raise + + def test_Parameters_check_missing(self): + p = Parameters() + self.assertRaises(Exception, lambda: p.check()) + + def test_Parameters_check_extra(self): + p = Parameters(xyz=10, **self.vals) + self.assertRaises(Exception, lambda: p.check()) + + def test_load_parameters_file_yaml(self): + with MockedOpen({"params.yml": "some: data\n"}): + self.assertEqual( + load_parameters_file({'parameters': 'params.yml'}), + {'some': 'data'}) + + def test_load_parameters_file_json(self): + with MockedOpen({"params.json": '{"some": "data"}'}): + self.assertEqual( + load_parameters_file({'parameters': 'params.json'}), + {'some': 'data'}) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_target_tasks.py b/taskcluster/taskgraph/test/test_target_tasks.py new file mode 100644 index 000000000..035ccefd8 --- /dev/null +++ b/taskcluster/taskgraph/test/test_target_tasks.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from .. import target_tasks +from .. import try_option_syntax +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + + +class FakeTryOptionSyntax(object): + + def __init__(self, message, task_graph): + self.trigger_tests = 0 + self.notifications = None + + def task_matches(self, attributes): + return 'at-at' in attributes + + +class TestTargetTasks(unittest.TestCase): + + def default_matches(self, run_on_projects, project): + method = target_tasks.get_method('default') + graph = TaskGraph(tasks={ + 'a': TestTask(kind='build', label='a', + attributes={'run_on_projects': run_on_projects}), + }, graph=Graph(nodes={'a'}, edges=set())) + parameters = {'project': project} + return 'a' in method(graph, parameters) + + def test_default_all(self): + """run_on_projects=[all] includes release, integration, and other projects""" + self.assertTrue(self.default_matches(['all'], 'mozilla-central')) + self.assertTrue(self.default_matches(['all'], 'mozilla-inbound')) + self.assertTrue(self.default_matches(['all'], 'mozilla-aurora')) + self.assertTrue(self.default_matches(['all'], 'baobab')) + + def test_default_integration(self): + """run_on_projects=[integration] includes integration projects""" + self.assertFalse(self.default_matches(['integration'], 'mozilla-central')) + self.assertTrue(self.default_matches(['integration'], 'mozilla-inbound')) + self.assertFalse(self.default_matches(['integration'], 'baobab')) + + def test_default_relesae(self): + """run_on_projects=[release] includes release projects""" + self.assertTrue(self.default_matches(['release'], 'mozilla-central')) + self.assertFalse(self.default_matches(['release'], 'mozilla-inbound')) + self.assertFalse(self.default_matches(['release'], 'baobab')) + + def test_default_nothing(self): + """run_on_projects=[] includes nothing""" + self.assertFalse(self.default_matches([], 'mozilla-central')) + self.assertFalse(self.default_matches([], 'mozilla-inbound')) + self.assertFalse(self.default_matches([], 'baobab')) + + def test_try_option_syntax(self): + tasks = { + 'a': TestTask(kind=None, label='a'), + 'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}), + } + graph = Graph(nodes=set('ab'), edges=set()) + tg = TaskGraph(tasks, graph) + params = {'message': 'try me'} + + orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax + try: + try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax + method = target_tasks.get_method('try_option_syntax') + self.assertEqual(method(tg, params), ['b']) + finally: + try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_task_docker_image.py b/taskcluster/taskgraph/test/test_task_docker_image.py new file mode 100644 index 000000000..8f247db3e --- /dev/null +++ b/taskcluster/taskgraph/test/test_task_docker_image.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import os + +from ..task import docker_image +from mozunit import main + + +KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image') + + +class TestDockerImageKind(unittest.TestCase): + + def setUp(self): + self.task = docker_image.DockerImageTask( + 'docker-image', + KIND_PATH, + {}, + {}, + index_paths=[]) + + def test_get_task_dependencies(self): + # this one's easy! + self.assertEqual(self.task.get_dependencies(None), []) + + # TODO: optimize_task + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_taskgraph.py b/taskcluster/taskgraph/test/test_taskgraph.py new file mode 100644 index 000000000..f8f09bce9 --- /dev/null +++ b/taskcluster/taskgraph/test/test_taskgraph.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..graph import Graph +from ..task.docker_image import DockerImageTask +from ..task.transform import TransformTask +from ..taskgraph import TaskGraph +from mozunit import main +from taskgraph.util.docker import INDEX_PREFIX + + +class TestTargetTasks(unittest.TestCase): + + def test_from_json(self): + task = { + "routes": [], + "extra": { + "imageMeta": { + "contextHash": "<hash>", + "imageName": "<image>", + "level": "1" + } + } + } + index_paths = ["{}.level-{}.<image>.hash.<hash>".format(INDEX_PREFIX, level) + for level in range(1, 4)] + graph = TaskGraph(tasks={ + 'a': TransformTask( + kind='fancy', + task={ + 'label': 'a', + 'attributes': {}, + 'dependencies': {}, + 'when': {}, + 'task': {'task': 'def'}, + }), + 'b': DockerImageTask(kind='docker-image', + label='b', + attributes={}, + task=task, + index_paths=index_paths), + }, graph=Graph(nodes={'a', 'b'}, edges=set())) + + tasks, new_graph = TaskGraph.from_json(graph.to_json()) + self.assertEqual(graph.tasks['a'], new_graph.tasks['a']) + self.assertEqual(graph, new_graph) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_transforms_base.py b/taskcluster/taskgraph/test/test_transforms_base.py new file mode 100644 index 000000000..0a0dfcaf2 --- /dev/null +++ b/taskcluster/taskgraph/test/test_transforms_base.py @@ -0,0 +1,143 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from mozunit import main +from taskgraph.transforms.base import ( + validate_schema, + get_keyed_by, + TransformSequence +) +from voluptuous import Schema + +schema = Schema({ + 'x': int, + 'y': basestring, +}) + +transforms = TransformSequence() + + +@transforms.add +def trans1(config, tests): + for test in tests: + test['one'] = 1 + yield test + + +@transforms.add +def trans2(config, tests): + for test in tests: + test['two'] = 2 + yield test + + +class TestTransformSequence(unittest.TestCase): + + def test_sequence(self): + tests = [{}, {'two': 1, 'second': True}] + res = list(transforms({}, tests)) + self.assertEqual(res, [ + {u'two': 2, u'one': 1}, + {u'second': True, u'two': 2, u'one': 1}, + ]) + + +class TestValidateSchema(unittest.TestCase): + + def test_valid(self): + validate_schema(schema, {'x': 10, 'y': 'foo'}, "pfx") + + def test_invalid(self): + try: + validate_schema(schema, {'x': 'not-int'}, "pfx") + self.fail("no exception raised") + except Exception, e: + self.failUnless(str(e).startswith("pfx\n")) + + +class TestKeyedBy(unittest.TestCase): + + def test_simple_value(self): + test = { + 'test-name': 'tname', + 'option': 10, + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 10) + + def test_by_value(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + 'b': 20, + }, + }, + 'other-value': 'b', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 20) + + def test_by_value_regex(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-test-platform': { + 'macosx64/.*': 10, + 'linux64/debug': 20, + 'default': 5, + }, + }, + 'test-platform': 'macosx64/debug', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 10) + + def test_by_value_default(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + 'default': 30, + }, + }, + 'other-value': 'xxx', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 30) + + def test_by_value_invalid_dict(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-something-else': {}, + 'by-other-value': {}, + }, + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + + def test_by_value_invalid_no_default(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + }, + }, + 'other-value': 'b', + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + + def test_by_value_invalid_no_by(self): + test = { + 'test-name': 'tname', + 'option': { + 'other-value': {}, + }, + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_try_option_syntax.py b/taskcluster/taskgraph/test/test_try_option_syntax.py new file mode 100644 index 000000000..29aa2d5a9 --- /dev/null +++ b/taskcluster/taskgraph/test/test_try_option_syntax.py @@ -0,0 +1,274 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import itertools + +from ..try_option_syntax import TryOptionSyntax +from ..try_option_syntax import RIDEALONG_BUILDS +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + +# an empty graph, for things that don't look at it +empty_graph = TaskGraph({}, Graph(set(), set())) + + +def unittest_task(n, tp): + return (n, TestTask('test', n, { + 'unittest_try_name': n, + 'test_platform': tp, + })) + + +def talos_task(n, tp): + return (n, TestTask('test', n, { + 'talos_try_name': n, + 'test_platform': tp, + })) + +tasks = {k: v for k, v in [ + unittest_task('mochitest-browser-chrome', 'linux'), + unittest_task('mochitest-e10s-browser-chrome', 'linux64'), + unittest_task('mochitest-chrome', 'linux'), + unittest_task('mochitest-webgl', 'linux'), + unittest_task('crashtest-e10s', 'linux'), + unittest_task('gtest', 'linux64'), + talos_task('dromaeojs', 'linux64'), +]} +unittest_tasks = {k: v for k, v in tasks.iteritems() + if 'unittest_try_name' in v.attributes} +talos_tasks = {k: v for k, v in tasks.iteritems() + if 'talos_try_name' in v.attributes} +graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set())) + + +class TestTryOptionSyntax(unittest.TestCase): + + def test_empty_message(self): + "Given an empty message, it should return an empty value" + tos = TryOptionSyntax('', empty_graph) + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + self.assertEqual(tos.unittests, []) + self.assertEqual(tos.talos, []) + self.assertEqual(tos.platforms, []) + + def test_message_without_try(self): + "Given a non-try message, it should return an empty value" + tos = TryOptionSyntax('Bug 1234: frobnicte the foo', empty_graph) + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + self.assertEqual(tos.unittests, []) + self.assertEqual(tos.talos, []) + self.assertEqual(tos.platforms, []) + + def test_unknown_args(self): + "unknown arguments are ignored" + tos = TryOptionSyntax('try: --doubledash -z extra', empty_graph) + # equilvant to "try:".. + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, None) + + def test_b_do(self): + "-b do should produce both build_types" + tos = TryOptionSyntax('try: -b do', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug', 'opt']) + + def test_b_d(self): + "-b d should produce build_types=['debug']" + tos = TryOptionSyntax('try: -b d', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug']) + + def test_b_o(self): + "-b o should produce build_types=['opt']" + tos = TryOptionSyntax('try: -b o', empty_graph) + self.assertEqual(sorted(tos.build_types), ['opt']) + + def test_build_o(self): + "--build o should produce build_types=['opt']" + tos = TryOptionSyntax('try: --build o', empty_graph) + self.assertEqual(sorted(tos.build_types), ['opt']) + + def test_b_dx(self): + "-b dx should produce build_types=['debug'], silently ignoring the x" + tos = TryOptionSyntax('try: -b dx', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug']) + + def test_j_job(self): + "-j somejob sets jobs=['somejob']" + tos = TryOptionSyntax('try: -j somejob', empty_graph) + self.assertEqual(sorted(tos.jobs), ['somejob']) + + def test_j_jobs(self): + "-j job1,job2 sets jobs=['job1', 'job2']" + tos = TryOptionSyntax('try: -j job1,job2', empty_graph) + self.assertEqual(sorted(tos.jobs), ['job1', 'job2']) + + def test_j_all(self): + "-j all sets jobs=None" + tos = TryOptionSyntax('try: -j all', empty_graph) + self.assertEqual(tos.jobs, None) + + def test_j_twice(self): + "-j job1 -j job2 sets jobs=job1, job2" + tos = TryOptionSyntax('try: -j job1 -j job2', empty_graph) + self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2'])) + + def test_p_all(self): + "-p all sets platforms=None" + tos = TryOptionSyntax('try: -p all', empty_graph) + self.assertEqual(tos.platforms, None) + + def test_p_linux(self): + "-p linux sets platforms=['linux', 'linux-l10n']" + tos = TryOptionSyntax('try: -p linux', empty_graph) + self.assertEqual(tos.platforms, ['linux', 'linux-l10n']) + + def test_p_linux_win32(self): + "-p linux,win32 sets platforms=['linux', 'linux-l10n', 'win32']" + tos = TryOptionSyntax('try: -p linux,win32', empty_graph) + self.assertEqual(sorted(tos.platforms), ['linux', 'linux-l10n', 'win32']) + + def test_p_expands_ridealongs(self): + "-p linux,linux64 includes the RIDEALONG_BUILDS" + tos = TryOptionSyntax('try: -p linux,linux64', empty_graph) + ridealongs = list(task + for task in itertools.chain.from_iterable( + RIDEALONG_BUILDS.itervalues() + ) + if 'android' not in task) # Don't include android-l10n + self.assertEqual(sorted(tos.platforms), sorted(['linux', 'linux64'] + ridealongs)) + + def test_u_none(self): + "-u none sets unittests=[]" + tos = TryOptionSyntax('try: -u none', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), []) + + def test_u_all(self): + "-u all sets unittests=[..whole list..]" + tos = TryOptionSyntax('try: -u all', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks])) + + def test_u_single(self): + "-u mochitest-webgl sets unittests=[mochitest-webgl]" + tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}])) + + def test_u_alias(self): + "-u mochitest-gl sets unittests=[mochitest-webgl]" + tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}])) + + def test_u_multi_alias(self): + "-u e10s sets unittests=[all e10s unittests]" + tos = TryOptionSyntax('try: -u e10s', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': t} for t in unittest_tasks if 'e10s' in t + ])) + + def test_u_commas(self): + "-u mochitest-webgl,gtest sets unittests=both" + tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'mochitest-webgl'}, + {'test': 'gtest'}, + ])) + + def test_u_chunks(self): + "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest" + tos = TryOptionSyntax('try: -u gtest-3,gtest-4', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'only_chunks': set('34')}, + ])) + + def test_u_platform(self): + "-u gtest[linux] selects the linux platform for gtest" + tos = TryOptionSyntax('try: -u gtest[linux]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux']}, + ])) + + def test_u_platforms(self): + "-u gtest[linux,win32] selects the linux and win32 platforms for gtest" + tos = TryOptionSyntax('try: -u gtest[linux,win32]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'win32']}, + ])) + + def test_u_platforms_pretty(self): + "-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan platforms for gtest" + tos = TryOptionSyntax('try: -u gtest[Ubuntu]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'linux64', 'linux64-asan']}, + ])) + + def test_u_platforms_negated(self): + "-u gtest[-linux] selects all platforms but linux for gtest" + tos = TryOptionSyntax('try: -u gtest[-linux]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux64']}, + ])) + + def test_u_platforms_negated_pretty(self): + "-u gtest[Ubuntu,-x64] selects just linux for gtest" + tos = TryOptionSyntax('try: -u gtest[Ubuntu,-x64]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux']}, + ])) + + def test_u_chunks_platforms(self): + "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest" + tos = TryOptionSyntax('try: -u gtest-1[linux,win32]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')}, + ])) + + def test_t_none(self): + "-t none sets talos=[]" + tos = TryOptionSyntax('try: -t none', graph_with_jobs) + self.assertEqual(sorted(tos.talos), []) + + def test_t_all(self): + "-t all sets talos=[..whole list..]" + tos = TryOptionSyntax('try: -t all', graph_with_jobs) + self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks])) + + def test_t_single(self): + "-t mochitest-webgl sets talos=[mochitest-webgl]" + tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs) + self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}])) + + # -t shares an implementation with -u, so it's not tested heavily + + def test_trigger_tests(self): + "--rebuild 10 sets trigger_tests" + tos = TryOptionSyntax('try: --rebuild 10', empty_graph) + self.assertEqual(tos.trigger_tests, 10) + + def test_interactive(self): + "--interactive sets interactive" + tos = TryOptionSyntax('try: --interactive', empty_graph) + self.assertEqual(tos.interactive, True) + + def test_all_email(self): + "--all-emails sets notifications" + tos = TryOptionSyntax('try: --all-emails', empty_graph) + self.assertEqual(tos.notifications, 'all') + + def test_fail_email(self): + "--failure-emails sets notifications" + tos = TryOptionSyntax('try: --failure-emails', empty_graph) + self.assertEqual(tos.notifications, 'failure') + + def test_no_email(self): + "no email settings don't set notifications" + tos = TryOptionSyntax('try:', empty_graph) + self.assertEqual(tos.notifications, None) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_util_attributes.py b/taskcluster/taskgraph/test/test_util_attributes.py new file mode 100644 index 000000000..c3575e917 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_attributes.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from taskgraph.util.attributes import attrmatch + + +class Attrmatch(unittest.TestCase): + + def test_trivial_match(self): + """Given no conditions, anything matches""" + self.assertTrue(attrmatch({})) + + def test_missing_attribute(self): + """If a filtering attribute is not present, no match""" + self.assertFalse(attrmatch({}, someattr=10)) + + def test_literal_attribute(self): + """Literal attributes must match exactly""" + self.assertTrue(attrmatch({'att': 10}, att=10)) + self.assertFalse(attrmatch({'att': 10}, att=20)) + + def test_set_attribute(self): + """Set attributes require set membership""" + self.assertTrue(attrmatch({'att': 10}, att=set([9, 10]))) + self.assertFalse(attrmatch({'att': 10}, att=set([19, 20]))) + + def test_callable_attribute(self): + """Callable attributes are called and any False causes the match to fail""" + self.assertTrue(attrmatch({'att': 10}, att=lambda val: True)) + self.assertFalse(attrmatch({'att': 10}, att=lambda val: False)) + + def even(val): + return val % 2 == 0 + self.assertTrue(attrmatch({'att': 10}, att=even)) + self.assertFalse(attrmatch({'att': 11}, att=even)) + + def test_all_matches_required(self): + """If only one attribute does not match, the result is False""" + self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3)) + self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3)) + self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3)) diff --git a/taskcluster/taskgraph/test/test_util_docker.py b/taskcluster/taskgraph/test/test_util_docker.py new file mode 100644 index 000000000..3bb7fe8f0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_docker.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import shutil +import stat +import tarfile +import tempfile +import unittest + +from ..util import docker +from mozunit import MockedOpen + + +MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + +class TestDocker(unittest.TestCase): + + def test_generate_context_hash(self): + tmpdir = tempfile.mkdtemp() + old_GECKO = docker.GECKO + docker.GECKO = tmpdir + try: + os.makedirs(os.path.join(tmpdir, 'docker', 'my-image')) + p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile') + with open(p, 'w') as f: + f.write("FROM node\nADD a-file\n") + os.chmod(p, MODE_STANDARD) + p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file') + with open(p, 'w') as f: + f.write("data\n") + os.chmod(p, MODE_STANDARD) + self.assertEqual( + docker.generate_context_hash(docker.GECKO, + os.path.join(docker.GECKO, 'docker/my-image'), + 'my-image'), + 'e61e675ce05e8c11424437db3f1004079374c1a5fe6ad6800346cebe137b0797' + ) + finally: + docker.GECKO = old_GECKO + shutil.rmtree(tmpdir) + + def test_docker_image_explicit_registry(self): + files = {} + files["{}/myimage/REGISTRY".format(docker.DOCKER_ROOT)] = "cool-images" + files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3" + with MockedOpen(files): + self.assertEqual(docker.docker_image('myimage'), "cool-images/myimage:1.2.3") + + def test_docker_image_default_registry(self): + files = {} + files["{}/REGISTRY".format(docker.DOCKER_ROOT)] = "mozilla" + files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3" + with MockedOpen(files): + self.assertEqual(docker.docker_image('myimage'), "mozilla/myimage:1.2.3") + + def test_create_context_tar_basic(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test_image') + os.mkdir(d) + with open(os.path.join(d, 'Dockerfile'), 'a'): + pass + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + with open(os.path.join(d, 'extra'), 'a'): + pass + os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'my_image') + self.assertEqual(h, '2a6d7f1627eba60daf85402418e041d728827d309143c6bc1c6bb3035bde6717') + + # File prefix should be "my_image" + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'my_image/Dockerfile', + 'my_image/extra', + ]) + finally: + shutil.rmtree(tmp) + + def test_create_context_topsrcdir_files(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include extra/file0\n') + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + extra = os.path.join(tmp, 'extra') + os.mkdir(extra) + with open(os.path.join(extra, 'file0'), 'a'): + pass + os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'test_image') + self.assertEqual(h, '20faeb7c134f21187b142b5fadba94ae58865dc929c6c293d8cbc0a087269338') + + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'test_image/Dockerfile', + 'test_image/topsrcdir/extra/file0', + ]) + finally: + shutil.rmtree(tmp) + + def test_create_context_absolute_path(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + # Absolute paths in %include syntax are not allowed. + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include /etc/shadow\n') + + with self.assertRaisesRegexp(Exception, 'cannot be absolute'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_outside_topsrcdir(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include foo/../../../etc/shadow\n') + + with self.assertRaisesRegexp(Exception, 'path outside topsrcdir'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_missing_extra(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include does/not/exist\n') + + with self.assertRaisesRegexp(Exception, 'path does not exist'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_extra_directory(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include extra\n') + fh.write(b'# %include file0\n') + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + extra = os.path.join(tmp, 'extra') + os.mkdir(extra) + for i in range(3): + p = os.path.join(extra, 'file%d' % i) + with open(p, 'wb') as fh: + fh.write(b'file%d' % i) + os.chmod(p, MODE_STANDARD) + + with open(os.path.join(tmp, 'file0'), 'a'): + pass + os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'my_image') + + self.assertEqual(h, 'e5440513ab46ae4c1d056269e1c6715d5da7d4bd673719d360411e35e5b87205') + + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'my_image/Dockerfile', + 'my_image/topsrcdir/extra/file0', + 'my_image/topsrcdir/extra/file1', + 'my_image/topsrcdir/extra/file2', + 'my_image/topsrcdir/file0', + ]) + finally: + shutil.rmtree(tmp) diff --git a/taskcluster/taskgraph/test/test_util_python_path.py b/taskcluster/taskgraph/test/test_util_python_path.py new file mode 100644 index 000000000..9615d1347 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_python_path.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from ..util import python_path + + +class TestObject(object): + + testClassProperty = object() + + +class TestPythonPath(unittest.TestCase): + + def test_find_object_no_such_module(self): + """find_object raises ImportError for a nonexistent module""" + self.assertRaises(ImportError, python_path.find_object, "no_such_module:someobj") + + def test_find_object_no_such_object(self): + """find_object raises AttributeError for a nonexistent object""" + self.assertRaises(AttributeError, python_path.find_object, + "taskgraph.test.test_util_python_path:NoSuchObject") + + def test_find_object_exists(self): + """find_object finds an existing object""" + obj = python_path.find_object( + "taskgraph.test.test_util_python_path:TestObject.testClassProperty") + self.assertIs(obj, TestObject.testClassProperty) diff --git a/taskcluster/taskgraph/test/test_util_templates.py b/taskcluster/taskgraph/test/test_util_templates.py new file mode 100755 index 000000000..47f7494a0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_templates.py @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import mozunit +import textwrap +from taskgraph.util.templates import ( + merge_to, + merge, + Templates, + TemplatesException +) + +files = {} +files['/fixtures/circular.yml'] = textwrap.dedent("""\ + $inherits: + from: 'circular_ref.yml' + variables: + woot: 'inherit' + """) + +files['/fixtures/inherit.yml'] = textwrap.dedent("""\ + $inherits: + from: 'templates.yml' + variables: + woot: 'inherit' + """) + +files['/fixtures/extend_child.yml'] = textwrap.dedent("""\ + list: ['1', '2', '3'] + was_list: ['1'] + obj: + level: 1 + deeper: + woot: 'bar' + list: ['baz'] + """) + +files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\ + $inherits: + from: 'circular.yml' + """) + +files['/fixtures/child_pass.yml'] = textwrap.dedent("""\ + values: + - {{a}} + - {{b}} + - {{c}} + """) + +files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\ + $inherits: + from: 'child_pass.yml' + variables: + a: 'a' + b: 'b' + c: 'c' + """) + +files['/fixtures/deep/2.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/1.yml + + """) + +files['/fixtures/deep/3.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/2.yml + + """) + +files['/fixtures/deep/4.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/3.yml + """) + +files['/fixtures/deep/1.yml'] = textwrap.dedent("""\ + variable: {{value}} + """) + +files['/fixtures/simple.yml'] = textwrap.dedent("""\ + is_simple: true + """) + +files['/fixtures/templates.yml'] = textwrap.dedent("""\ + content: 'content' + variable: '{{woot}}' + """) + +files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\ + $inherits: + from: 'extend_child.yml' + + list: ['4'] + was_list: + replaced: true + obj: + level: 2 + from_parent: true + deeper: + list: ['bar'] + """) + + +class TemplatesTest(unittest.TestCase): + + def setUp(self): + self.mocked_open = mozunit.MockedOpen(files) + self.mocked_open.__enter__() + self.subject = Templates('/fixtures') + + def tearDown(self): + self.mocked_open.__exit__(None, None, None) + + def test_invalid_path(self): + with self.assertRaisesRegexp(TemplatesException, 'must be a directory'): + Templates('/zomg/not/a/dir') + + def test_no_templates(self): + content = self.subject.load('simple.yml', {}) + self.assertEquals(content, { + 'is_simple': True + }) + + def test_with_templates(self): + content = self.subject.load('templates.yml', { + 'woot': 'bar' + }) + + self.assertEquals(content, { + 'content': 'content', + 'variable': 'bar' + }) + + def test_inheritance(self): + ''' + The simple single pass inheritance case. + ''' + content = self.subject.load('inherit.yml', {}) + self.assertEqual(content, { + 'content': 'content', + 'variable': 'inherit' + }) + + def test_inheritance_implicat_pass(self): + ''' + Implicitly pass parameters from the child to the ancestor. + ''' + content = self.subject.load('inherit_pass.yml', { + 'a': 'overriden' + }) + + self.assertEqual(content, {'values': ['overriden', 'b', 'c']}) + + def test_inheritance_circular(self): + ''' + Circular reference handling. + ''' + with self.assertRaisesRegexp(TemplatesException, 'circular'): + self.subject.load('circular.yml', {}) + + def test_deep_inheritance(self): + content = self.subject.load('deep/4.yml', { + 'value': 'myvalue' + }) + self.assertEqual(content, {'variable': 'myvalue'}) + + def test_inheritance_with_simple_extensions(self): + content = self.subject.load('extend_parent.yml', {}) + self.assertEquals(content, { + 'list': ['1', '2', '3', '4'], + 'obj': { + 'from_parent': True, + 'deeper': { + 'woot': 'bar', + 'list': ['baz', 'bar'] + }, + 'level': 2, + }, + 'was_list': {'replaced': True} + }) + + +class MergeTest(unittest.TestCase): + + def test_merge_to_dicts(self): + source = {'a': 1, 'b': 2} + dest = {'b': '20', 'c': 30} + expected = { + 'a': 1, # source only + 'b': 2, # source overrides dest + 'c': 30, # dest only + } + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_to_lists(self): + source = {'x': [3, 4]} + dest = {'x': [1, 2]} + expected = {'x': [1, 2, 3, 4]} # dest first + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_diff_types(self): + source = {'x': [1, 2]} + dest = {'x': 'abc'} + expected = {'x': [1, 2]} # source wins + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge(self): + first = {'a': 1, 'b': 2, 'd': 11} + second = {'b': 20, 'c': 30} + third = {'c': 300, 'd': 400} + expected = { + 'a': 1, + 'b': 20, + 'c': 300, + 'd': 400, + } + self.assertEqual(merge(first, second, third), expected) + + # inputs haven't changed.. + self.assertEqual(first, {'a': 1, 'b': 2, 'd': 11}) + self.assertEqual(second, {'b': 20, 'c': 30}) + self.assertEqual(third, {'c': 300, 'd': 400}) + +if __name__ == '__main__': + mozunit.main() diff --git a/taskcluster/taskgraph/test/test_util_time.py b/taskcluster/taskgraph/test/test_util_time.py new file mode 100755 index 000000000..f001c9d9c --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_time.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +from datetime import datetime +from taskgraph.util.time import ( + InvalidString, + UnknownTimeMeasurement, + value_of, + json_time_from_now +) + + +class FromNowTest(unittest.TestCase): + + def test_invalid_str(self): + with self.assertRaises(InvalidString): + value_of('wtfs') + + def test_missing_unit(self): + with self.assertRaises(InvalidString): + value_of('1') + + def test_missing_unknown_unit(self): + with self.assertRaises(UnknownTimeMeasurement): + value_of('1z') + + def test_value_of(self): + self.assertEqual(value_of('1s').total_seconds(), 1) + self.assertEqual(value_of('1 second').total_seconds(), 1) + self.assertEqual(value_of('1min').total_seconds(), 60) + self.assertEqual(value_of('1h').total_seconds(), 3600) + self.assertEqual(value_of('1d').total_seconds(), 86400) + self.assertEqual(value_of('1mo').total_seconds(), 2592000) + self.assertEqual(value_of('1 month').total_seconds(), 2592000) + self.assertEqual(value_of('1y').total_seconds(), 31536000) + + with self.assertRaises(UnknownTimeMeasurement): + value_of('1m').total_seconds() # ambiguous between minute and month + + def test_json_from_now_utc_now(self): + # Just here to ensure we don't raise. + json_time_from_now('1 years') + + def test_json_from_now(self): + now = datetime(2014, 1, 1) + self.assertEqual(json_time_from_now('1 years', now), + '2015-01-01T00:00:00Z') + self.assertEqual(json_time_from_now('6 days', now), + '2014-01-07T00:00:00Z') + +if __name__ == '__main__': + mozunit.main() diff --git a/taskcluster/taskgraph/test/test_util_treeherder.py b/taskcluster/taskgraph/test/test_util_treeherder.py new file mode 100644 index 000000000..cf7513c00 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_treeherder.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from taskgraph.util.treeherder import split_symbol, join_symbol + + +class TestSymbols(unittest.TestCase): + + def test_split_no_group(self): + self.assertEqual(split_symbol('xy'), ('?', 'xy')) + + def test_split_with_group(self): + self.assertEqual(split_symbol('ab(xy)'), ('ab', 'xy')) + + def test_join_no_group(self): + self.assertEqual(join_symbol('?', 'xy'), 'xy') + + def test_join_with_group(self): + self.assertEqual(join_symbol('ab', 'xy'), 'ab(xy)') diff --git a/taskcluster/taskgraph/test/test_util_yaml.py b/taskcluster/taskgraph/test/test_util_yaml.py new file mode 100644 index 000000000..d4ff410db --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_yaml.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..util import yaml +from mozunit import MockedOpen + +FOO_YML = """\ +prop: + - val1 +""" + + +class TestYaml(unittest.TestCase): + + def test_load(self): + with MockedOpen({'/dir1/dir2/foo.yml': FOO_YML}): + self.assertEqual(yaml.load_yaml("/dir1/dir2", "foo.yml"), + {'prop': ['val1']}) diff --git a/taskcluster/taskgraph/test/util.py b/taskcluster/taskgraph/test/util.py new file mode 100644 index 000000000..cf9a49ad3 --- /dev/null +++ b/taskcluster/taskgraph/test/util.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from ..task import base + + +class TestTask(base.Task): + + def __init__(self, kind=None, label=None, attributes=None, task=None): + super(TestTask, self).__init__( + kind or 'test', + label or 'test-label', + attributes or {}, + task or {}) + + @classmethod + def load_tasks(cls, kind, path, config, parameters): + return [] + + def get_dependencies(self, taskgraph): + return [] |